From eb61a36fd7bd74f8d4ae448703db55c087aaad2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 11 Mar 2020 02:57:52 +0100 Subject: [PATCH 01/45] workers: RPC scaffolding --- api/api_storage.go | 24 ++++-- api/api_worker.go | 19 +++++ api/apistruct/struct.go | 82 +++++++++++++++---- api/client/client.go | 10 +++ chain/vm/syscalls.go | 3 +- cli/cmd.go | 1 - cmd/lotus-seal-worker/main.go | 89 ++++++++++---------- cmd/lotus-seal-worker/rpc.go | 35 ++++++++ cmd/lotus-seal-worker/storage.go | 22 +++++ cmd/lotus-seal-worker/sub.go | 132 ------------------------------ go.mod | 2 + node/impl/storminer.go | 25 ++++-- storage/sealmgr/advmgr/local.go | 4 +- storage/sealmgr/advmgr/manager.go | 19 ++++- storage/sealmgr/advmgr/remote.go | 46 +++++++++++ 15 files changed, 299 insertions(+), 214 deletions(-) create mode 100644 api/api_worker.go create mode 100644 cmd/lotus-seal-worker/rpc.go create mode 100644 cmd/lotus-seal-worker/storage.go delete mode 100644 cmd/lotus-seal-worker/sub.go create mode 100644 storage/sealmgr/advmgr/remote.go diff --git a/api/api_storage.go b/api/api_storage.go index 6b3705b79..180e8892a 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/chain/types" ) @@ -107,16 +108,12 @@ type StorageMiner interface { SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error - /*WorkerStats(context.Context) (sealsched.WorkerStats, error)*/ + // WorkerConnect tells the node to connect to workers RPC + WorkerConnect(context.Context, string) error + WorkerAttachStorage(context.Context, StorageInfo) error + WorkerDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error + WorkerFindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]StorageInfo, error) - /*// WorkerQueue registers a remote worker - WorkerQueue(context.Context, WorkerCfg) (<-chan WorkerTask, error) - - // WorkerQueue registers a remote worker - WorkerQueue(context.Context, sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) - - WorkerDone(ctx context.Context, task uint64, res sectorbuilder.SealRes) error - */ MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) @@ -128,6 +125,15 @@ type StorageMiner interface { StorageAddLocal(ctx context.Context, path string) error } +type StorageInfo struct { + ID string + URLs []string // TODO: Support non-http transports + Cost int + + CanSeal bool + CanStore bool +} + type SealRes struct { Err string GoErr error `json:"-"` diff --git a/api/api_worker.go b/api/api_worker.go new file mode 100644 index 000000000..f3dbce9aa --- /dev/null +++ b/api/api_worker.go @@ -0,0 +1,19 @@ +package api + +import ( + "context" + + "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/storage/sealmgr" +) + +type WorkerApi interface { + Version(context.Context) (build.Version, error) + // TODO: Info() (name, ...) ? + + TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) // TaskType -> Weight + + storage.Sealer +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 9eb73c637..4f037dced 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -3,6 +3,7 @@ package apistruct import ( "context" + "github.com/filecoin-project/specs-storage/storage" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" @@ -16,9 +17,12 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/storage/sealmgr" ) // All permissions are listed in permissioned.go @@ -167,6 +171,7 @@ type StorageMinerStruct struct { MarketImportDealData func(context.Context, cid.Cid, string) error `perm:"write"` MarketListDeals func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error) `perm:"read"` + /* Market */ SetPrice func(context.Context, types.BigInt) error `perm:"admin"` PledgeSector func(context.Context) error `perm:"write"` @@ -175,12 +180,10 @@ type StorageMinerStruct struct { SectorsRefs func(context.Context) (map[string][]api.SealedRef, error) `perm:"read"` SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error `perm:"write"` - /* WorkerStats func(context.Context) (sectorbuilder.WorkerStats, error) `perm:"read"` - - WorkerQueue func(ctx context.Context, cfg sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) `perm:"admin"` // TODO: worker perm - WorkerDone func(ctx context.Context, task uint64, res sectorbuilder.SealRes) error `perm:"admin"` - */ - SetPrice func(context.Context, types.BigInt) error `perm:"admin"` + WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm + WorkerAttachStorage func(context.Context, api.StorageInfo) error `perm:"admin"` + WorkerDeclareSector func(ctx context.Context, storageId string, s abi.SectorID) error `perm:"admin"` + WorkerFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]api.StorageInfo, error) `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -189,6 +192,22 @@ type StorageMinerStruct struct { } } +type WorkerStruct struct { + Internal struct { + // TODO: lower perms + + Version func(context.Context) (build.Version, error) `perm:"admin"` + + TaskTypes func(context.Context) (map[sealmgr.TaskType]struct{}, error) `perm:"admin"` + + SealPreCommit1 func(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"` + SealPreCommit2 func(context.Context, abi.SectorNumber, storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) `perm:"admin"` + SealCommit1 func(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (storage.Commit1Out, error) `perm:"admin"` + SealCommit2 func(context.Context, abi.SectorNumber, storage.Commit1Out) (storage.Proof, error) `perm:"admin"` + FinalizeSector func(context.Context, abi.SectorNumber) error `perm:"admin"` + } +} + func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]api.Permission, error) { return c.Internal.AuthVerify(ctx, token) } @@ -627,17 +646,21 @@ func (c *StorageMinerStruct) SectorsUpdate(ctx context.Context, id abi.SectorNum return c.Internal.SectorsUpdate(ctx, id, state) } -/*func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (sealsched.WorkerStats, error) { - return c.Internal.WorkerStats(ctx) -}*/ - -/*func (c *StorageMinerStruct) WorkerQueue(ctx context.Context, cfg sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) { - return c.Internal.WorkerQueue(ctx, cfg) +func (c *StorageMinerStruct) WorkerConnect(ctx context.Context, url string) error { + return c.Internal.WorkerConnect(ctx, url) } -func (c *StorageMinerStruct) WorkerDone(ctx context.Context, task uint64, res sectorbuilder.SealRes) error { - return c.Internal.WorkerDone(ctx, task, res) -}*/ +func (c *StorageMinerStruct) WorkerAttachStorage(ctx context.Context, si api.StorageInfo) error { + return c.Internal.WorkerAttachStorage(ctx, si) +} + +func (c *StorageMinerStruct) WorkerDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error { + return c.Internal.WorkerDeclareSector(ctx, storageId, s) +} + +func (c *StorageMinerStruct) WorkerFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]api.StorageInfo, error) { + return c.Internal.WorkerFindSector(ctx, si, types) +} func (c *StorageMinerStruct) MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error { return c.Internal.MarketImportDealData(ctx, propcid, path) @@ -667,6 +690,35 @@ func (c *StorageMinerStruct) StorageAddLocal(ctx context.Context, path string) e return c.Internal.StorageAddLocal(ctx, path) } +func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) { + return w.Internal.Version(ctx) +} + +func (w *WorkerStruct) TaskTypes(ctx context.Context) (map[sealmgr.TaskType]struct{}, error) { + return w.Internal.TaskTypes(ctx) +} + +func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) { + return w.Internal.SealPreCommit1(ctx, sectorNum, ticket, pieces) +} + +func (w *WorkerStruct) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, p1o storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { + return w.Internal.SealPreCommit2(ctx, sectorNum, p1o) +} + +func (w *WorkerStruct) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (storage.Commit1Out, error) { + return w.Internal.SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID) +} + +func (w *WorkerStruct) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, c1o storage.Commit1Out) (storage.Proof, error) { + return w.Internal.SealCommit2(ctx, sectorNum, c1o) +} + +func (w *WorkerStruct) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { + return w.Internal.FinalizeSector(ctx, sectorNum) +} + var _ api.Common = &CommonStruct{} var _ api.FullNode = &FullNodeStruct{} var _ api.StorageMiner = &StorageMinerStruct{} +var _ api.WorkerApi = &WorkerStruct{} diff --git a/api/client/client.go b/api/client/client.go index 0e19f65c2..0380d45b5 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -42,3 +42,13 @@ func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMine return &res, closer, err } + +func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerApi, jsonrpc.ClientCloser, error) { + var res apistruct.WorkerStruct + closer, err := jsonrpc.NewMergeClient(addr, "Filecoin", + []interface{}{ + &res.Internal, + }, requestHeader) + + return &res, closer, err +} diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 63438d0c1..720aa3adf 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -5,13 +5,14 @@ import ( "fmt" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" "golang.org/x/xerrors" + + "github.com/filecoin-project/go-sectorbuilder" ) func init() { diff --git a/cli/cmd.go b/cli/cmd.go index 48b93fb58..fc312f191 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -134,7 +134,6 @@ func GetAPIInfo(ctx *cli.Context, t repo.RepoType) (APIInfo, error) { } func GetRawAPI(ctx *cli.Context, t repo.RepoType) (string, http.Header, error) { - ainfo, err := GetAPIInfo(ctx, t) if err != nil { return "", nil, xerrors.Errorf("could not get API info: %w", err) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index efa4aa015..b44abc113 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -4,9 +4,15 @@ import ( "os" logging "github.com/ipfs/go-log/v2" + manet "github.com/multiformats/go-multiaddr-net" + "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" + paramfetch "github.com/filecoin-project/go-paramfetch" + + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" + lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/node/repo" ) @@ -75,60 +81,53 @@ var runCmd = &cli.Command{ Name: "run", Usage: "Start lotus worker", Action: func(cctx *cli.Context) error { - /* if !cctx.Bool("enable-gpu-proving") { - os.Setenv("BELLMAN_NO_GPU", "true") - } + if !cctx.Bool("enable-gpu-proving") { + os.Setenv("BELLMAN_NO_GPU", "true") + } - nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return xerrors.Errorf("getting miner api: %w", err) - } - defer closer() - ctx := lcli.ReqContext(cctx) + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return xerrors.Errorf("getting miner api: %w", err) + } + defer closer() + ctx := lcli.ReqContext(cctx) - ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) - if err != nil { - return xerrors.Errorf("could not get api info: %w", err) - } - _, storageAddr, err := manet.DialArgs(ainfo.Addr) + v, err := nodeApi.Version(ctx) + if err != nil { + return err + } + if v.APIVersion != build.APIVersion { + return xerrors.Errorf("lotus-storage-miner API version doesn't match: local: ", api.Version{APIVersion: build.APIVersion}) + } - r, err := homedir.Expand(cctx.String("repo")) - if err != nil { - return err - } + go func() { + <-ctx.Done() + log.Warn("Shutting down..") + }() - v, err := nodeApi.Version(ctx) - if err != nil { - return err - } - if v.APIVersion != build.APIVersion { - return xerrors.Errorf("lotus-storage-miner API version doesn't match: local: ", api.Version{APIVersion: build.APIVersion}) - } + act, err := nodeApi.ActorAddress(ctx) + if err != nil { + return err + } + ssize, err := nodeApi.ActorSectorSize(ctx, act) + if err != nil { + return err + } - go func() { - <-ctx.Done() - log.Warn("Shutting down..") - }() + if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil { + return xerrors.Errorf("get params: %w", err) + } - limiter := &limits{ - workLimit: make(chan struct{}, workers), - transferLimit: make(chan struct{}, transfers), - } + ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) + if err != nil { + return xerrors.Errorf("could not get api info: %w", err) + } + _, storageAddr, err := manet.DialArgs(ainfo.Addr) - act, err := nodeApi.ActorAddress(ctx) - if err != nil { - return err - } - ssize, err := nodeApi.ActorSectorSize(ctx, act) - if err != nil { - return err - } - if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil { - return xerrors.Errorf("get params: %w", err) - } - /*ppt, spt, err := api.ProofTypeFromSectorSize(ssize) + /* + /*ppt, spt, err := api.ProofTypeFromSectorSize(ssize) if err != nil { return err } diff --git a/cmd/lotus-seal-worker/rpc.go b/cmd/lotus-seal-worker/rpc.go new file mode 100644 index 000000000..5f8b5e7b1 --- /dev/null +++ b/cmd/lotus-seal-worker/rpc.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-storage/storage" + "github.com/ipfs/go-cid" +) + +type worker struct { + +} + +func (w *worker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) { + panic("implement me") +} + +func (w *worker) SealPreCommit2(context.Context, abi.SectorNumber, storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { + panic("implement me") +} + +func (w *worker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (storage.Commit1Out, error) { + panic("implement me") +} + +func (w *worker) SealCommit2(context.Context, abi.SectorNumber, storage.Commit1Out) (storage.Proof, error) { + panic("implement me") +} + +func (w *worker) FinalizeSector(context.Context, abi.SectorNumber) error { + panic("implement me") +} + +var _ storage.Sealer = &worker{} diff --git a/cmd/lotus-seal-worker/storage.go b/cmd/lotus-seal-worker/storage.go new file mode 100644 index 000000000..cb2775597 --- /dev/null +++ b/cmd/lotus-seal-worker/storage.go @@ -0,0 +1,22 @@ +package main + +import ( + "context" + + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/api" +) + +type workerStorage struct { + path string // TODO: multi-path support + + api api.StorageMiner +} + +func (w *workerStorage) AcquireSector(ctx context.Context, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { + w.api.WorkerFindSector() +} + +var _ sectorbuilder.SectorProvider = &workerStorage{} diff --git a/cmd/lotus-seal-worker/sub.go b/cmd/lotus-seal-worker/sub.go deleted file mode 100644 index d11df2acc..000000000 --- a/cmd/lotus-seal-worker/sub.go +++ /dev/null @@ -1,132 +0,0 @@ -package main - -/* -import ( - "context" - "net/http" - - "github.com/filecoin-project/go-sectorbuilder" - "golang.org/x/xerrors" - - lapi "github.com/filecoin-project/lotus/api" -) - -type worker struct { - api lapi.StorageMiner - minerEndpoint string - repo string - auth http.Header - - limiter *limits - sb *sectorbuilder.SectorBuilder -} - -func acceptJobs(ctx context.Context, api lapi.StorageMiner, sb *sectorbuilder.SectorBuilder, limiter *limits, endpoint string, auth http.Header, repo string, noprecommit, nocommit bool) error { - w := &worker{ - api: api, - minerEndpoint: endpoint, - auth: auth, - repo: repo, - - limiter: limiter, - sb: sb, - } - - tasks, err := api.WorkerQueue(ctx, sectorbuilder.WorkerCfg{ - NoPreCommit: noprecommit, - NoCommit: nocommit, - }) - if err != nil { - return err - } - -loop: - for { - log.Infof("Waiting for new task") - - select { - case task := <-tasks: - log.Infof("New task: %d, sector %d, action: %d", task.TaskID, task.SectorNum, task.Type) - - res := w.processTask(ctx, task) - - log.Infof("Task %d done, err: %+v", task.TaskID, res.GoErr) - - if err := api.WorkerDone(ctx, task.TaskID, res); err != nil { - log.Error(err) - } - case <-ctx.Done(): - break loop - } - } - - log.Warn("acceptJobs exit") - return nil -} - -func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask) sectorbuilder.SealRes { - switch task.Type { - case sectorbuilder.WorkerPreCommit: - case sectorbuilder.WorkerCommit: - default: - return errRes(xerrors.Errorf("unknown task type %d", task.Type)) - } - - if err := w.fetchSector(task.SectorNum, task.Type); err != nil { - return errRes(xerrors.Errorf("fetching sector: %w", err)) - } - - log.Infof("Data fetched, starting computation") - - var res sectorbuilder.SealRes - - switch task.Type { - case sectorbuilder.WorkerPreCommit: - w.limiter.workLimit <- struct{}{} - sealedCid, unsealedCid, err := w.sb.SealPreCommit(ctx, task.SectorNum, task.SealTicket, task.Pieces) - <-w.limiter.workLimit - - if err != nil { - return errRes(xerrors.Errorf("precomitting: %w", err)) - } - res.Rspco.CommD = unsealedCid - res.Rspco.CommR = sealedCid - - if err := w.push("sealed", task.SectorNum); err != nil { - return errRes(xerrors.Errorf("pushing precommited data: %w", err)) - } - - if err := w.push("cache", task.SectorNum); err != nil { - return errRes(xerrors.Errorf("pushing precommited data: %w", err)) - } - - if err := w.remove("staging", task.SectorNum); err != nil { - return errRes(xerrors.Errorf("cleaning up staged sector: %w", err)) - } - case sectorbuilder.WorkerCommit: - w.limiter.workLimit <- struct{}{} - proof, err := w.sb.SealCommit(ctx, task.SectorNum, task.SealTicket, task.SealSeed, task.Pieces, task.SealedCID, task.UnsealedCID) - <-w.limiter.workLimit - - if err != nil { - return errRes(xerrors.Errorf("comitting: %w", err)) - } - - res.Proof = proof - - if err := w.push("cache", task.SectorNum); err != nil { - return errRes(xerrors.Errorf("pushing precommited data: %w", err)) - } - - if err := w.remove("sealed", task.SectorNum); err != nil { - return errRes(xerrors.Errorf("cleaning up sealed sector: %w", err)) - } - } - - return res -} - -func errRes(err error) sectorbuilder.SealRes { - return sectorbuilder.SealRes{Err: err.Error(), GoErr: err} -} -*/ diff --git a/go.mod b/go.mod index 1ab22a12a..822874a80 100644 --- a/go.mod +++ b/go.mod @@ -117,3 +117,5 @@ replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi replace github.com/coreos/go-systemd => github.com/coreos/go-systemd/v22 v22.0.0 + +replace github.com/filecoin-project/go-sectorbuilder => /home/magik6k/gohack/github.com/filecoin-project/go-sectorbuilder diff --git a/node/impl/storminer.go b/node/impl/storminer.go index f4bee869b..38b7cddc9 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -13,7 +13,9 @@ import ( "github.com/filecoin-project/go-address" storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/lotus/api" @@ -256,15 +258,26 @@ func (sm *StorageMinerAPI) SectorsUpdate(ctx context.Context, id abi.SectorNumbe return sm.Miner.ForceSectorState(ctx, id, state) } -/* -func (sm *StorageMinerAPI) WorkerQueue(ctx context.Context, cfg sectorbuilder.WorkerCfg) (<-chan sectorbuilder.WorkerTask, error) { - return sm.SectorBuilder.AddWorker(ctx, cfg) +func (sm *StorageMinerAPI) WorkerConnect(ctx context.Context, url string) error { + _, err := advmgr.ConnectRemote(ctx, sm.Full, url) + if err != nil { + return err + } + + panic("todo register ") } -func (sm *StorageMinerAPI) WorkerDone(ctx context.Context, task uint64, res sectorbuilder.SealRes) error { - return sm.SectorBuilder.TaskDone(ctx, task, res) +func (sm *StorageMinerAPI) WorkerAttachStorage(ctx context.Context, si api.StorageInfo) error { + panic("implement me") +} + +func (sm *StorageMinerAPI) WorkerDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error { + panic("implement me") +} + +func (sm *StorageMinerAPI) WorkerFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]api.StorageInfo, error) { + panic("implement me") } -*/ func (sm *StorageMinerAPI) MarketImportDealData(ctx context.Context, propCid cid.Cid, path string) error { fi, err := os.Open(path) diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go index 63f852d7b..44e15abad 100644 --- a/storage/sealmgr/advmgr/local.go +++ b/storage/sealmgr/advmgr/local.go @@ -91,13 +91,13 @@ func (l *localWorker) FinalizeSector(ctx context.Context, sectorNum abi.SectorNu return sb.FinalizeSector(ctx, sectorNum) } -func (l *localWorker) TaskTypes() map[sealmgr.TaskType]struct{} { +func (l *localWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) { return map[sealmgr.TaskType]struct{}{ sealmgr.TTAddPiece: {}, sealmgr.TTPreCommit1: {}, sealmgr.TTPreCommit2: {}, sealmgr.TTCommit2: {}, - } + }, nil } func (l *localWorker) Paths() []Path { diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 9a8a6c5a4..c97b5ebeb 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -5,6 +5,7 @@ import ( "io" "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" @@ -17,6 +18,8 @@ import ( "github.com/filecoin-project/lotus/storage/sealmgr" ) +var log = logging.Logger("advmgr") + type SectorIDCounter interface { Next() (abi.SectorNumber, error) } @@ -39,7 +42,7 @@ type Path struct { type Worker interface { sectorbuilder.Sealer - TaskTypes() map[sealmgr.TaskType]struct{} + TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) Paths() []Path } @@ -122,7 +125,12 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []config.Stor paths := map[int]config.StorageMeta{} for i, worker := range m.workers { - if _, ok := worker.TaskTypes()[task]; !ok { + tt, err := worker.TaskTypes(context.TODO()) + if err != nil { + log.Errorf("error getting supported worker task types: %+v", err) + continue + } + if _, ok := tt[task]; !ok { continue } @@ -221,7 +229,12 @@ func (m *Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, t func (m *Manager) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) { for _, worker := range m.workers { - if _, ok := worker.TaskTypes()[sealmgr.TTCommit2]; !ok { + tt, err := worker.TaskTypes(context.TODO()) + if err != nil { + log.Errorf("error getting supported worker task types: %+v", err) + continue + } + if _, ok := tt[sealmgr.TTCommit2]; !ok { continue } diff --git a/storage/sealmgr/advmgr/remote.go b/storage/sealmgr/advmgr/remote.go new file mode 100644 index 000000000..78d7ee81d --- /dev/null +++ b/storage/sealmgr/advmgr/remote.go @@ -0,0 +1,46 @@ +package advmgr + +import ( + "context" + "net/http" + + "github.com/filecoin-project/specs-actors/actors/abi" + storage2 "github.com/filecoin-project/specs-storage/storage" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apistruct" + "github.com/filecoin-project/lotus/api/client" +) + +type remote struct { + api.WorkerApi +} + +func (r *remote) AddPiece(ctx context.Context, sector abi.SectorNumber, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage2.Data) (abi.PieceInfo, error) { + panic("implement me") +} + +func (r *remote) Paths() []Path { + panic("implement me") +} + +func ConnectRemote(ctx context.Context, fa api.FullNode, url string) (*remote, error) { + token, err := fa.AuthNew(ctx, []api.Permission{apistruct.PermAdmin}) + if err != nil { + return nil, xerrors.Errorf("creating auth token for remote connection: %w", err) + } + + headers := http.Header{} + headers.Add("Authorization", "Bearer "+string(token)) + + wapi, close, err := client.NewWorkerRPC(url, headers) + if err != nil { + return nil, xerrors.Errorf("creating jsonrpc client: %w", err) + } + _ = close // TODO + + return &remote{wapi}, nil +} + +var _ Worker = &remote{} From 7e2e30f94b52c56b5f1f6a095d9340115aa04cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 11 Mar 2020 06:49:17 +0100 Subject: [PATCH 02/45] workers: Transfer logic refactoring --- api/api_storage.go | 10 ++ api/api_worker.go | 1 + api/apistruct/struct.go | 5 + cmd/lotus-seal-worker/rpc.go | 5 +- cmd/lotus-seal-worker/storage.go | 24 ++++- cmd/lotus-seal-worker/transfer.go | 130 ++---------------------- node/impl/storminer.go | 118 +-------------------- storage/sealmgr/advmgr/local.go | 10 +- storage/sealmgr/advmgr/manager.go | 38 ++++--- storage/sealmgr/advmgr/remote.go | 9 +- storage/sealmgr/advmgr/storage.go | 102 ++++++++++++++++++- storage/sealmgr/advmgr/storage_local.go | 10 ++ 12 files changed, 185 insertions(+), 277 deletions(-) create mode 100644 storage/sealmgr/advmgr/storage_local.go diff --git a/api/api_storage.go b/api/api_storage.go index 180e8892a..be912401b 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -134,6 +134,16 @@ type StorageInfo struct { CanStore bool } +type StoragePath struct { + ID string + Weight uint64 + + LocalPath string + + CanSeal bool + CanStore bool +} + type SealRes struct { Err string GoErr error `json:"-"` diff --git a/api/api_worker.go b/api/api_worker.go index f3dbce9aa..67a22f77a 100644 --- a/api/api_worker.go +++ b/api/api_worker.go @@ -14,6 +14,7 @@ type WorkerApi interface { // TODO: Info() (name, ...) ? TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) // TaskType -> Weight + Paths(context.Context) ([]StoragePath, error) storage.Sealer } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 4f037dced..e978daa29 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -199,6 +199,7 @@ type WorkerStruct struct { Version func(context.Context) (build.Version, error) `perm:"admin"` TaskTypes func(context.Context) (map[sealmgr.TaskType]struct{}, error) `perm:"admin"` + Paths func(context.Context) ([]api.StoragePath, error) `perm:"admin"` SealPreCommit1 func(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"` SealPreCommit2 func(context.Context, abi.SectorNumber, storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) `perm:"admin"` @@ -698,6 +699,10 @@ func (w *WorkerStruct) TaskTypes(ctx context.Context) (map[sealmgr.TaskType]stru return w.Internal.TaskTypes(ctx) } +func (w *WorkerStruct) Paths(ctx context.Context) ([]api.StoragePath, error) { + return w.Internal.Paths(ctx) +} + func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) { return w.Internal.SealPreCommit1(ctx, sectorNum, ticket, pieces) } diff --git a/cmd/lotus-seal-worker/rpc.go b/cmd/lotus-seal-worker/rpc.go index 5f8b5e7b1..0acf9cd5f 100644 --- a/cmd/lotus-seal-worker/rpc.go +++ b/cmd/lotus-seal-worker/rpc.go @@ -3,13 +3,14 @@ package main import ( "context" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-storage/storage" - "github.com/ipfs/go-cid" ) type worker struct { - + spt abi.RegisteredProof } func (w *worker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) { diff --git a/cmd/lotus-seal-worker/storage.go b/cmd/lotus-seal-worker/storage.go index cb2775597..36bb560da 100644 --- a/cmd/lotus-seal-worker/storage.go +++ b/cmd/lotus-seal-worker/storage.go @@ -2,6 +2,8 @@ package main import ( "context" + "net/http" + "sort" "github.com/filecoin-project/specs-actors/actors/abi" @@ -11,12 +13,32 @@ import ( type workerStorage struct { path string // TODO: multi-path support + mid abi.ActorID + auth http.Header api api.StorageMiner } func (w *workerStorage) AcquireSector(ctx context.Context, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { - w.api.WorkerFindSector() + asid := abi.SectorID{ + Miner: w.mid, + Number: id, + } + + // extract local storage; prefer + + si, err := w.api.WorkerFindSector(ctx, asid, existing) + if err != nil { + return sectorbuilder.SectorPaths{}, nil, err + } + + sort.Slice(si, func(i, j int) bool { + return si[i].Cost < si[j].Cost + }) + + best := si[0].URLs // TODO: not necessarily true + + w.fetch(best, ) } var _ sectorbuilder.SectorProvider = &workerStorage{} diff --git a/cmd/lotus-seal-worker/transfer.go b/cmd/lotus-seal-worker/transfer.go index fe4a07dba..7a4a29e7e 100644 --- a/cmd/lotus-seal-worker/transfer.go +++ b/cmd/lotus-seal-worker/transfer.go @@ -1,37 +1,19 @@ package main -/* + import ( - "fmt" - "io" "mime" "net/http" "os" - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/go-sectorbuilder/fs" - "github.com/filecoin-project/specs-actors/actors/abi" files "github.com/ipfs/go-ipfs-files" "golang.org/x/xerrors" - "gopkg.in/cheggaaa/pb.v1" - "path/filepath" "github.com/filecoin-project/lotus/lib/tarutil" ) -func (w *worker) sizeForType(typ string) int64 { - size := int64(w.sb.SectorSize()) - if typ == "cache" { - size *= 10 - } - return size -} - -func (w *worker) fetch(typ string, sectorID abi.SectorNumber) error { - outname := filepath.Join(w.repo, typ, w.sb.SectorName(sectorID)) - - url := w.minerEndpoint + "/remote/" + typ + "/" + fmt.Sprint(sectorID) - log.Infof("Fetch %s %s", typ, url) +func (w *workerStorage) fetch(url, outname string) error { + log.Infof("Fetch %s", url) req, err := http.NewRequest("GET", url, nil) if err != nil { @@ -49,7 +31,7 @@ func (w *worker) fetch(typ string, sectorID abi.SectorNumber) error { return xerrors.Errorf("non-200 code: %d", resp.StatusCode) } - bar := pb.New64(w.sizeForType(typ)) + /*bar := pb.New64(w.sizeForType(typ)) bar.ShowPercent = true bar.ShowSpeed = true bar.Units = pb.U_BYTES @@ -57,7 +39,7 @@ func (w *worker) fetch(typ string, sectorID abi.SectorNumber) error { barreader := bar.NewProxyReader(resp.Body) bar.Start() - defer bar.Finish() + defer bar.Finish()*/ mediatype, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) if err != nil { @@ -70,109 +52,11 @@ func (w *worker) fetch(typ string, sectorID abi.SectorNumber) error { switch mediatype { case "application/x-tar": - return tarutil.ExtractTar(barreader, outname) + return tarutil.ExtractTar(resp.Body, outname) case "application/octet-stream": - return files.WriteTo(files.NewReaderFile(barreader), outname) + return files.WriteTo(files.NewReaderFile(resp.Body), outname) default: return xerrors.Errorf("unknown content type: '%s'", mediatype) } } - -func (w *worker) push(typ string, sectorID abi.SectorNumber) error { - w.limiter.transferLimit <- struct{}{} - defer func() { - <-w.limiter.transferLimit - }() - - filename, err := w.sb.SectorPath(fs.DataType(typ), sectorID) - if err != nil { - return err - } - - url := w.minerEndpoint + "/remote/" + typ + "/" + fmt.Sprint(sectorID) - log.Infof("Push %s %s", typ, url) - - stat, err := os.Stat(string(filename)) - if err != nil { - return err - } - - var r io.Reader - if stat.IsDir() { - r, err = tarutil.TarDirectory(string(filename)) - } else { - r, err = os.OpenFile(string(filename), os.O_RDONLY, 0644) - } - if err != nil { - return xerrors.Errorf("opening push reader: %w", err) - } - - bar := pb.New64(w.sizeForType(typ)) - bar.ShowPercent = true - bar.ShowSpeed = true - bar.ShowCounters = true - bar.Units = pb.U_BYTES - - bar.Start() - defer bar.Finish() - //todo set content size - - header := w.auth - - if stat.IsDir() { - header.Set("Content-Type", "application/x-tar") - } else { - header.Set("Content-Type", "application/octet-stream") - } - - req, err := http.NewRequest("PUT", url, bar.NewProxyReader(r)) - if err != nil { - return err - } - req.Header = header - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - if resp.StatusCode != 200 { - return xerrors.Errorf("non-200 response: %d", resp.StatusCode) - } - - if err := resp.Body.Close(); err != nil { - return err - } - - // TODO: keep files around for later stages of sealing - return w.remove(typ, sectorID) -} - -func (w *worker) remove(typ string, sectorID abi.SectorNumber) error { - filename := filepath.Join(w.repo, typ, w.sb.SectorName(sectorID)) - return os.RemoveAll(filename) -} - -func (w *worker) fetchSector(sectorID abi.SectorNumber, typ sectorbuilder.WorkerTaskType) error { - w.limiter.transferLimit <- struct{}{} - defer func() { - <-w.limiter.transferLimit - }() - - var err error - switch typ { - case sectorbuilder.WorkerPreCommit: - err = w.fetch("staging", sectorID) - case sectorbuilder.WorkerCommit: - err = w.fetch("sealed", sectorID) - if err != nil { - return xerrors.Errorf("fetch sealed: %w", err) - } - err = w.fetch("cache", sectorID) - } - if err != nil { - return xerrors.Errorf("fetch failed: %w", err) - } - return nil -} -*/ diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 38b7cddc9..a2d3dcb9a 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -7,7 +7,6 @@ import ( "os" "strconv" - "github.com/gorilla/mux" "github.com/ipfs/go-cid" "golang.org/x/xerrors" @@ -48,122 +47,7 @@ func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { return } - mux := mux.NewRouter() - - mux.HandleFunc("/remote/{type}/{id}", sm.remoteGetSector).Methods("GET") - mux.HandleFunc("/remote/{type}/{id}", sm.remotePutSector).Methods("PUT") - - log.Infof("SERVEGETREMOTE %s", r.URL) - - mux.ServeHTTP(w, r) -} - -func (sm *StorageMinerAPI) remoteGetSector(w http.ResponseWriter, r *http.Request) { - panic("todo") - /* vars := mux.Vars(r) - - id, err := strconv.ParseUint(vars["id"], 10, 64) - if err != nil { - log.Error("parsing sector id: ", err) - w.WriteHeader(500) - return - } - - path, err := sm.SectorBuilder.SectorPath(fs.DataType(vars["type"]), abi.SectorNumber(id)) - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - stat, err := os.Stat(string(path)) - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - var rd io.Reader - if stat.IsDir() { - rd, err = tarutil.TarDirectory(string(path)) - w.Header().Set("Content-Type", "application/x-tar") - } else { - rd, err = os.OpenFile(string(path), os.O_RDONLY, 0644) - w.Header().Set("Content-Type", "application/octet-stream") - } - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - w.WriteHeader(200) - if _, err := io.Copy(w, rd); err != nil { - log.Error(err) - return - }*/ -} - -func (sm *StorageMinerAPI) remotePutSector(w http.ResponseWriter, r *http.Request) { - panic("todo") - /* vars := mux.Vars(r) - - id, err := strconv.ParseUint(vars["id"], 10, 64) - if err != nil { - log.Error("parsing sector id: ", err) - w.WriteHeader(500) - return - } - - // This is going to get better with worker-to-worker transfers - - path, err := sm.SectorBuilder.SectorPath(fs.DataType(vars["type"]), abi.SectorNumber(id)) - if err != nil { - if err != fs.ErrNotFound { - log.Error(err) - w.WriteHeader(500) - return - } - - path, err = sm.SectorBuilder.AllocSectorPath(fs.DataType(vars["type"]), abi.SectorNumber(id), true) - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - } - - mediatype, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - if err := os.RemoveAll(string(path)); err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - switch mediatype { - case "application/x-tar": - if err := tarutil.ExtractTar(r.Body, string(path)); err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - default: - if err := files.WriteTo(files.NewReaderFile(r.Body), string(path)); err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - } - - w.WriteHeader(200) - - log.Infof("received %s sector (%s): %d bytes", vars["type"], vars["sname"], r.ContentLength)*/ + sm.StorageMgr.ServeHTTP(w, r) } /* diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go index 44e15abad..878a05f4f 100644 --- a/storage/sealmgr/advmgr/local.go +++ b/storage/sealmgr/advmgr/local.go @@ -4,13 +4,15 @@ import ( "context" "io" - storage2 "github.com/filecoin-project/specs-storage/storage" "github.com/ipfs/go-cid" "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" + storage2 "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/storage/sealmgr" ) @@ -100,8 +102,8 @@ func (l *localWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, }, nil } -func (l *localWorker) Paths() []Path { - return l.storage.local() +func (l *localWorker) Paths(context.Context) ([]api.StoragePath, error) { + return l.storage.local(), nil } var _ Worker = &localWorker{} diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index c97b5ebeb..2cee6a631 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -3,6 +3,7 @@ package advmgr import ( "context" "io" + "net/http" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" @@ -11,6 +12,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/specs-actors/actors/abi" storage2 "github.com/filecoin-project/specs-storage/storage" @@ -24,26 +27,11 @@ type SectorIDCounter interface { Next() (abi.SectorNumber, error) } -type LocalStorage interface { - GetStorage() (config.StorageConfig, error) - SetStorage(func(*config.StorageConfig)) error -} - -type Path struct { - ID string - Weight uint64 - - LocalPath string - - CanSeal bool - CanStore bool -} - type Worker interface { sectorbuilder.Sealer TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) - Paths() []Path + Paths(context.Context) ([]api.StoragePath, error) } type Manager struct { @@ -107,6 +95,10 @@ func (m *Manager) AddLocalStorage(path string) error { return nil } +func (m *Manager) ServeHTTP(w http.ResponseWriter, r *http.Request) { + m.storage.ServeHTTP(w, r) +} + func (m *Manager) SectorSize() abi.SectorSize { sz, _ := m.scfg.SealProofType.SectorSize() return sz @@ -134,16 +126,22 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []config.Stor continue } + phs, err := worker.Paths(context.TODO()) + if err != nil { + log.Errorf("error getting worker paths: %+v", err) + continue + } + // check if the worker has access to the path we selected var st *config.StorageMeta - for _, p := range worker.Paths() { - for _, m := range inPaths { - if p.ID == m.ID { + for _, p := range phs { + for _, meta := range inPaths { + if p.ID == meta.ID { if st != nil && st.Weight > p.Weight { continue } - p := m // copy + p := meta // copy st = &p } } diff --git a/storage/sealmgr/advmgr/remote.go b/storage/sealmgr/advmgr/remote.go index 78d7ee81d..f1eab1891 100644 --- a/storage/sealmgr/advmgr/remote.go +++ b/storage/sealmgr/advmgr/remote.go @@ -9,7 +9,6 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/api/client" ) @@ -18,15 +17,11 @@ type remote struct { } func (r *remote) AddPiece(ctx context.Context, sector abi.SectorNumber, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage2.Data) (abi.PieceInfo, error) { - panic("implement me") -} - -func (r *remote) Paths() []Path { - panic("implement me") + return abi.PieceInfo{},xerrors.New("unsupported") } func ConnectRemote(ctx context.Context, fa api.FullNode, url string) (*remote, error) { - token, err := fa.AuthNew(ctx, []api.Permission{apistruct.PermAdmin}) + token, err := fa.AuthNew(ctx, []api.Permission{"admin"}) if err != nil { return nil, xerrors.Errorf("creating auth token for remote connection: %w", err) } diff --git a/storage/sealmgr/advmgr/storage.go b/storage/sealmgr/advmgr/storage.go index 6d84a7e99..789180588 100644 --- a/storage/sealmgr/advmgr/storage.go +++ b/storage/sealmgr/advmgr/storage.go @@ -3,14 +3,20 @@ package advmgr import ( "encoding/json" "fmt" + "io" "io/ioutil" + "net/http" "os" "path/filepath" "sync" + "github.com/gorilla/mux" "golang.org/x/xerrors" "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/lib/tarutil" + "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/lotus/node/config" @@ -129,6 +135,9 @@ func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing if s&fileType == 0 { continue } + if p.local == "" { + continue // TODO: fetch + } spath := filepath.Join(p.local, fileType.String(), fmt.Sprintf("s-t0%d-%d", mid, id)) @@ -239,14 +248,14 @@ func (st *storage) findSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbu return out, nil } -func (st *storage) local() []Path { - var out []Path +func (st *storage) local() []api.StoragePath { + var out []api.StoragePath for _, p := range st.paths { if p.local == "" { continue } - out = append(out, Path{ + out = append(out, api.StoragePath{ ID: p.meta.ID, Weight: p.meta.Weight, LocalPath: p.local, @@ -258,6 +267,93 @@ func (st *storage) local() []Path { return out } +func (st *storage) ServeHTTP(w http.ResponseWriter, r *http.Request) { // /storage/ + mux := mux.NewRouter() + + mux.HandleFunc("/{type}/{id}", st.remoteGetSector).Methods("GET") + + log.Infof("SERVEGETREMOTE %s", r.URL) + + mux.ServeHTTP(w, r) +} + + +func (st *storage) remoteGetSector(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + id, err := parseSectorID(vars["id"]) + if err != nil { + log.Error(err) + w.WriteHeader(500) + return + } + + ft, err := ftFromString(vars["type"]) + if err != nil { + return + } + paths, done, err := st.acquireSector(id.Miner, id.Number, ft, 0, false) + if err != nil { + return + } + defer done() + + var path string + switch ft { + case sectorbuilder.FTUnsealed: + path = paths.Unsealed + case sectorbuilder.FTSealed: + path = paths.Sealed + case sectorbuilder.FTCache: + path = paths.Cache + } + if path == "" { + log.Error("acquired path was empty") + w.WriteHeader(500) + return + } + + stat, err := os.Stat(path) + if err != nil { + log.Error(err) + w.WriteHeader(500) + return + } + + var rd io.Reader + if stat.IsDir() { + rd, err = tarutil.TarDirectory(path) + w.Header().Set("Content-Type", "application/x-tar") + } else { + rd, err = os.OpenFile(path, os.O_RDONLY, 0644) + w.Header().Set("Content-Type", "application/octet-stream") + } + if err != nil { + log.Error(err) + w.WriteHeader(500) + return + } + + w.WriteHeader(200) + if _, err := io.Copy(w, rd); err != nil { // TODO: default 32k buf may be too small + log.Error(err) + return + } +} + +func ftFromString(t string) (sectorbuilder.SectorFileType, error) { + switch t { + case sectorbuilder.FTUnsealed.String(): + return sectorbuilder.FTUnsealed, nil + case sectorbuilder.FTSealed.String(): + return sectorbuilder.FTSealed, nil + case sectorbuilder.FTCache.String(): + return sectorbuilder.FTCache, nil + default: + return 0, xerrors.Errorf("unknown sector file type: '%s'", t) + } +} + func parseSectorID(baseName string) (abi.SectorID, error) { var n abi.SectorNumber var mid abi.ActorID diff --git a/storage/sealmgr/advmgr/storage_local.go b/storage/sealmgr/advmgr/storage_local.go new file mode 100644 index 000000000..3aba45733 --- /dev/null +++ b/storage/sealmgr/advmgr/storage_local.go @@ -0,0 +1,10 @@ +package advmgr + +import ( + "github.com/filecoin-project/lotus/node/config" +) + +type LocalStorage interface { + GetStorage() (config.StorageConfig, error) + SetStorage(func(*config.StorageConfig)) error +} From 71afcb03335147a7c79176a72c639f4144bc4dcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 11 Mar 2020 08:22:21 +0100 Subject: [PATCH 03/45] workrs: refactor sector store for more composability --- cmd/lotus-seal-worker/storage.go | 6 + node/builder.go | 5 +- storage/sealmgr/advmgr/local.go | 10 +- storage/sealmgr/advmgr/manager.go | 34 ++-- storage/sealmgr/advmgr/roprov.go | 9 +- storage/sealmgr/advmgr/storage_local.go | 10 -- storage/sealmgr/sectorutil/utils.go | 31 ++++ storage/sealmgr/stores/http_handler.go | 107 +++++++++++ storage/sealmgr/stores/interface.go | 11 ++ .../{advmgr/storage.go => stores/local.go} | 167 ++++-------------- 10 files changed, 223 insertions(+), 167 deletions(-) delete mode 100644 storage/sealmgr/advmgr/storage_local.go create mode 100644 storage/sealmgr/sectorutil/utils.go create mode 100644 storage/sealmgr/stores/http_handler.go create mode 100644 storage/sealmgr/stores/interface.go rename storage/sealmgr/{advmgr/storage.go => stores/local.go} (55%) diff --git a/cmd/lotus-seal-worker/storage.go b/cmd/lotus-seal-worker/storage.go index 36bb560da..6c86a5945 100644 --- a/cmd/lotus-seal-worker/storage.go +++ b/cmd/lotus-seal-worker/storage.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" ) type workerStorage struct { @@ -38,6 +39,11 @@ func (w *workerStorage) AcquireSector(ctx context.Context, id abi.SectorNumber, best := si[0].URLs // TODO: not necessarily true + sname := sectorutil.SectorName(abi.SectorID{ + Miner: w.mid, + Number: id, + }) + w.fetch(best, ) } diff --git a/node/builder.go b/node/builder.go index 4dca1bc6f..661edcccb 100644 --- a/node/builder.go +++ b/node/builder.go @@ -23,7 +23,10 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery" "github.com/filecoin-project/go-fil-markets/storagemarket" deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "github.com/filecoin-project/specs-actors/actors/runtime" storage2 "github.com/filecoin-project/specs-storage/storage" @@ -256,7 +259,7 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig), - Override(new(advmgr.LocalStorage), From(new(repo.LockedRepo))), + Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), Override(new(advmgr.SectorIDCounter), modules.SectorIDCounter), Override(new(*advmgr.Manager), advmgr.New), diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go index 878a05f4f..47b562e01 100644 --- a/storage/sealmgr/advmgr/local.go +++ b/storage/sealmgr/advmgr/local.go @@ -13,13 +13,14 @@ import ( "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" "github.com/filecoin-project/lotus/storage/sealmgr" ) type localWorker struct { scfg *sectorbuilder.Config - storage *storage + storage *stores.Local } type localWorkerPathProvider struct { @@ -32,7 +33,10 @@ func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, id abi.Sect return sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("get miner ID: %w", err) } - return l.w.storage.acquireSector(abi.ActorID(mid), id, existing, allocate, sealing) + return l.w.storage.AcquireSector(abi.SectorID{ + Miner: abi.ActorID(mid), + Number: id, + }, existing, allocate, sealing) } func (l *localWorker) sb() (sectorbuilder.Basic, error) { @@ -103,7 +107,7 @@ func (l *localWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, } func (l *localWorker) Paths(context.Context) ([]api.StoragePath, error) { - return l.storage.local(), nil + return l.storage.Local(), nil } var _ Worker = &localWorker{} diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 2cee6a631..d6e8404a1 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -11,8 +11,10 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" "github.com/filecoin-project/specs-actors/actors/abi" storage2 "github.com/filecoin-project/specs-storage/storage" @@ -39,16 +41,16 @@ type Manager struct { scfg *sectorbuilder.Config sc SectorIDCounter - storage *storage + ls stores.LocalStorage + storage *stores.Local + remoteHnd *stores.FetchHandler storage2.Prover } -func New(ls LocalStorage, cfg *sectorbuilder.Config, sc SectorIDCounter) (*Manager, error) { - stor := &storage{ - localStorage: ls, - } - if err := stor.open(); err != nil { +func New(ls stores.LocalStorage, cfg *sectorbuilder.Config, sc SectorIDCounter) (*Manager, error) { + stor, err := stores.NewLocal(ls) + if err != nil { return nil, err } @@ -69,7 +71,9 @@ func New(ls LocalStorage, cfg *sectorbuilder.Config, sc SectorIDCounter) (*Manag scfg: cfg, sc: sc, + ls: ls, storage: stor, + remoteHnd: &stores.FetchHandler{Store: stor}, Prover: prover, } @@ -83,11 +87,11 @@ func (m *Manager) AddLocalStorage(path string) error { return xerrors.Errorf("expanding local path: %w", err) } - if err := m.storage.openPath(path); err != nil { + if err := m.storage.OpenPath(path); err != nil { return xerrors.Errorf("opening local path: %w", err) } - if err := m.storage.localStorage.SetStorage(func(sc *config.StorageConfig) { + if err := m.ls.SetStorage(func(sc *config.StorageConfig) { sc.StoragePaths = append(sc.StoragePaths, config.LocalPath{Path: path}) }); err != nil { return xerrors.Errorf("get storage config: %w", err) @@ -96,7 +100,7 @@ func (m *Manager) AddLocalStorage(path string) error { } func (m *Manager) ServeHTTP(w http.ResponseWriter, r *http.Request) { - m.storage.ServeHTTP(w, r) + m.remoteHnd.ServeHTTP(w, r) } func (m *Manager) SectorSize() abi.SectorSize { @@ -163,9 +167,9 @@ func (m *Manager) AddPiece(ctx context.Context, sn abi.SectorNumber, existingPie var best []config.StorageMeta var err error if len(existingPieces) == 0 { // new - best, err = m.storage.findBestAllocStorage(sectorbuilder.FTUnsealed, true) + best, err = m.storage.FindBestAllocStorage(sectorbuilder.FTUnsealed, true) } else { // append to existing - best, err = m.storage.findSector(m.minerID(), sn, sectorbuilder.FTUnsealed) + best, err = m.storage.FindSector(m.minerID(), sn, sectorbuilder.FTUnsealed) } if err != nil { return abi.PieceInfo{}, xerrors.Errorf("finding sector path: %w", err) @@ -185,7 +189,7 @@ func (m *Manager) AddPiece(ctx context.Context, sn abi.SectorNumber, existingPie func (m *Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) { // TODO: also consider where the unsealed data sits - best, err := m.storage.findBestAllocStorage(sectorbuilder.FTCache|sectorbuilder.FTSealed, true) + best, err := m.storage.FindBestAllocStorage(sectorbuilder.FTCache|sectorbuilder.FTSealed, true) if err != nil { return nil, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -200,7 +204,7 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber func (m *Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { // TODO: allow workers to fetch the sectors - best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed) + best, err := m.storage.FindSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -213,7 +217,7 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber } func (m *Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output storage2.Commit1Out, err error) { - best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed) + best, err := m.storage.FindSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed) if err != nil { return nil, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -243,7 +247,7 @@ func (m *Manager) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, p } func (m *Manager) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { - best, err := m.storage.findSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed|sectorbuilder.FTUnsealed) + best, err := m.storage.FindSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed|sectorbuilder.FTUnsealed) if err != nil { return xerrors.Errorf("finding sealed sector: %w", err) } diff --git a/storage/sealmgr/advmgr/roprov.go b/storage/sealmgr/advmgr/roprov.go index 620ba6223..64513cdcd 100644 --- a/storage/sealmgr/advmgr/roprov.go +++ b/storage/sealmgr/advmgr/roprov.go @@ -4,13 +4,15 @@ import ( "context" "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "github.com/filecoin-project/specs-actors/actors/abi" "golang.org/x/xerrors" ) type readonlyProvider struct { miner abi.ActorID - stor *storage + stor *stores.Local } func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { @@ -18,5 +20,8 @@ func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorNumbe return sectorbuilder.SectorPaths{}, nil, xerrors.New("read-only storage") } - return l.stor.acquireSector(l.miner, id, existing, allocate, sealing) + return l.stor.AcquireSector(abi.SectorID{ + Miner: l.miner, + Number: id, + }, existing, allocate, sealing) } diff --git a/storage/sealmgr/advmgr/storage_local.go b/storage/sealmgr/advmgr/storage_local.go deleted file mode 100644 index 3aba45733..000000000 --- a/storage/sealmgr/advmgr/storage_local.go +++ /dev/null @@ -1,10 +0,0 @@ -package advmgr - -import ( - "github.com/filecoin-project/lotus/node/config" -) - -type LocalStorage interface { - GetStorage() (config.StorageConfig, error) - SetStorage(func(*config.StorageConfig)) error -} diff --git a/storage/sealmgr/sectorutil/utils.go b/storage/sealmgr/sectorutil/utils.go new file mode 100644 index 000000000..4c0d0328b --- /dev/null +++ b/storage/sealmgr/sectorutil/utils.go @@ -0,0 +1,31 @@ +package sectorutil + +import ( + "fmt" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +func ParseSectorID(baseName string) (abi.SectorID, error) { + var n abi.SectorNumber + var mid abi.ActorID + read, err := fmt.Sscanf(baseName, "s-t0%d-%d", &mid, &n) + if err != nil { + return abi.SectorID{}, xerrors.Errorf(": %w", err) + } + + if read != 2 { + return abi.SectorID{}, xerrors.Errorf("parseSectorID expected to scan 2 values, got %d", read) + } + + return abi.SectorID{ + Miner: mid, + Number: n, + }, nil +} + +func SectorName(sid abi.SectorID) string { + return fmt.Sprintf("s-t0%d-%d", sid.Miner, sid.Number) +} diff --git a/storage/sealmgr/stores/http_handler.go b/storage/sealmgr/stores/http_handler.go new file mode 100644 index 000000000..960a91d23 --- /dev/null +++ b/storage/sealmgr/stores/http_handler.go @@ -0,0 +1,107 @@ +package stores + +import ( + "io" + "net/http" + "os" + + "github.com/gorilla/mux" + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/lib/tarutil" + "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" +) + +var log = logging.Logger("stores") + +type FetchHandler struct { + Store +} + +func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // /storage/ + mux := mux.NewRouter() + + mux.HandleFunc("/{type}/{id}", handler.remoteGetSector).Methods("GET") + + log.Infof("SERVEGETREMOTE %s", r.URL) + + mux.ServeHTTP(w, r) +} + +func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + id, err := sectorutil.ParseSectorID(vars["id"]) + if err != nil { + log.Error(err) + w.WriteHeader(500) + return + } + + ft, err := ftFromString(vars["type"]) + if err != nil { + return + } + paths, done, err := handler.Store.AcquireSector(id, ft, 0, false) + if err != nil { + return + } + defer done() + + var path string + switch ft { + case sectorbuilder.FTUnsealed: + path = paths.Unsealed + case sectorbuilder.FTSealed: + path = paths.Sealed + case sectorbuilder.FTCache: + path = paths.Cache + } + if path == "" { + log.Error("acquired path was empty") + w.WriteHeader(500) + return + } + + stat, err := os.Stat(path) + if err != nil { + log.Error(err) + w.WriteHeader(500) + return + } + + var rd io.Reader + if stat.IsDir() { + rd, err = tarutil.TarDirectory(path) + w.Header().Set("Content-Type", "application/x-tar") + } else { + rd, err = os.OpenFile(path, os.O_RDONLY, 0644) + w.Header().Set("Content-Type", "application/octet-stream") + } + if err != nil { + log.Error(err) + w.WriteHeader(500) + return + } + + w.WriteHeader(200) + if _, err := io.Copy(w, rd); err != nil { // TODO: default 32k buf may be too small + log.Error(err) + return + } +} + +func ftFromString(t string) (sectorbuilder.SectorFileType, error) { + switch t { + case sectorbuilder.FTUnsealed.String(): + return sectorbuilder.FTUnsealed, nil + case sectorbuilder.FTSealed.String(): + return sectorbuilder.FTSealed, nil + case sectorbuilder.FTCache.String(): + return sectorbuilder.FTCache, nil + default: + return 0, xerrors.Errorf("unknown sector file type: '%s'", t) + } +} diff --git a/storage/sealmgr/stores/interface.go b/storage/sealmgr/stores/interface.go new file mode 100644 index 000000000..08dddad7a --- /dev/null +++ b/storage/sealmgr/stores/interface.go @@ -0,0 +1,11 @@ +package stores + +import ( + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/go-sectorbuilder" +) + +type Store interface { + AcquireSector(s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) +} diff --git a/storage/sealmgr/advmgr/storage.go b/storage/sealmgr/stores/local.go similarity index 55% rename from storage/sealmgr/advmgr/storage.go rename to storage/sealmgr/stores/local.go index 789180588..9b799ab6f 100644 --- a/storage/sealmgr/advmgr/storage.go +++ b/storage/sealmgr/stores/local.go @@ -1,32 +1,31 @@ -package advmgr +package stores import ( "encoding/json" - "fmt" - "io" "io/ioutil" - "net/http" "os" "path/filepath" "sync" - "github.com/gorilla/mux" + "github.com/filecoin-project/specs-actors/actors/abi" "golang.org/x/xerrors" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/lib/tarutil" - - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" ) -const metaFile = "sectorstore.json" +type LocalStorage interface { + GetStorage() (config.StorageConfig, error) + SetStorage(func(*config.StorageConfig)) error +} + +const MetaFile = "sectorstore.json" var pathTypes = []sectorbuilder.SectorFileType{sectorbuilder.FTUnsealed, sectorbuilder.FTSealed, sectorbuilder.FTCache} -type storage struct { +type Local struct { localLk sync.RWMutex localStorage LocalStorage @@ -42,8 +41,15 @@ type path struct { sectors map[abi.SectorID]sectorbuilder.SectorFileType } -func (st *storage) openPath(p string) error { - mb, err := ioutil.ReadFile(filepath.Join(p, metaFile)) +func NewLocal(ls LocalStorage) (*Local, error) { + l := &Local{ + localStorage: ls, + } + return l, l.open() +} + +func (st *Local) OpenPath(p string) error { + mb, err := ioutil.ReadFile(filepath.Join(p, MetaFile)) if err != nil { return xerrors.Errorf("reading storage metadata for %s: %w", p, err) } @@ -75,7 +81,7 @@ func (st *storage) openPath(p string) error { } for _, ent := range ents { - sid, err := parseSectorID(ent.Name()) + sid, err := sectorutil.ParseSectorID(ent.Name()) if err != nil { return xerrors.Errorf("parse sector id %s: %w", ent.Name(), err) } @@ -89,7 +95,7 @@ func (st *storage) openPath(p string) error { return nil } -func (st *storage) open() error { +func (st *Local) open() error { st.localLk.Lock() defer st.localLk.Unlock() @@ -99,7 +105,7 @@ func (st *storage) open() error { } for _, path := range cfg.StoragePaths { - err := st.openPath(path.Path) + err := st.OpenPath(path.Path) if err != nil { return xerrors.Errorf("opening path %s: %w", path.Path, err) } @@ -108,7 +114,7 @@ func (st *storage) open() error { return nil } -func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { +func (st *Local) AcquireSector(sid abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { if existing|allocate != existing^allocate { return sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector") } @@ -124,10 +130,7 @@ func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing for _, p := range st.paths { p.lk.Lock() - s, ok := p.sectors[abi.SectorID{ - Miner: mid, - Number: id, - }] + s, ok := p.sectors[sid] p.lk.Unlock() if !ok { continue @@ -136,10 +139,10 @@ func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing continue } if p.local == "" { - continue // TODO: fetch + continue } - spath := filepath.Join(p.local, fileType.String(), fmt.Sprintf("s-t0%d-%d", mid, id)) + spath := filepath.Join(p.local, fileType.String(), sectorutil.SectorName(sid)) switch fileType { case sectorbuilder.FTUnsealed: @@ -170,16 +173,13 @@ func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing } p.lk.Lock() - p.sectors[abi.SectorID{ - Miner: mid, - Number: id, - }] |= fileType + p.sectors[sid] |= fileType p.lk.Unlock() // TODO: Check free space // TODO: Calc weights - best = filepath.Join(p.local, fileType.String(), fmt.Sprintf("s-t0%d-%d", mid, id)) + best = filepath.Join(p.local, fileType.String(), sectorutil.SectorName(sid)) break // todo: the first path won't always be the best } @@ -203,7 +203,7 @@ func (st *storage) acquireSector(mid abi.ActorID, id abi.SectorNumber, existing return out, st.localLk.RUnlock, nil } -func (st *storage) findBestAllocStorage(allocate sectorbuilder.SectorFileType, sealing bool) ([]config.StorageMeta, error) { +func (st *Local) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sealing bool) ([]config.StorageMeta, error) { var out []config.StorageMeta for _, p := range st.paths { @@ -227,7 +227,7 @@ func (st *storage) findBestAllocStorage(allocate sectorbuilder.SectorFileType, s return out, nil } -func (st *storage) findSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbuilder.SectorFileType) ([]config.StorageMeta, error) { +func (st *Local) FindSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbuilder.SectorFileType) ([]config.StorageMeta, error) { var out []config.StorageMeta for _, p := range st.paths { p.lk.Lock() @@ -248,7 +248,7 @@ func (st *storage) findSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbu return out, nil } -func (st *storage) local() []api.StoragePath { +func (st *Local) Local() []api.StoragePath { var out []api.StoragePath for _, p := range st.paths { if p.local == "" { @@ -266,108 +266,3 @@ func (st *storage) local() []api.StoragePath { return out } - -func (st *storage) ServeHTTP(w http.ResponseWriter, r *http.Request) { // /storage/ - mux := mux.NewRouter() - - mux.HandleFunc("/{type}/{id}", st.remoteGetSector).Methods("GET") - - log.Infof("SERVEGETREMOTE %s", r.URL) - - mux.ServeHTTP(w, r) -} - - -func (st *storage) remoteGetSector(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - - id, err := parseSectorID(vars["id"]) - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - ft, err := ftFromString(vars["type"]) - if err != nil { - return - } - paths, done, err := st.acquireSector(id.Miner, id.Number, ft, 0, false) - if err != nil { - return - } - defer done() - - var path string - switch ft { - case sectorbuilder.FTUnsealed: - path = paths.Unsealed - case sectorbuilder.FTSealed: - path = paths.Sealed - case sectorbuilder.FTCache: - path = paths.Cache - } - if path == "" { - log.Error("acquired path was empty") - w.WriteHeader(500) - return - } - - stat, err := os.Stat(path) - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - var rd io.Reader - if stat.IsDir() { - rd, err = tarutil.TarDirectory(path) - w.Header().Set("Content-Type", "application/x-tar") - } else { - rd, err = os.OpenFile(path, os.O_RDONLY, 0644) - w.Header().Set("Content-Type", "application/octet-stream") - } - if err != nil { - log.Error(err) - w.WriteHeader(500) - return - } - - w.WriteHeader(200) - if _, err := io.Copy(w, rd); err != nil { // TODO: default 32k buf may be too small - log.Error(err) - return - } -} - -func ftFromString(t string) (sectorbuilder.SectorFileType, error) { - switch t { - case sectorbuilder.FTUnsealed.String(): - return sectorbuilder.FTUnsealed, nil - case sectorbuilder.FTSealed.String(): - return sectorbuilder.FTSealed, nil - case sectorbuilder.FTCache.String(): - return sectorbuilder.FTCache, nil - default: - return 0, xerrors.Errorf("unknown sector file type: '%s'", t) - } -} - -func parseSectorID(baseName string) (abi.SectorID, error) { - var n abi.SectorNumber - var mid abi.ActorID - read, err := fmt.Sscanf(baseName, "s-t0%d-%d", &mid, &n) - if err != nil { - return abi.SectorID{}, xerrors.Errorf(": %w", err) - } - - if read != 2 { - return abi.SectorID{}, xerrors.Errorf("parseSectorID expected to scan 2 values, got %d", read) - } - - return abi.SectorID{ - Miner: mid, - Number: n, - }, nil -} From 56968d858c055336655bfcc79993ecc2b3045c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 11 Mar 2020 22:23:16 +0100 Subject: [PATCH 04/45] workers: Make LocalWorker public --- cmd/lotus-seal-worker/rpc.go | 31 ++++--------------------------- cmd/lotus-seal-worker/storage.go | 7 +++++-- storage/sealmgr/advmgr/local.go | 24 ++++++++++++------------ storage/sealmgr/advmgr/manager.go | 2 +- 4 files changed, 22 insertions(+), 42 deletions(-) diff --git a/cmd/lotus-seal-worker/rpc.go b/cmd/lotus-seal-worker/rpc.go index 0acf9cd5f..cd77e1c5a 100644 --- a/cmd/lotus-seal-worker/rpc.go +++ b/cmd/lotus-seal-worker/rpc.go @@ -1,36 +1,13 @@ package main import ( - "context" - - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/go-sectorbuilder" ) -type worker struct { - spt abi.RegisteredProof -} - -func (w *worker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) { - panic("implement me") -} - -func (w *worker) SealPreCommit2(context.Context, abi.SectorNumber, storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { - panic("implement me") -} - -func (w *worker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (storage.Commit1Out, error) { - panic("implement me") -} - -func (w *worker) SealCommit2(context.Context, abi.SectorNumber, storage.Commit1Out) (storage.Proof, error) { - panic("implement me") -} - -func (w *worker) FinalizeSector(context.Context, abi.SectorNumber) error { - panic("implement me") +type worker struct { // TODO: use advmgr.LocalWorker here + sectorbuilder.Basic } var _ storage.Sealer = &worker{} diff --git a/cmd/lotus-seal-worker/storage.go b/cmd/lotus-seal-worker/storage.go index 6c86a5945..2fc9cd0a2 100644 --- a/cmd/lotus-seal-worker/storage.go +++ b/cmd/lotus-seal-worker/storage.go @@ -6,16 +6,18 @@ import ( "sort" "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" ) type workerStorage struct { path string // TODO: multi-path support - mid abi.ActorID + mid abi.ActorID // ewwhh TODO: passthru in sectobuilder/ffi + local *stores.Local auth http.Header api api.StorageMiner } @@ -28,6 +30,7 @@ func (w *workerStorage) AcquireSector(ctx context.Context, id abi.SectorNumber, // extract local storage; prefer + si, err := w.api.WorkerFindSector(ctx, asid, existing) if err != nil { return sectorbuilder.SectorPaths{}, nil, err diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go index 47b562e01..f0b46bf25 100644 --- a/storage/sealmgr/advmgr/local.go +++ b/storage/sealmgr/advmgr/local.go @@ -18,13 +18,13 @@ import ( "github.com/filecoin-project/lotus/storage/sealmgr" ) -type localWorker struct { +type LocalWorker struct { scfg *sectorbuilder.Config storage *stores.Local } type localWorkerPathProvider struct { - w *localWorker + w *LocalWorker } func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { @@ -39,11 +39,11 @@ func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, id abi.Sect }, existing, allocate, sealing) } -func (l *localWorker) sb() (sectorbuilder.Basic, error) { +func (l *LocalWorker) sb() (sectorbuilder.Basic, error) { return sectorbuilder.New(&localWorkerPathProvider{w: l}, l.scfg) } -func (l *localWorker) AddPiece(ctx context.Context, sn abi.SectorNumber, epcs []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { +func (l *LocalWorker) AddPiece(ctx context.Context, sn abi.SectorNumber, epcs []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { sb, err := l.sb() if err != nil { return abi.PieceInfo{}, err @@ -52,7 +52,7 @@ func (l *localWorker) AddPiece(ctx context.Context, sn abi.SectorNumber, epcs [] return sb.AddPiece(ctx, sn, epcs, sz, r) } -func (l *localWorker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) { +func (l *LocalWorker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) { sb, err := l.sb() if err != nil { return nil, err @@ -61,7 +61,7 @@ func (l *localWorker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNu return sb.SealPreCommit1(ctx, sectorNum, ticket, pieces) } -func (l *localWorker) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { +func (l *LocalWorker) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { sb, err := l.sb() if err != nil { return cid.Undef, cid.Undef, err @@ -70,7 +70,7 @@ func (l *localWorker) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNu return sb.SealPreCommit2(ctx, sectorNum, phase1Out) } -func (l *localWorker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output storage2.Commit1Out, err error) { +func (l *LocalWorker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output storage2.Commit1Out, err error) { sb, err := l.sb() if err != nil { return nil, err @@ -79,7 +79,7 @@ func (l *localWorker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumbe return sb.SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID) } -func (l *localWorker) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) { +func (l *LocalWorker) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) { sb, err := l.sb() if err != nil { return nil, err @@ -88,7 +88,7 @@ func (l *localWorker) SealCommit2(ctx context.Context, sectorNum abi.SectorNumbe return sb.SealCommit2(ctx, sectorNum, phase1Out) } -func (l *localWorker) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { +func (l *LocalWorker) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { sb, err := l.sb() if err != nil { return err @@ -97,7 +97,7 @@ func (l *localWorker) FinalizeSector(ctx context.Context, sectorNum abi.SectorNu return sb.FinalizeSector(ctx, sectorNum) } -func (l *localWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) { +func (l *LocalWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) { return map[sealmgr.TaskType]struct{}{ sealmgr.TTAddPiece: {}, sealmgr.TTPreCommit1: {}, @@ -106,8 +106,8 @@ func (l *localWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, }, nil } -func (l *localWorker) Paths(context.Context) ([]api.StoragePath, error) { +func (l *LocalWorker) Paths(context.Context) ([]api.StoragePath, error) { return l.storage.Local(), nil } -var _ Worker = &localWorker{} +var _ Worker = &LocalWorker{} diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index d6e8404a1..83d580c7c 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -66,7 +66,7 @@ func New(ls stores.LocalStorage, cfg *sectorbuilder.Config, sc SectorIDCounter) m := &Manager{ workers: []Worker{ - &localWorker{scfg: cfg, storage: stor}, + &LocalWorker{scfg: cfg, storage: stor}, }, scfg: cfg, sc: sc, From 86871e5abcaf3cdec3178b747aed924f04386928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 13 Mar 2020 01:23:05 +0100 Subject: [PATCH 05/45] workers: Basic Remote store --- cmd/lotus-seal-worker/main.go | 41 ------ cmd/lotus-seal-worker/rpc.go | 5 +- cmd/lotus-seal-worker/storage.go | 53 -------- cmd/lotus-seal-worker/transfer.go | 62 --------- go.mod | 2 - go.sum | 2 + storage/sealmgr/sectorutil/utils.go | 25 ++++ storage/sealmgr/stores/http_handler.go | 10 +- storage/sealmgr/stores/interface.go | 5 +- storage/sealmgr/stores/local.go | 20 +-- storage/sealmgr/stores/remote.go | 166 +++++++++++++++++++++++++ 11 files changed, 201 insertions(+), 190 deletions(-) delete mode 100644 cmd/lotus-seal-worker/storage.go delete mode 100644 cmd/lotus-seal-worker/transfer.go create mode 100644 storage/sealmgr/stores/remote.go diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index b44abc113..0ce5f732e 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -4,7 +4,6 @@ import ( "os" logging "github.com/ipfs/go-log/v2" - manet "github.com/multiformats/go-multiaddr-net" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" @@ -118,47 +117,7 @@ var runCmd = &cli.Command{ return xerrors.Errorf("get params: %w", err) } - ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) - if err != nil { - return xerrors.Errorf("could not get api info: %w", err) - } - _, storageAddr, err := manet.DialArgs(ainfo.Addr) - - - /* - /*ppt, spt, err := api.ProofTypeFromSectorSize(ssize) - if err != nil { - return err - } - - /*sb, err := sectorbuilder.NewStandalone(§orbuilder.Config{ - SealProofType: spt, - PoStProofType: ppt, - Miner: act, - WorkerThreads: workers, - Paths: sectorbuilder.SimplePath(r), - }) - if err != nil { - return err - } - - nQueues := workers + transfers - var wg sync.WaitGroup - wg.Add(nQueues) - - for i := 0; i < nQueues; i++ { - go func() { - defer wg.Done() - - /* if err := acceptJobs(ctx, nodeApi, sb, limiter, "http://"+storageAddr, ainfo.AuthHeader(), r, cctx.Bool("no-precommit"), cctx.Bool("no-commit")); err != nil { - log.Warnf("%+v", err) - return - } - }() - } - - wg.Wait()*/ return nil }, } diff --git a/cmd/lotus-seal-worker/rpc.go b/cmd/lotus-seal-worker/rpc.go index cd77e1c5a..80b98155b 100644 --- a/cmd/lotus-seal-worker/rpc.go +++ b/cmd/lotus-seal-worker/rpc.go @@ -1,13 +1,12 @@ package main import ( + "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" "github.com/filecoin-project/specs-storage/storage" - - "github.com/filecoin-project/go-sectorbuilder" ) type worker struct { // TODO: use advmgr.LocalWorker here - sectorbuilder.Basic + *advmgr.LocalWorker } var _ storage.Sealer = &worker{} diff --git a/cmd/lotus-seal-worker/storage.go b/cmd/lotus-seal-worker/storage.go deleted file mode 100644 index 2fc9cd0a2..000000000 --- a/cmd/lotus-seal-worker/storage.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import ( - "context" - "net/http" - "sort" - - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" -) - -type workerStorage struct { - path string // TODO: multi-path support - mid abi.ActorID // ewwhh TODO: passthru in sectobuilder/ffi - - local *stores.Local - auth http.Header - api api.StorageMiner -} - -func (w *workerStorage) AcquireSector(ctx context.Context, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { - asid := abi.SectorID{ - Miner: w.mid, - Number: id, - } - - // extract local storage; prefer - - - si, err := w.api.WorkerFindSector(ctx, asid, existing) - if err != nil { - return sectorbuilder.SectorPaths{}, nil, err - } - - sort.Slice(si, func(i, j int) bool { - return si[i].Cost < si[j].Cost - }) - - best := si[0].URLs // TODO: not necessarily true - - sname := sectorutil.SectorName(abi.SectorID{ - Miner: w.mid, - Number: id, - }) - - w.fetch(best, ) -} - -var _ sectorbuilder.SectorProvider = &workerStorage{} diff --git a/cmd/lotus-seal-worker/transfer.go b/cmd/lotus-seal-worker/transfer.go deleted file mode 100644 index 7a4a29e7e..000000000 --- a/cmd/lotus-seal-worker/transfer.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - - -import ( - "mime" - "net/http" - "os" - - files "github.com/ipfs/go-ipfs-files" - "golang.org/x/xerrors" - - "github.com/filecoin-project/lotus/lib/tarutil" -) - -func (w *workerStorage) fetch(url, outname string) error { - log.Infof("Fetch %s", url) - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return xerrors.Errorf("request: %w", err) - } - req.Header = w.auth - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return xerrors.Errorf("do request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return xerrors.Errorf("non-200 code: %d", resp.StatusCode) - } - - /*bar := pb.New64(w.sizeForType(typ)) - bar.ShowPercent = true - bar.ShowSpeed = true - bar.Units = pb.U_BYTES - - barreader := bar.NewProxyReader(resp.Body) - - bar.Start() - defer bar.Finish()*/ - - mediatype, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) - if err != nil { - return xerrors.Errorf("parse media type: %w", err) - } - - if err := os.RemoveAll(outname); err != nil { - return xerrors.Errorf("removing dest: %w", err) - } - - switch mediatype { - case "application/x-tar": - return tarutil.ExtractTar(resp.Body, outname) - case "application/octet-stream": - return files.WriteTo(files.NewReaderFile(resp.Body), outname) - default: - return xerrors.Errorf("unknown content type: '%s'", mediatype) - } - -} diff --git a/go.mod b/go.mod index 822874a80..1ab22a12a 100644 --- a/go.mod +++ b/go.mod @@ -117,5 +117,3 @@ replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi replace github.com/coreos/go-systemd => github.com/coreos/go-systemd/v22 v22.0.0 - -replace github.com/filecoin-project/go-sectorbuilder => /home/magik6k/gohack/github.com/filecoin-project/go-sectorbuilder diff --git a/go.sum b/go.sum index d058d9c3e..aa1c9e0c3 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= @@ -121,6 +122,7 @@ github.com/filecoin-project/go-fil-markets v0.0.0-20200304003055-d449a980d4bd h1 github.com/filecoin-project/go-fil-markets v0.0.0-20200304003055-d449a980d4bd/go.mod h1:rfRwhd3ujcCXnD4N9oEM2wjh8GRZGoeNXME+UPG/9ts= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE= +github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 h1:eYxi6vI5CyeXD15X1bB3bledDXbqKxqf0wQzTLgwYwA= github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f56/go.mod h1:tzTc9BxxSbjlIzhFwm5h9oBkXKkRuLxeiWspntwnKyw= diff --git a/storage/sealmgr/sectorutil/utils.go b/storage/sealmgr/sectorutil/utils.go index 4c0d0328b..01862b7b4 100644 --- a/storage/sealmgr/sectorutil/utils.go +++ b/storage/sealmgr/sectorutil/utils.go @@ -2,6 +2,7 @@ package sectorutil import ( "fmt" + "github.com/filecoin-project/go-sectorbuilder" "golang.org/x/xerrors" @@ -29,3 +30,27 @@ func ParseSectorID(baseName string) (abi.SectorID, error) { func SectorName(sid abi.SectorID) string { return fmt.Sprintf("s-t0%d-%d", sid.Miner, sid.Number) } + +func PathByType(sps sectorbuilder.SectorPaths, fileType sectorbuilder.SectorFileType) string { + switch fileType { + case sectorbuilder.FTUnsealed: + return sps.Unsealed + case sectorbuilder.FTSealed: + return sps.Sealed + case sectorbuilder.FTCache: + return sps.Cache + } + + panic("requested unknown path type") +} + +func SetPathByType(sps *sectorbuilder.SectorPaths, fileType sectorbuilder.SectorFileType, p string) { + switch fileType { + case sectorbuilder.FTUnsealed: + sps.Unsealed = p + case sectorbuilder.FTSealed: + sps.Sealed = p + case sectorbuilder.FTCache: + sps.Cache = p + } +} diff --git a/storage/sealmgr/stores/http_handler.go b/storage/sealmgr/stores/http_handler.go index 960a91d23..40264b74c 100644 --- a/storage/sealmgr/stores/http_handler.go +++ b/storage/sealmgr/stores/http_handler.go @@ -50,15 +50,7 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ } defer done() - var path string - switch ft { - case sectorbuilder.FTUnsealed: - path = paths.Unsealed - case sectorbuilder.FTSealed: - path = paths.Sealed - case sectorbuilder.FTCache: - path = paths.Cache - } + path := sectorutil.PathByType(paths, ft) if path == "" { log.Error("acquired path was empty") w.WriteHeader(500) diff --git a/storage/sealmgr/stores/interface.go b/storage/sealmgr/stores/interface.go index 08dddad7a..a6ea5ec99 100644 --- a/storage/sealmgr/stores/interface.go +++ b/storage/sealmgr/stores/interface.go @@ -1,11 +1,12 @@ package stores import ( - "github.com/filecoin-project/specs-actors/actors/abi" + "context" "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" ) type Store interface { - AcquireSector(s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) + AcquireSector(ctx context.Context, s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) } diff --git a/storage/sealmgr/stores/local.go b/storage/sealmgr/stores/local.go index 9b799ab6f..942ea2086 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sealmgr/stores/local.go @@ -143,15 +143,7 @@ func (st *Local) AcquireSector(sid abi.SectorID, existing sectorbuilder.SectorFi } spath := filepath.Join(p.local, fileType.String(), sectorutil.SectorName(sid)) - - switch fileType { - case sectorbuilder.FTUnsealed: - out.Unsealed = spath - case sectorbuilder.FTSealed: - out.Sealed = spath - case sectorbuilder.FTCache: - out.Cache = spath - } + sectorutil.SetPathByType(&out, fileType, spath) existing ^= fileType } @@ -188,15 +180,7 @@ func (st *Local) AcquireSector(sid abi.SectorID, existing sectorbuilder.SectorFi return sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("couldn't find a suitable path for a sector") } - switch fileType { - case sectorbuilder.FTUnsealed: - out.Unsealed = best - case sectorbuilder.FTSealed: - out.Sealed = best - case sectorbuilder.FTCache: - out.Cache = best - } - + sectorutil.SetPathByType(&out, fileType, best) allocate ^= fileType } diff --git a/storage/sealmgr/stores/remote.go b/storage/sealmgr/stores/remote.go new file mode 100644 index 000000000..77f10c741 --- /dev/null +++ b/storage/sealmgr/stores/remote.go @@ -0,0 +1,166 @@ +package stores + +import ( + "context" + "mime" + "net/http" + "os" + "sort" + "sync" + + "github.com/hashicorp/go-multierror" + files "github.com/ipfs/go-ipfs-files" + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/lib/tarutil" + "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" +) + +type Remote struct { + local Store + remote SectorIndex + auth http.Header + + fetchLk sync.Mutex // TODO: this can be much smarter + // TODO: allow multiple parallel fetches + // (make sure to not fetch the same sector data twice) +} + +type SectorIndex interface { + FindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]api.StorageInfo, error) +} + +func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { + if existing|allocate != existing^allocate { + return sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector") + } + + r.fetchLk.Lock() + defer r.fetchLk.Unlock() + + paths, done, err := r.local.AcquireSector(ctx, s, existing, allocate, sealing) + if err != nil { + return sectorbuilder.SectorPaths{}, nil, err + } + + for _, fileType := range pathTypes { + if fileType&existing == 0 { + continue + } + + if sectorutil.PathByType(paths, fileType) != "" { + continue + } + + ap, rdone, err := r.acquireFromRemote(ctx, s, fileType, sealing) + if err != nil { + done() + return sectorbuilder.SectorPaths{}, nil, err + } + + done = mergeDone(done, rdone) + sectorutil.SetPathByType(&paths, fileType, ap) + + } + + return paths, done, nil +} + +func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, func(), error) { + si, err := r.remote.FindSector(ctx, s, fileType) + if err != nil { + return "", nil, err + } + + sort.Slice(si, func(i, j int) bool { + return si[i].Cost < si[j].Cost + }) + + apaths, done, err := r.local.AcquireSector(ctx, s, 0, fileType, sealing) + if err != nil { + return "", nil, xerrors.Errorf("allocate local sector for fetching: %w", err) + } + dest := sectorutil.PathByType(apaths, fileType) + + var merr error + for _, info := range si { + for _, url := range info.URLs { + err := r.fetch(url, dest) + if err != nil { + merr = multierror.Append(merr, xerrors.Errorf("fetch error %s (storage %s) -> %s: %w", url, info.ID, dest, err)) + continue + } + + if merr != nil { + log.Warnw("acquireFromRemote encountered errors when fetching sector from remote", "errors", merr) + } + return dest, done, nil + } + } + + done() + return "", nil, xerrors.Errorf("failed to acquire sector %v from remote: %w", s, merr) +} + + +func (r *Remote) fetch(url, outname string) error { + log.Infof("Fetch %s -> %s", url, outname) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return xerrors.Errorf("request: %w", err) + } + req.Header = r.auth + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return xerrors.Errorf("do request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return xerrors.Errorf("non-200 code: %d", resp.StatusCode) + } + + /*bar := pb.New64(w.sizeForType(typ)) + bar.ShowPercent = true + bar.ShowSpeed = true + bar.Units = pb.U_BYTES + + barreader := bar.NewProxyReader(resp.Body) + + bar.Start() + defer bar.Finish()*/ + + mediatype, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil { + return xerrors.Errorf("parse media type: %w", err) + } + + if err := os.RemoveAll(outname); err != nil { + return xerrors.Errorf("removing dest: %w", err) + } + + switch mediatype { + case "application/x-tar": + return tarutil.ExtractTar(resp.Body, outname) + case "application/octet-stream": + return files.WriteTo(files.NewReaderFile(resp.Body), outname) + default: + return xerrors.Errorf("unknown content type: '%s'", mediatype) + } + +} + +func mergeDone(a func(), b func()) func() { + return func() { + a() + b() + } +} + +var _ Store = &Remote{} From bc7fede68b0f3b1f18e571085f09bd7f6163649f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 13 Mar 2020 02:37:38 +0100 Subject: [PATCH 06/45] workers: Wire up most pieces --- api/api_storage.go | 2 +- api/apistruct/permissioned.go | 6 ++ api/apistruct/struct.go | 6 +- cmd/lotus-seal-worker/main.go | 105 ++++++++++++++++++++++--- cmd/lotus-seal-worker/rpc.go | 7 ++ node/impl/storminer.go | 2 +- node/repo/fsrepo.go | 3 + storage/sealmgr/advmgr/local.go | 23 +++++- storage/sealmgr/advmgr/roprov.go | 2 +- storage/sealmgr/stores/http_handler.go | 2 +- storage/sealmgr/stores/local.go | 3 +- storage/sealmgr/stores/remote.go | 8 ++ 12 files changed, 145 insertions(+), 24 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index be912401b..e4d5d8a25 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -112,7 +112,7 @@ type StorageMiner interface { WorkerConnect(context.Context, string) error WorkerAttachStorage(context.Context, StorageInfo) error WorkerDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error - WorkerFindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]StorageInfo, error) + FindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]StorageInfo, error) MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) diff --git a/api/apistruct/permissioned.go b/api/apistruct/permissioned.go index 4c29f6688..5e513859a 100644 --- a/api/apistruct/permissioned.go +++ b/api/apistruct/permissioned.go @@ -43,6 +43,12 @@ func PermissionedFullAPI(a api.FullNode) api.FullNode { return &out } +func PermissionedWorkerAPI(a api.WorkerApi) api.WorkerApi { + var out WorkerStruct + permissionedAny(a, &out.Internal) + return &out +} + func HasPerm(ctx context.Context, perm api.Permission) bool { callerPerms, ok := ctx.Value(permCtxKey).([]api.Permission) if !ok { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index e978daa29..a6b2019b6 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -183,7 +183,7 @@ type StorageMinerStruct struct { WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm WorkerAttachStorage func(context.Context, api.StorageInfo) error `perm:"admin"` WorkerDeclareSector func(ctx context.Context, storageId string, s abi.SectorID) error `perm:"admin"` - WorkerFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]api.StorageInfo, error) `perm:"admin"` + FindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]api.StorageInfo, error) `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -659,8 +659,8 @@ func (c *StorageMinerStruct) WorkerDeclareSector(ctx context.Context, storageId return c.Internal.WorkerDeclareSector(ctx, storageId, s) } -func (c *StorageMinerStruct) WorkerFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]api.StorageInfo, error) { - return c.Internal.WorkerFindSector(ctx, si, types) +func (c *StorageMinerStruct) FindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]api.StorageInfo, error) { + return c.Internal.FindSector(ctx, si, types) } func (c *StorageMinerStruct) MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error { diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 0ce5f732e..c3e867da3 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -1,23 +1,36 @@ package main import ( + "context" + "net/http" "os" + "os/signal" + "syscall" + "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" paramfetch "github.com/filecoin-project/go-paramfetch" + manet "github.com/multiformats/go-multiaddr-net" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/auth" + "github.com/filecoin-project/lotus/lib/jsonrpc" "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" ) var log = logging.Logger("main") +const FlagStorageRepo = "workerrepo" + const ( workers = 1 // TODO: Configurability transfers = 1 @@ -38,7 +51,7 @@ func main() { Version: build.UserVersion, Flags: []cli.Flag{ &cli.StringFlag{ - Name: "repo", + Name: "workerrepo", EnvVars: []string{"WORKER_PATH"}, Value: "~/.lotusworker", // TODO: Consider XDG_DATA_HOME }, @@ -52,12 +65,6 @@ func main() { Usage: "enable use of GPU for mining operations", Value: true, }, - &cli.BoolFlag{ - Name: "no-precommit", - }, - &cli.BoolFlag{ - Name: "no-commit", - }, }, Commands: local, @@ -71,11 +78,6 @@ func main() { } } -type limits struct { - workLimit chan struct{} - transferLimit chan struct{} -} - var runCmd = &cli.Command{ Name: "run", Usage: "Start lotus worker", @@ -98,6 +100,7 @@ var runCmd = &cli.Command{ if v.APIVersion != build.APIVersion { return xerrors.Errorf("lotus-storage-miner API version doesn't match: local: ", api.Version{APIVersion: build.APIVersion}) } + log.Infof("Remote version %s", v) go func() { <-ctx.Done() @@ -117,7 +120,83 @@ var runCmd = &cli.Command{ return xerrors.Errorf("get params: %w", err) } + repoPath := cctx.String(FlagStorageRepo) + r, err := repo.NewFS(repoPath) + if err != nil { + return err + } - return nil + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + return xerrors.Errorf("repo at '%s' is not initialized, run 'lotus-seal-worker init' to set it up", repoPath) + } + + lr, err := r.Lock(repo.Worker) + if err != nil { + return err + } + + localStore, err := stores.NewLocal(lr) + if err != nil { + return err + } + + endpoint, err := r.APIEndpoint() + if err != nil { + return err + } + + lst, err := manet.Listen(endpoint) + if err != nil { + return xerrors.Errorf("could not listen: %w", err) + } + + _, spt, err := api.ProofTypeFromSectorSize(ssize) + if err != nil { + return xerrors.Errorf("getting proof type: %w", err) + } + + sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner) + if err != nil { + return xerrors.Errorf("could not get api info: %w", err) + } + + remote := stores.NewRemote(localStore, nodeApi, sminfo.AuthHeader()) + + workerApi := &worker{ + LocalWorker: advmgr.NewLocalWorker(act, spt, remote, localStore), + } + + mux := mux.NewRouter() + + rpcServer := jsonrpc.NewServer() + rpcServer.Register("Filecoin", apistruct.PermissionedWorkerAPI(workerApi)) + + mux.Handle("/rpc/v0", rpcServer) + mux.PathPrefix("/remote").HandlerFunc((&stores.FetchHandler{Store: localStore}).ServeHTTP) + mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof + + ah := &auth.Handler{ + Verify: nodeApi.AuthVerify, + Next: mux.ServeHTTP, + } + + srv := &http.Server{Handler: ah} + + sigChan := make(chan os.Signal, 2) + go func() { + <-sigChan + log.Warn("Shutting down..") + if err := srv.Shutdown(context.TODO()); err != nil { + log.Errorf("shutting down RPC server failed: %s", err) + } + log.Warn("Graceful shutdown successful") + }() + signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT) + + return srv.Serve(manet.NetListener(lst)) }, } diff --git a/cmd/lotus-seal-worker/rpc.go b/cmd/lotus-seal-worker/rpc.go index 80b98155b..9de4d1079 100644 --- a/cmd/lotus-seal-worker/rpc.go +++ b/cmd/lotus-seal-worker/rpc.go @@ -1,6 +1,9 @@ package main import ( + "context" + + "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" "github.com/filecoin-project/specs-storage/storage" ) @@ -9,4 +12,8 @@ type worker struct { // TODO: use advmgr.LocalWorker here *advmgr.LocalWorker } +func (w *worker) Version(context.Context) (build.Version, error) { + return build.APIVersion, nil +} + var _ storage.Sealer = &worker{} diff --git a/node/impl/storminer.go b/node/impl/storminer.go index a2d3dcb9a..9bbc58f1e 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -159,7 +159,7 @@ func (sm *StorageMinerAPI) WorkerDeclareSector(ctx context.Context, storageId st panic("implement me") } -func (sm *StorageMinerAPI) WorkerFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]api.StorageInfo, error) { +func (sm *StorageMinerAPI) FindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]api.StorageInfo, error) { panic("implement me") } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 12f9540d8..926541359 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -40,6 +40,7 @@ const ( _ = iota // Default is invalid FullNode RepoType = iota StorageMiner + Worker ) func defConfForType(t RepoType) interface{} { @@ -48,6 +49,8 @@ func defConfForType(t RepoType) interface{} { return config.DefaultFullNode() case StorageMiner: return config.DefaultStorageMiner() + case Worker: + return &struct {}{} default: panic(fmt.Sprintf("unknown RepoType(%d)", int(t))) } diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go index f0b46bf25..1b5d9859a 100644 --- a/storage/sealmgr/advmgr/local.go +++ b/storage/sealmgr/advmgr/local.go @@ -20,7 +20,24 @@ import ( type LocalWorker struct { scfg *sectorbuilder.Config - storage *stores.Local + storage stores.Store + localStore *stores.Local +} + +func NewLocalWorker(ma address.Address, spt abi.RegisteredProof, store stores.Store, local *stores.Local) *LocalWorker { + ppt, err := spt.RegisteredPoStProof() + if err != nil { + panic(err) + } + return &LocalWorker{ + scfg: §orbuilder.Config{ + SealProofType: spt, + PoStProofType: ppt, + Miner: ma, + }, + storage: store, + localStore: local, + } } type localWorkerPathProvider struct { @@ -33,7 +50,7 @@ func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, id abi.Sect return sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("get miner ID: %w", err) } - return l.w.storage.AcquireSector(abi.SectorID{ + return l.w.storage.AcquireSector(ctx, abi.SectorID{ Miner: abi.ActorID(mid), Number: id, }, existing, allocate, sealing) @@ -107,7 +124,7 @@ func (l *LocalWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, } func (l *LocalWorker) Paths(context.Context) ([]api.StoragePath, error) { - return l.storage.Local(), nil + return l.localStore.Local(), nil } var _ Worker = &LocalWorker{} diff --git a/storage/sealmgr/advmgr/roprov.go b/storage/sealmgr/advmgr/roprov.go index 64513cdcd..2269a3257 100644 --- a/storage/sealmgr/advmgr/roprov.go +++ b/storage/sealmgr/advmgr/roprov.go @@ -20,7 +20,7 @@ func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorNumbe return sectorbuilder.SectorPaths{}, nil, xerrors.New("read-only storage") } - return l.stor.AcquireSector(abi.SectorID{ + return l.stor.AcquireSector(ctx, abi.SectorID{ Miner: l.miner, Number: id, }, existing, allocate, sealing) diff --git a/storage/sealmgr/stores/http_handler.go b/storage/sealmgr/stores/http_handler.go index 40264b74c..75cb080b5 100644 --- a/storage/sealmgr/stores/http_handler.go +++ b/storage/sealmgr/stores/http_handler.go @@ -44,7 +44,7 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ if err != nil { return } - paths, done, err := handler.Store.AcquireSector(id, ft, 0, false) + paths, done, err := handler.Store.AcquireSector(r.Context(), id, ft, 0, false) if err != nil { return } diff --git a/storage/sealmgr/stores/local.go b/storage/sealmgr/stores/local.go index 942ea2086..3420a0aea 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sealmgr/stores/local.go @@ -1,6 +1,7 @@ package stores import ( + "context" "encoding/json" "io/ioutil" "os" @@ -114,7 +115,7 @@ func (st *Local) open() error { return nil } -func (st *Local) AcquireSector(sid abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { +func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { if existing|allocate != existing^allocate { return sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector") } diff --git a/storage/sealmgr/stores/remote.go b/storage/sealmgr/stores/remote.go index 77f10c741..e278dd701 100644 --- a/storage/sealmgr/stores/remote.go +++ b/storage/sealmgr/stores/remote.go @@ -30,6 +30,14 @@ type Remote struct { // (make sure to not fetch the same sector data twice) } +func NewRemote(local Store, remote SectorIndex, auth http.Header) *Remote { + return &Remote{ + local: local, + remote: remote, + auth: auth, + } +} + type SectorIndex interface { FindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]api.StorageInfo, error) } From a75ad492f1331ea7203663a58b8a51bdf34344e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 13 Mar 2020 11:40:24 +0100 Subject: [PATCH 07/45] workers: Declare sectors after fetching in remote store --- api/api_storage.go | 4 ++-- api/apistruct/struct.go | 12 ++++++------ storage/sealmgr/stores/local.go | 17 +++++++++++++---- storage/sealmgr/stores/remote.go | 27 ++++++++++++++++----------- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index e4d5d8a25..13aec9544 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -111,8 +111,8 @@ type StorageMiner interface { // WorkerConnect tells the node to connect to workers RPC WorkerConnect(context.Context, string) error WorkerAttachStorage(context.Context, StorageInfo) error - WorkerDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error - FindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]StorageInfo, error) + StorageDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error + StorageFindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]StorageInfo, error) MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index a6b2019b6..1713f8b2d 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -182,8 +182,8 @@ type StorageMinerStruct struct { WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm WorkerAttachStorage func(context.Context, api.StorageInfo) error `perm:"admin"` - WorkerDeclareSector func(ctx context.Context, storageId string, s abi.SectorID) error `perm:"admin"` - FindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]api.StorageInfo, error) `perm:"admin"` + StorageDeclareSector func(ctx context.Context, storageId string, s abi.SectorID) error `perm:"admin"` + StorageFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]api.StorageInfo, error) `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -655,12 +655,12 @@ func (c *StorageMinerStruct) WorkerAttachStorage(ctx context.Context, si api.Sto return c.Internal.WorkerAttachStorage(ctx, si) } -func (c *StorageMinerStruct) WorkerDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error { - return c.Internal.WorkerDeclareSector(ctx, storageId, s) +func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error { + return c.Internal.StorageDeclareSector(ctx, storageId, s) } -func (c *StorageMinerStruct) FindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]api.StorageInfo, error) { - return c.Internal.FindSector(ctx, si, types) +func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]api.StorageInfo, error) { + return c.Internal.StorageFindSector(ctx, si, types) } func (c *StorageMinerStruct) MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error { diff --git a/storage/sealmgr/stores/local.go b/storage/sealmgr/stores/local.go index 3420a0aea..88de7bf7b 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sealmgr/stores/local.go @@ -116,13 +116,19 @@ func (st *Local) open() error { } func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { + out, _, done, err := st.acquireSector(ctx, sid, existing, allocate, sealing) + return out, done, err +} + +func (st *Local) acquireSector(ctx context.Context, sid abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, sectorbuilder.SectorPaths, func(), error) { if existing|allocate != existing^allocate { - return sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector") + return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector") } st.localLk.RLock() var out sectorbuilder.SectorPaths + var storageIDs sectorbuilder.SectorPaths for _, fileType := range pathTypes { if fileType&existing == 0 { @@ -145,6 +151,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing s spath := filepath.Join(p.local, fileType.String(), sectorutil.SectorName(sid)) sectorutil.SetPathByType(&out, fileType, spath) + sectorutil.SetPathByType(&storageIDs, fileType, p.meta.ID) existing ^= fileType } @@ -155,7 +162,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing s continue } - var best string + var best, bestID string for _, p := range st.paths { if sealing && !p.meta.CanSeal { @@ -173,19 +180,21 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing s // TODO: Calc weights best = filepath.Join(p.local, fileType.String(), sectorutil.SectorName(sid)) + bestID = p.meta.ID break // todo: the first path won't always be the best } if best == "" { st.localLk.RUnlock() - return sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("couldn't find a suitable path for a sector") + return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("couldn't find a suitable path for a sector") } sectorutil.SetPathByType(&out, fileType, best) + sectorutil.SetPathByType(&storageIDs, fileType, bestID) allocate ^= fileType } - return out, st.localLk.RUnlock, nil + return out, storageIDs, st.localLk.RUnlock, nil } func (st *Local) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sealing bool) ([]config.StorageMeta, error) { diff --git a/storage/sealmgr/stores/remote.go b/storage/sealmgr/stores/remote.go index e278dd701..05b2ce5dc 100644 --- a/storage/sealmgr/stores/remote.go +++ b/storage/sealmgr/stores/remote.go @@ -21,7 +21,7 @@ import ( ) type Remote struct { - local Store + local *Local remote SectorIndex auth http.Header @@ -30,7 +30,7 @@ type Remote struct { // (make sure to not fetch the same sector data twice) } -func NewRemote(local Store, remote SectorIndex, auth http.Header) *Remote { +func NewRemote(local *Local, remote SectorIndex, auth http.Header) *Remote { return &Remote{ local: local, remote: remote, @@ -39,7 +39,8 @@ func NewRemote(local Store, remote SectorIndex, auth http.Header) *Remote { } type SectorIndex interface { - FindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]api.StorageInfo, error) + StorageDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error + StorageFindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]api.StorageInfo, error) } func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { @@ -64,7 +65,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec continue } - ap, rdone, err := r.acquireFromRemote(ctx, s, fileType, sealing) + ap, storageID, rdone, err := r.acquireFromRemote(ctx, s, fileType, sealing) if err != nil { done() return sectorbuilder.SectorPaths{}, nil, err @@ -73,26 +74,30 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec done = mergeDone(done, rdone) sectorutil.SetPathByType(&paths, fileType, ap) + if err := r.remote.StorageDeclareSector(ctx, storageID, s); err != nil { + log.Warnf("declaring sector %v in %s failed: %+v", s, storageID, err) + } } return paths, done, nil } -func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, func(), error) { - si, err := r.remote.FindSector(ctx, s, fileType) +func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, string, func(), error) { + si, err := r.remote.StorageFindSector(ctx, s, fileType) if err != nil { - return "", nil, err + return "", "", nil, err } sort.Slice(si, func(i, j int) bool { return si[i].Cost < si[j].Cost }) - apaths, done, err := r.local.AcquireSector(ctx, s, 0, fileType, sealing) + apaths, ids, done, err := r.local.acquireSector(ctx, s, 0, fileType, sealing) if err != nil { - return "", nil, xerrors.Errorf("allocate local sector for fetching: %w", err) + return "", "", nil, xerrors.Errorf("allocate local sector for fetching: %w", err) } dest := sectorutil.PathByType(apaths, fileType) + storageID := sectorutil.PathByType(ids, fileType) var merr error for _, info := range si { @@ -106,12 +111,12 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType if merr != nil { log.Warnw("acquireFromRemote encountered errors when fetching sector from remote", "errors", merr) } - return dest, done, nil + return dest, storageID, done, nil } } done() - return "", nil, xerrors.Errorf("failed to acquire sector %v from remote: %w", s, merr) + return "", "", nil, xerrors.Errorf("failed to acquire sector %v from remote: %w", s, merr) } From 2c7f57983811b3612f00b98f26ede1e5b70f3698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 13 Mar 2020 12:59:19 +0100 Subject: [PATCH 08/45] workers: Implement SectorIndex --- api/api_storage.go | 27 +----- api/api_worker.go | 3 +- api/apistruct/struct.go | 21 ++--- cmd/lotus-seed/seed/seed.go | 6 +- cmd/lotus-storage-miner/init.go | 5 +- cmd/lotus-storage-miner/storage.go | 6 +- node/config/storage.go | 9 -- node/impl/storminer.go | 7 +- node/repo/fsrepo.go | 2 +- node/repo/memrepo.go | 5 +- storage/sealmgr/advmgr/local.go | 11 ++- storage/sealmgr/advmgr/manager.go | 19 ++--- storage/sealmgr/advmgr/remote.go | 2 +- storage/sealmgr/advmgr/roprov.go | 2 +- storage/sealmgr/stores/index.go | 128 +++++++++++++++++++++++++++++ storage/sealmgr/stores/local.go | 45 +++++++--- storage/sealmgr/stores/remote.go | 23 ++---- 17 files changed, 217 insertions(+), 104 deletions(-) create mode 100644 storage/sealmgr/stores/index.go diff --git a/api/api_storage.go b/api/api_storage.go index 13aec9544..94cb1fdfa 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -7,11 +7,11 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/go-fil-markets/storagemarket" - "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" ) // alias because cbor-gen doesn't like non-alias types @@ -110,9 +110,7 @@ type StorageMiner interface { // WorkerConnect tells the node to connect to workers RPC WorkerConnect(context.Context, string) error - WorkerAttachStorage(context.Context, StorageInfo) error - StorageDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error - StorageFindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]StorageInfo, error) + stores.SectorIndex MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) @@ -125,25 +123,6 @@ type StorageMiner interface { StorageAddLocal(ctx context.Context, path string) error } -type StorageInfo struct { - ID string - URLs []string // TODO: Support non-http transports - Cost int - - CanSeal bool - CanStore bool -} - -type StoragePath struct { - ID string - Weight uint64 - - LocalPath string - - CanSeal bool - CanStore bool -} - type SealRes struct { Err string GoErr error `json:"-"` diff --git a/api/api_worker.go b/api/api_worker.go index 67a22f77a..bafd6ca83 100644 --- a/api/api_worker.go +++ b/api/api_worker.go @@ -2,6 +2,7 @@ package api import ( "context" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" "github.com/filecoin-project/specs-storage/storage" @@ -14,7 +15,7 @@ type WorkerApi interface { // TODO: Info() (name, ...) ? TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) // TaskType -> Weight - Paths(context.Context) ([]StoragePath, error) + Paths(context.Context) ([]stores.StoragePath, error) storage.Sealer } diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 1713f8b2d..31d9cfc71 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -2,6 +2,7 @@ package apistruct import ( "context" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" "github.com/filecoin-project/specs-storage/storage" "github.com/ipfs/go-cid" @@ -180,10 +181,10 @@ type StorageMinerStruct struct { SectorsRefs func(context.Context) (map[string][]api.SealedRef, error) `perm:"read"` SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error `perm:"write"` - WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm - WorkerAttachStorage func(context.Context, api.StorageInfo) error `perm:"admin"` - StorageDeclareSector func(ctx context.Context, storageId string, s abi.SectorID) error `perm:"admin"` - StorageFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]api.StorageInfo, error) `perm:"admin"` + WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm + WorkerAttachStorage func(context.Context, stores.StorageInfo) error `perm:"admin"` + StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, sectorbuilder.SectorFileType) error `perm:"admin"` + StorageFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]stores.StorageInfo, error) `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -199,7 +200,7 @@ type WorkerStruct struct { Version func(context.Context) (build.Version, error) `perm:"admin"` TaskTypes func(context.Context) (map[sealmgr.TaskType]struct{}, error) `perm:"admin"` - Paths func(context.Context) ([]api.StoragePath, error) `perm:"admin"` + Paths func(context.Context) ([]stores.StoragePath, error) `perm:"admin"` SealPreCommit1 func(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"` SealPreCommit2 func(context.Context, abi.SectorNumber, storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) `perm:"admin"` @@ -651,15 +652,15 @@ func (c *StorageMinerStruct) WorkerConnect(ctx context.Context, url string) erro return c.Internal.WorkerConnect(ctx, url) } -func (c *StorageMinerStruct) WorkerAttachStorage(ctx context.Context, si api.StorageInfo) error { +func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo) error { return c.Internal.WorkerAttachStorage(ctx, si) } -func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error { - return c.Internal.StorageDeclareSector(ctx, storageId, s) +func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error { + return c.Internal.StorageDeclareSector(ctx, storageId, s, ft) } -func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]api.StorageInfo, error) { +func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]stores.StorageInfo, error) { return c.Internal.StorageFindSector(ctx, si, types) } @@ -699,7 +700,7 @@ func (w *WorkerStruct) TaskTypes(ctx context.Context) (map[sealmgr.TaskType]stru return w.Internal.TaskTypes(ctx) } -func (w *WorkerStruct) Paths(ctx context.Context) ([]api.StoragePath, error) { +func (w *WorkerStruct) Paths(ctx context.Context) ([]stores.StoragePath, error) { return w.Internal.Paths(ctx) } diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 3f5946ccc..f18a9433f 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" "io/ioutil" "os" "path/filepath" @@ -26,7 +27,6 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/genesis" - "github.com/filecoin-project/lotus/node/config" ) var log = logging.Logger("preseal") @@ -134,8 +134,8 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum } { - b, err := json.MarshalIndent(&config.StorageMeta{ - ID: uuid.New().String(), + b, err := json.MarshalIndent(&stores.StorageMeta{ + ID: stores.ID(uuid.New().String()), Weight: 0, // read-only CanSeal: false, CanStore: false, diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index a8034ebfd..990aee236 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -7,6 +7,7 @@ import ( "encoding/binary" "encoding/json" "fmt" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" "io/ioutil" "os" "path/filepath" @@ -187,8 +188,8 @@ var initCmd = &cli.Command{ } if !cctx.Bool("no-local-storage") { - b, err := json.MarshalIndent(&config.StorageMeta{ - ID: uuid.New().String(), + b, err := json.MarshalIndent(&stores.StorageMeta{ + ID: stores.ID(uuid.New().String()), Weight: 10, CanSeal: true, CanStore: true, diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 7976e6dd5..31ab45c7c 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" "io/ioutil" "os" "path/filepath" @@ -12,7 +13,6 @@ import ( "gopkg.in/urfave/cli.v2" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/node/config" ) const metaFile = "sectorstore.json" @@ -79,8 +79,8 @@ var storageAttachCmd = &cli.Command{ return err } - cfg := &config.StorageMeta{ - ID: uuid.New().String(), + cfg := &stores.StorageMeta{ + ID: stores.ID(uuid.New().String()), Weight: cctx.Uint64("weight"), CanSeal: cctx.Bool("seal"), CanStore: cctx.Bool("store"), diff --git a/node/config/storage.go b/node/config/storage.go index 3e6c9d2e2..834447c3a 100644 --- a/node/config/storage.go +++ b/node/config/storage.go @@ -18,15 +18,6 @@ type StorageConfig struct { StoragePaths []LocalPath } -// [path]/metadata.json -type StorageMeta struct { - ID string - Weight uint64 // 0 = readonly - - CanSeal bool - CanStore bool -} - func StorageFromFile(path string, def *StorageConfig) (*StorageConfig, error) { file, err := os.Open(path) switch { diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 9bbc58f1e..4082ff45e 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -3,6 +3,7 @@ package impl import ( "context" "encoding/json" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" "net/http" "os" "strconv" @@ -151,15 +152,15 @@ func (sm *StorageMinerAPI) WorkerConnect(ctx context.Context, url string) error panic("todo register ") } -func (sm *StorageMinerAPI) WorkerAttachStorage(ctx context.Context, si api.StorageInfo) error { +func (sm *StorageMinerAPI) StorageAttach(ctx context.Context, si stores.StorageInfo) error { panic("implement me") } -func (sm *StorageMinerAPI) WorkerDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error { +func (sm *StorageMinerAPI) StorageDeclareSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error { panic("implement me") } -func (sm *StorageMinerAPI) FindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]api.StorageInfo, error) { +func (sm *StorageMinerAPI) StorageFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]stores.StorageInfo, error) { panic("implement me") } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 926541359..b5798c754 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -50,7 +50,7 @@ func defConfForType(t RepoType) interface{} { case StorageMiner: return config.DefaultStorageMiner() case Worker: - return &struct {}{} + return &struct{}{} default: panic(fmt.Sprintf("unknown RepoType(%d)", int(t))) } diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 4dd7dc62e..016b0f21e 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -2,6 +2,7 @@ package repo import ( "encoding/json" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" "io/ioutil" "os" "path/filepath" @@ -81,8 +82,8 @@ func (lmem *lockedMemRepo) Path() string { panic(err) } - b, err := json.MarshalIndent(&config.StorageMeta{ - ID: uuid.New().String(), + b, err := json.MarshalIndent(&stores.StorageMeta{ + ID: stores.ID(uuid.New().String()), Weight: 10, CanSeal: true, CanStore: true, diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go index 1b5d9859a..59948058f 100644 --- a/storage/sealmgr/advmgr/local.go +++ b/storage/sealmgr/advmgr/local.go @@ -12,15 +12,14 @@ import ( storage2 "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/storage/sealmgr/stores" "github.com/filecoin-project/lotus/storage/sealmgr" ) type LocalWorker struct { - scfg *sectorbuilder.Config - storage stores.Store + scfg *sectorbuilder.Config + storage stores.Store localStore *stores.Local } @@ -30,7 +29,7 @@ func NewLocalWorker(ma address.Address, spt abi.RegisteredProof, store stores.St panic(err) } return &LocalWorker{ - scfg: §orbuilder.Config{ + scfg: §orbuilder.Config{ SealProofType: spt, PoStProofType: ppt, Miner: ma, @@ -51,7 +50,7 @@ func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, id abi.Sect } return l.w.storage.AcquireSector(ctx, abi.SectorID{ - Miner: abi.ActorID(mid), + Miner: abi.ActorID(mid), Number: id, }, existing, allocate, sealing) } @@ -123,7 +122,7 @@ func (l *LocalWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, }, nil } -func (l *LocalWorker) Paths(context.Context) ([]api.StoragePath, error) { +func (l *LocalWorker) Paths(context.Context) ([]stores.StoragePath, error) { return l.localStore.Local(), nil } diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 83d580c7c..f7360a3c7 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -13,7 +13,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/storage/sealmgr/stores" "github.com/filecoin-project/specs-actors/actors/abi" @@ -33,7 +32,7 @@ type Worker interface { sectorbuilder.Sealer TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) - Paths(context.Context) ([]api.StoragePath, error) + Paths(context.Context) ([]stores.StoragePath, error) } type Manager struct { @@ -41,8 +40,8 @@ type Manager struct { scfg *sectorbuilder.Config sc SectorIDCounter - ls stores.LocalStorage - storage *stores.Local + ls stores.LocalStorage + storage *stores.Local remoteHnd *stores.FetchHandler storage2.Prover @@ -71,8 +70,8 @@ func New(ls stores.LocalStorage, cfg *sectorbuilder.Config, sc SectorIDCounter) scfg: cfg, sc: sc, - ls: ls, - storage: stor, + ls: ls, + storage: stor, remoteHnd: &stores.FetchHandler{Store: stor}, Prover: prover, @@ -116,9 +115,9 @@ func (m *Manager) ReadPieceFromSealedSector(context.Context, abi.SectorNumber, s panic("implement me") } -func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []config.StorageMeta) ([]Worker, map[int]config.StorageMeta) { +func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.StorageMeta) ([]Worker, map[int]stores.StorageMeta) { var workers []Worker - paths := map[int]config.StorageMeta{} + paths := map[int]stores.StorageMeta{} for i, worker := range m.workers { tt, err := worker.TaskTypes(context.TODO()) @@ -137,7 +136,7 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []config.Stor } // check if the worker has access to the path we selected - var st *config.StorageMeta + var st *stores.StorageMeta for _, p := range phs { for _, meta := range inPaths { if p.ID == meta.ID { @@ -164,7 +163,7 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []config.Stor func (m *Manager) AddPiece(ctx context.Context, sn abi.SectorNumber, existingPieces []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { // TODO: consider multiple paths vs workers when initially allocating - var best []config.StorageMeta + var best []stores.StorageMeta var err error if len(existingPieces) == 0 { // new best, err = m.storage.FindBestAllocStorage(sectorbuilder.FTUnsealed, true) diff --git a/storage/sealmgr/advmgr/remote.go b/storage/sealmgr/advmgr/remote.go index f1eab1891..066f6af1a 100644 --- a/storage/sealmgr/advmgr/remote.go +++ b/storage/sealmgr/advmgr/remote.go @@ -17,7 +17,7 @@ type remote struct { } func (r *remote) AddPiece(ctx context.Context, sector abi.SectorNumber, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage2.Data) (abi.PieceInfo, error) { - return abi.PieceInfo{},xerrors.New("unsupported") + return abi.PieceInfo{}, xerrors.New("unsupported") } func ConnectRemote(ctx context.Context, fa api.FullNode, url string) (*remote, error) { diff --git a/storage/sealmgr/advmgr/roprov.go b/storage/sealmgr/advmgr/roprov.go index 2269a3257..4a8fa8261 100644 --- a/storage/sealmgr/advmgr/roprov.go +++ b/storage/sealmgr/advmgr/roprov.go @@ -21,7 +21,7 @@ func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorNumbe } return l.stor.AcquireSector(ctx, abi.SectorID{ - Miner: l.miner, + Miner: l.miner, Number: id, }, existing, allocate, sealing) } diff --git a/storage/sealmgr/stores/index.go b/storage/sealmgr/stores/index.go new file mode 100644 index 000000000..37118764e --- /dev/null +++ b/storage/sealmgr/stores/index.go @@ -0,0 +1,128 @@ +package stores + +import ( + "context" + "net/url" + gopath "path" + "sync" + + "golang.org/x/xerrors" + + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" + + "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" +) + +// ID identifies sector storage by UUID. One sector storage should map to one +// filesystem, local or networked / shared by multiple machines +type ID string + +type StorageInfo struct { + ID ID + URLs []string // TODO: Support non-http transports + Cost int + + CanSeal bool + CanStore bool +} + +type SectorIndex interface { // part of storage-miner api + StorageAttach(context.Context, StorageInfo) error + + StorageDeclareSector(ctx context.Context, storageId ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error + StorageFindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]StorageInfo, error) +} + +type decl struct { + abi.SectorID + sectorbuilder.SectorFileType +} + +type Index struct { + lk sync.Mutex + + sectors map[decl][]ID + stores map[ID]*StorageInfo +} + +func NewIndex() *Index { + return &Index{ + sectors: map[decl][]ID{}, + stores: map[ID]*StorageInfo{}, + } +} + +func (i *Index) StorageAttach(ctx context.Context, si StorageInfo) error { + i.lk.Lock() + defer i.lk.Unlock() + + if _, ok := i.stores[si.ID]; ok { + for _, u := range si.URLs { + if _, err := url.Parse(u); err != nil { + return xerrors.Errorf("failed to parse url %s: %w", si.URLs, err) + } + } + + i.stores[si.ID].URLs = append(i.stores[si.ID].URLs, si.URLs...) + return nil + } + i.stores[si.ID] = &si + return nil +} + +func (i *Index) StorageDeclareSector(ctx context.Context, storageId ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error { + i.lk.Lock() + defer i.lk.Unlock() + + d := decl{s, ft} + + for _, sid := range i.sectors[d] { + if sid == storageId { + log.Warnf("sector %v redeclared in %s", storageId) + return nil + } + } + + i.sectors[d] = append(i.sectors[d], storageId) + return nil +} + +func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sectorbuilder.SectorFileType) ([]StorageInfo, error) { + i.lk.Lock() + defer i.lk.Unlock() + + storageIDs := i.sectors[decl{s, ft}] + out := make([]StorageInfo, len(storageIDs)) + + for j, id := range storageIDs { + st, ok := i.stores[id] + if !ok { + log.Warnf("storage %s is not present in sector index (referenced by sector %v)", id, s) + continue + } + + urls := make([]string, len(st.URLs)) + for k, u := range st.URLs { + rl, err := url.Parse(u) + if err != nil { + return nil, xerrors.Errorf("failed to parse url: %w", err) + } + + rl.Path = gopath.Join(rl.Path, ft.String(), sectorutil.SectorName(s)) + urls[k] = rl.String() + } + + out[j] = StorageInfo{ + ID: id, + URLs: nil, + Cost: st.Cost, + CanSeal: st.CanSeal, + CanStore: st.CanStore, + } + } + + return out, nil +} + +var _ SectorIndex = &Index{} diff --git a/storage/sealmgr/stores/local.go b/storage/sealmgr/stores/local.go index 88de7bf7b..e5f3f9548 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sealmgr/stores/local.go @@ -12,11 +12,29 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" ) +type StoragePath struct { + ID ID + Weight uint64 + + LocalPath string + + CanSeal bool + CanStore bool +} + +// [path]/sectorstore.json +type StorageMeta struct { + ID ID + Weight uint64 // 0 = readonly + + CanSeal bool + CanStore bool +} + type LocalStorage interface { GetStorage() (config.StorageConfig, error) SetStorage(func(*config.StorageConfig)) error @@ -36,7 +54,7 @@ type Local struct { type path struct { lk sync.Mutex - meta config.StorageMeta + meta StorageMeta local string sectors map[abi.SectorID]sectorbuilder.SectorFileType @@ -55,7 +73,7 @@ func (st *Local) OpenPath(p string) error { return xerrors.Errorf("reading storage metadata for %s: %w", p, err) } - var meta config.StorageMeta + var meta StorageMeta if err := json.Unmarshal(mb, &meta); err != nil { return xerrors.Errorf("unmarshalling storage metadata for %s: %w", p, err) } @@ -151,7 +169,7 @@ func (st *Local) acquireSector(ctx context.Context, sid abi.SectorID, existing s spath := filepath.Join(p.local, fileType.String(), sectorutil.SectorName(sid)) sectorutil.SetPathByType(&out, fileType, spath) - sectorutil.SetPathByType(&storageIDs, fileType, p.meta.ID) + sectorutil.SetPathByType(&storageIDs, fileType, string(p.meta.ID)) existing ^= fileType } @@ -162,7 +180,8 @@ func (st *Local) acquireSector(ctx context.Context, sid abi.SectorID, existing s continue } - var best, bestID string + var best string + var bestID ID for _, p := range st.paths { if sealing && !p.meta.CanSeal { @@ -190,15 +209,15 @@ func (st *Local) acquireSector(ctx context.Context, sid abi.SectorID, existing s } sectorutil.SetPathByType(&out, fileType, best) - sectorutil.SetPathByType(&storageIDs, fileType, bestID) + sectorutil.SetPathByType(&storageIDs, fileType, string(bestID)) allocate ^= fileType } return out, storageIDs, st.localLk.RUnlock, nil } -func (st *Local) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sealing bool) ([]config.StorageMeta, error) { - var out []config.StorageMeta +func (st *Local) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sealing bool) ([]StorageMeta, error) { + var out []StorageMeta for _, p := range st.paths { if sealing && !p.meta.CanSeal { @@ -221,8 +240,8 @@ func (st *Local) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sea return out, nil } -func (st *Local) FindSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbuilder.SectorFileType) ([]config.StorageMeta, error) { - var out []config.StorageMeta +func (st *Local) FindSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbuilder.SectorFileType) ([]StorageMeta, error) { + var out []StorageMeta for _, p := range st.paths { p.lk.Lock() t := p.sectors[abi.SectorID{ @@ -242,14 +261,14 @@ func (st *Local) FindSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbuil return out, nil } -func (st *Local) Local() []api.StoragePath { - var out []api.StoragePath +func (st *Local) Local() []StoragePath { + var out []StoragePath for _, p := range st.paths { if p.local == "" { continue } - out = append(out, api.StoragePath{ + out = append(out, StoragePath{ ID: p.meta.ID, Weight: p.meta.Weight, LocalPath: p.local, diff --git a/storage/sealmgr/stores/remote.go b/storage/sealmgr/stores/remote.go index 05b2ce5dc..b0b19a884 100644 --- a/storage/sealmgr/stores/remote.go +++ b/storage/sealmgr/stores/remote.go @@ -15,15 +15,14 @@ import ( "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/lib/tarutil" "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" ) type Remote struct { - local *Local + local *Local remote SectorIndex - auth http.Header + auth http.Header fetchLk sync.Mutex // TODO: this can be much smarter // TODO: allow multiple parallel fetches @@ -32,17 +31,12 @@ type Remote struct { func NewRemote(local *Local, remote SectorIndex, auth http.Header) *Remote { return &Remote{ - local: local, - remote: remote, - auth: auth, + local: local, + remote: remote, + auth: auth, } } -type SectorIndex interface { - StorageDeclareSector(ctx context.Context, storageId string, s abi.SectorID) error - StorageFindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]api.StorageInfo, error) -} - func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { if existing|allocate != existing^allocate { return sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector") @@ -74,7 +68,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec done = mergeDone(done, rdone) sectorutil.SetPathByType(&paths, fileType, ap) - if err := r.remote.StorageDeclareSector(ctx, storageID, s); err != nil { + if err := r.remote.StorageDeclareSector(ctx, storageID, s, fileType); err != nil { log.Warnf("declaring sector %v in %s failed: %+v", s, storageID, err) } } @@ -82,7 +76,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec return paths, done, nil } -func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, string, func(), error) { +func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, ID, func(), error) { si, err := r.remote.StorageFindSector(ctx, s, fileType) if err != nil { return "", "", nil, err @@ -111,7 +105,7 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType if merr != nil { log.Warnw("acquireFromRemote encountered errors when fetching sector from remote", "errors", merr) } - return dest, storageID, done, nil + return dest, ID(storageID), done, nil } } @@ -119,7 +113,6 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType return "", "", nil, xerrors.Errorf("failed to acquire sector %v from remote: %w", s, merr) } - func (r *Remote) fetch(url, outname string) error { log.Infof("Fetch %s -> %s", url, outname) From 96730bae757372c5587f35406c1815a55c26a915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 13 Mar 2020 17:54:55 +0100 Subject: [PATCH 09/45] workers: Report sectors paths after worker creates them --- cmd/lotus-seal-worker/main.go | 2 +- node/builder.go | 1 + node/impl/storminer.go | 13 +-------- storage/sealmgr/advmgr/local.go | 37 +++++++++++++++++++++----- storage/sealmgr/advmgr/roprov.go | 4 ++- storage/sealmgr/stores/http_handler.go | 2 +- storage/sealmgr/stores/index.go | 21 ++++++++++----- storage/sealmgr/stores/interface.go | 2 +- storage/sealmgr/stores/local.go | 7 +---- storage/sealmgr/stores/remote.go | 15 ++++++----- 10 files changed, 62 insertions(+), 42 deletions(-) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index c3e867da3..e9845e29c 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -167,7 +167,7 @@ var runCmd = &cli.Command{ remote := stores.NewRemote(localStore, nodeApi, sminfo.AuthHeader()) workerApi := &worker{ - LocalWorker: advmgr.NewLocalWorker(act, spt, remote, localStore), + LocalWorker: advmgr.NewLocalWorker(act, spt, remote, localStore, stores.NewIndex()), } mux := mux.NewRouter() diff --git a/node/builder.go b/node/builder.go index 661edcccb..7ec2b3943 100644 --- a/node/builder.go +++ b/node/builder.go @@ -258,6 +258,7 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, + Override(new(*stores.Index), stores.NewIndex()), Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig), Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), Override(new(advmgr.SectorIDCounter), modules.SectorIDCounter), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 4082ff45e..ac4e3a282 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -39,6 +39,7 @@ type StorageMinerAPI struct { BlockMiner *miner.Miner Full api.FullNode StorageMgr *advmgr.Manager `optional:"true"` + *stores.Index } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { @@ -152,18 +153,6 @@ func (sm *StorageMinerAPI) WorkerConnect(ctx context.Context, url string) error panic("todo register ") } -func (sm *StorageMinerAPI) StorageAttach(ctx context.Context, si stores.StorageInfo) error { - panic("implement me") -} - -func (sm *StorageMinerAPI) StorageDeclareSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error { - panic("implement me") -} - -func (sm *StorageMinerAPI) StorageFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]stores.StorageInfo, error) { - panic("implement me") -} - func (sm *StorageMinerAPI) MarketImportDealData(ctx context.Context, propCid cid.Cid, path string) error { fi, err := os.Open(path) if err != nil { diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go index 59948058f..7d6b5f779 100644 --- a/storage/sealmgr/advmgr/local.go +++ b/storage/sealmgr/advmgr/local.go @@ -8,22 +8,25 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" storage2 "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" - "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" ) +var pathTypes = []sectorbuilder.SectorFileType{sectorbuilder.FTUnsealed, sectorbuilder.FTSealed, sectorbuilder.FTCache} + type LocalWorker struct { scfg *sectorbuilder.Config storage stores.Store localStore *stores.Local + sindex stores.SectorIndex } -func NewLocalWorker(ma address.Address, spt abi.RegisteredProof, store stores.Store, local *stores.Local) *LocalWorker { +func NewLocalWorker(ma address.Address, spt abi.RegisteredProof, store stores.Store, local *stores.Local, sindex stores.SectorIndex) *LocalWorker { ppt, err := spt.RegisteredPoStProof() if err != nil { panic(err) @@ -36,6 +39,7 @@ func NewLocalWorker(ma address.Address, spt abi.RegisteredProof, store stores.St }, storage: store, localStore: local, + sindex: sindex, } } @@ -49,10 +53,31 @@ func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, id abi.Sect return sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("get miner ID: %w", err) } - return l.w.storage.AcquireSector(ctx, abi.SectorID{ + sector := abi.SectorID{ Miner: abi.ActorID(mid), Number: id, - }, existing, allocate, sealing) + } + + paths, storageIDs, done, err := l.w.storage.AcquireSector(ctx, sector, existing, allocate, sealing) + if err != nil { + return sectorbuilder.SectorPaths{}, nil, err + } + + return paths, func() { + done() + + for _, fileType := range pathTypes { + if fileType&allocate == 0 { + continue + } + + sid := sectorutil.PathByType(storageIDs, fileType) + + if err := l.w.sindex.StorageDeclareSector(ctx, stores.ID(sid), sector, fileType); err != nil { + log.Errorf("declare sector error: %+v", err) + } + } + }, nil } func (l *LocalWorker) sb() (sectorbuilder.Basic, error) { diff --git a/storage/sealmgr/advmgr/roprov.go b/storage/sealmgr/advmgr/roprov.go index 4a8fa8261..ce40d2ad3 100644 --- a/storage/sealmgr/advmgr/roprov.go +++ b/storage/sealmgr/advmgr/roprov.go @@ -20,8 +20,10 @@ func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorNumbe return sectorbuilder.SectorPaths{}, nil, xerrors.New("read-only storage") } - return l.stor.AcquireSector(ctx, abi.SectorID{ + p, _, done, err := l.stor.AcquireSector(ctx, abi.SectorID{ Miner: l.miner, Number: id, }, existing, allocate, sealing) + + return p, done, err } diff --git a/storage/sealmgr/stores/http_handler.go b/storage/sealmgr/stores/http_handler.go index 75cb080b5..f1fa42c8e 100644 --- a/storage/sealmgr/stores/http_handler.go +++ b/storage/sealmgr/stores/http_handler.go @@ -44,7 +44,7 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ if err != nil { return } - paths, done, err := handler.Store.AcquireSector(r.Context(), id, ft, 0, false) + paths, _, done, err := handler.Store.AcquireSector(r.Context(), id, ft, 0, false) if err != nil { return } diff --git a/storage/sealmgr/stores/index.go b/storage/sealmgr/stores/index.go index 37118764e..68fdccfcb 100644 --- a/storage/sealmgr/stores/index.go +++ b/storage/sealmgr/stores/index.go @@ -75,16 +75,23 @@ func (i *Index) StorageDeclareSector(ctx context.Context, storageId ID, s abi.Se i.lk.Lock() defer i.lk.Unlock() - d := decl{s, ft} - - for _, sid := range i.sectors[d] { - if sid == storageId { - log.Warnf("sector %v redeclared in %s", storageId) - return nil + for _, fileType := range pathTypes { + if fileType&ft == 0 { + continue } + + d := decl{s, fileType} + + for _, sid := range i.sectors[d] { + if sid == storageId { + log.Warnf("sector %v redeclared in %s", storageId) + return nil + } + } + + i.sectors[d] = append(i.sectors[d], storageId) } - i.sectors[d] = append(i.sectors[d], storageId) return nil } diff --git a/storage/sealmgr/stores/interface.go b/storage/sealmgr/stores/interface.go index a6ea5ec99..6e855307d 100644 --- a/storage/sealmgr/stores/interface.go +++ b/storage/sealmgr/stores/interface.go @@ -8,5 +8,5 @@ import ( ) type Store interface { - AcquireSector(ctx context.Context, s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) + AcquireSector(ctx context.Context, s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (paths sectorbuilder.SectorPaths, stores sectorbuilder.SectorPaths, done func(), err error) } diff --git a/storage/sealmgr/stores/local.go b/storage/sealmgr/stores/local.go index e5f3f9548..26e1896a1 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sealmgr/stores/local.go @@ -133,12 +133,7 @@ func (st *Local) open() error { return nil } -func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { - out, _, done, err := st.acquireSector(ctx, sid, existing, allocate, sealing) - return out, done, err -} - -func (st *Local) acquireSector(ctx context.Context, sid abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, sectorbuilder.SectorPaths, func(), error) { +func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, sectorbuilder.SectorPaths, func(), error) { if existing|allocate != existing^allocate { return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector") } diff --git a/storage/sealmgr/stores/remote.go b/storage/sealmgr/stores/remote.go index b0b19a884..04272b450 100644 --- a/storage/sealmgr/stores/remote.go +++ b/storage/sealmgr/stores/remote.go @@ -37,17 +37,17 @@ func NewRemote(local *Local, remote SectorIndex, auth http.Header) *Remote { } } -func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { +func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, sectorbuilder.SectorPaths, func(), error) { if existing|allocate != existing^allocate { - return sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector") + return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, xerrors.New("can't both find and allocate a sector") } r.fetchLk.Lock() defer r.fetchLk.Unlock() - paths, done, err := r.local.AcquireSector(ctx, s, existing, allocate, sealing) + paths, stores, done, err := r.local.AcquireSector(ctx, s, existing, allocate, sealing) if err != nil { - return sectorbuilder.SectorPaths{}, nil, err + return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, err } for _, fileType := range pathTypes { @@ -62,18 +62,19 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec ap, storageID, rdone, err := r.acquireFromRemote(ctx, s, fileType, sealing) if err != nil { done() - return sectorbuilder.SectorPaths{}, nil, err + return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, err } done = mergeDone(done, rdone) sectorutil.SetPathByType(&paths, fileType, ap) + sectorutil.SetPathByType(&stores, fileType, string(storageID)) if err := r.remote.StorageDeclareSector(ctx, storageID, s, fileType); err != nil { log.Warnf("declaring sector %v in %s failed: %+v", s, storageID, err) } } - return paths, done, nil + return paths, stores, done, nil } func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, ID, func(), error) { @@ -86,7 +87,7 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType return si[i].Cost < si[j].Cost }) - apaths, ids, done, err := r.local.acquireSector(ctx, s, 0, fileType, sealing) + apaths, ids, done, err := r.local.AcquireSector(ctx, s, 0, fileType, sealing) if err != nil { return "", "", nil, xerrors.Errorf("allocate local sector for fetching: %w", err) } From 7e997e40f375e85cb94e3d4417aee73c423d3b47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 16 Mar 2020 18:50:07 +0100 Subject: [PATCH 10/45] workers: Simple storage diagnostics --- api/api_storage.go | 4 ++ api/apistruct/struct.go | 10 +++++ cmd/lotus-seal-worker/main.go | 49 ++++++++++++++------- cmd/lotus-storage-miner/init.go | 4 +- cmd/lotus-storage-miner/run.go | 14 ++---- cmd/lotus-storage-miner/storage.go | 52 ++++++++++++++++++++++- go.mod | 1 + go.sum | 2 - node/builder.go | 22 +++++++--- node/modules/dtypes/api.go | 7 ++- storage/sealmgr/advmgr/manager.go | 8 +++- storage/sealmgr/stores/index.go | 68 +++++++++++++++++++++++++++--- storage/sealmgr/stores/local.go | 14 ++++++ 13 files changed, 209 insertions(+), 46 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 94cb1fdfa..fe7e4eb51 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -108,6 +108,10 @@ type StorageMiner interface { SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error + StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) + + StorageInfo(context.Context, stores.ID) (stores.StorageInfo, error) + // WorkerConnect tells the node to connect to workers RPC WorkerConnect(context.Context, string) error stores.SectorIndex diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 31d9cfc71..6cca7fecb 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -185,6 +185,8 @@ type StorageMinerStruct struct { WorkerAttachStorage func(context.Context, stores.StorageInfo) error `perm:"admin"` StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, sectorbuilder.SectorFileType) error `perm:"admin"` StorageFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]stores.StorageInfo, error) `perm:"admin"` + StorageList func(ctx context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` + StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -664,6 +666,14 @@ func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.Secto return c.Internal.StorageFindSector(ctx, si, types) } +func (c *StorageMinerStruct) StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) { + return c.Internal.StorageList(ctx) +} + +func (c *StorageMinerStruct) StorageInfo(ctx context.Context, id stores.ID) (stores.StorageInfo, error) { + return c.Internal.StorageInfo(ctx, id) +} + func (c *StorageMinerStruct) MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error { return c.Internal.MarketImportDealData(ctx, propcid, path) } diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index e9845e29c..2c55af1ec 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "net" "net/http" "os" "os/signal" @@ -13,8 +14,6 @@ import ( "gopkg.in/urfave/cli.v2" paramfetch "github.com/filecoin-project/go-paramfetch" - manet "github.com/multiformats/go-multiaddr-net" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/build" @@ -31,11 +30,6 @@ var log = logging.Logger("main") const FlagStorageRepo = "workerrepo" -const ( - workers = 1 // TODO: Configurability - transfers = 1 -) - func main() { lotuslog.SetupLogLevels() @@ -81,11 +75,23 @@ func main() { var runCmd = &cli.Command{ Name: "run", Usage: "Start lotus worker", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "address", + Usage: "Locally reachable address", + }, + }, Action: func(cctx *cli.Context) error { if !cctx.Bool("enable-gpu-proving") { os.Setenv("BELLMAN_NO_GPU", "true") } + if cctx.String("address") == "" { + return xerrors.Errorf("--address flag is required") + } + + // Connect to storage-miner + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return xerrors.Errorf("getting miner api: %w", err) @@ -107,6 +113,8 @@ var runCmd = &cli.Command{ log.Warn("Shutting down..") }() + // Check params + act, err := nodeApi.ActorAddress(ctx) if err != nil { return err @@ -120,6 +128,8 @@ var runCmd = &cli.Command{ return xerrors.Errorf("get params: %w", err) } + // Open repo + repoPath := cctx.String(FlagStorageRepo) r, err := repo.NewFS(repoPath) if err != nil { @@ -144,16 +154,15 @@ var runCmd = &cli.Command{ return err } - endpoint, err := r.APIEndpoint() - if err != nil { + if err := stores.DeclareLocalStorage( + ctx, + nodeApi, + localStore, + []string{"http://" + cctx.String("address") + "/remote"}, // TODO: Less hardcoded + 1); err != nil { return err } - - lst, err := manet.Listen(endpoint) - if err != nil { - return xerrors.Errorf("could not listen: %w", err) - } - + // Setup remote sector store _, spt, err := api.ProofTypeFromSectorSize(ssize) if err != nil { return xerrors.Errorf("getting proof type: %w", err) @@ -166,6 +175,8 @@ var runCmd = &cli.Command{ remote := stores.NewRemote(localStore, nodeApi, sminfo.AuthHeader()) + // Create / expose the worker + workerApi := &worker{ LocalWorker: advmgr.NewLocalWorker(act, spt, remote, localStore, stores.NewIndex()), } @@ -195,8 +206,14 @@ var runCmd = &cli.Command{ } log.Warn("Graceful shutdown successful") }() + signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT) - return srv.Serve(manet.NetListener(lst)) + nl, err := net.Listen("tcp4", cctx.String("address")) + if err != nil { + return err + } + + return srv.Serve(nl) }, } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index fdf2e486b..2c91ebbab 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -392,11 +392,11 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return err } - smgr, err := advmgr.New(lr, §orbuilder.Config{ + smgr, err := advmgr.New(lr, stores.NewIndex(), §orbuilder.Config{ SealProofType: spt, PoStProofType: ppt, Miner: a, - }, nil) + }, nil, nil) if err != nil { return err } diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 080dabcee..1454d8b42 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -22,6 +22,7 @@ import ( "github.com/filecoin-project/lotus/lib/jsonrpc" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" + "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/repo" ) @@ -29,10 +30,6 @@ var runCmd = &cli.Command{ Name: "run", Usage: "Start a lotus storage miner process", Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "api", - Value: "2345", - }, &cli.BoolFlag{ Name: "enable-gpu-proving", Usage: "enable use of GPU for mining operations", @@ -93,13 +90,8 @@ var runCmd = &cli.Command{ node.Repo(r), node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("api") }, - node.Override(node.SetApiEndpointKey, func(lr repo.LockedRepo) error { - apima, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/" + - cctx.String("api")) - if err != nil { - return err - } - return lr.SetAPIEndpoint(apima) + node.Override(new(dtypes.APIEndpoint), func() (dtypes.APIEndpoint, error) { + return multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/" + cctx.String("api")) })), node.Override(new(api.FullNode), nodeApi), ) diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 31ab45c7c..cf07a5ece 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -2,7 +2,7 @@ package main import ( "encoding/json" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "fmt" "io/ioutil" "os" "path/filepath" @@ -12,7 +12,10 @@ import ( "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" + "github.com/filecoin-project/go-sectorbuilder" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" ) const metaFile = "sectorstore.json" @@ -22,6 +25,7 @@ var storageCmd = &cli.Command{ Usage: "manage sector storage", Subcommands: []*cli.Command{ storageAttachCmd, + storageListCmd, }, } @@ -103,3 +107,49 @@ var storageAttachCmd = &cli.Command{ return nodeApi.StorageAddLocal(ctx, p) }, } + +var storageListCmd = &cli.Command{ + Name: "list", + Action: func(cctx *cli.Context) error { + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := lcli.ReqContext(cctx) + + st, err := nodeApi.StorageList(ctx) + if err != nil { + return err + } + + for id, sectors := range st { + var u, s, c int + for _, decl := range sectors { + if decl.SectorFileType§orbuilder.FTUnsealed > 0 { + u++ + } + if decl.SectorFileType§orbuilder.FTSealed > 0 { + s++ + } + if decl.SectorFileType§orbuilder.FTCache > 0 { + c++ + } + } + + fmt.Printf("%s:\n", id) + fmt.Printf("\tUnsealed: %d; Sealed: %d; Caches: %d\n", u, s, c) + + si, err := nodeApi.StorageInfo(ctx, id) + if err != nil { + return err + } + fmt.Printf("\tSeal: %t; Store: %t; Cost: %d\n", si.CanSeal, si.CanStore, si.Cost) + for _, l := range si.URLs { + fmt.Printf("\tReachable %s\n", l) // TODO; try pinging maybe?? print latency? + } + } + + return nil + }, +} diff --git a/go.mod b/go.mod index 7cf2b95ed..e860fa423 100644 --- a/go.mod +++ b/go.mod @@ -101,6 +101,7 @@ require ( go.uber.org/multierr v1.4.0 go.uber.org/zap v1.13.0 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 + golang.org/x/tools v0.0.0-20191216173652-a0e659d51361 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 gotest.tools v2.2.0+incompatible diff --git a/go.sum b/go.sum index 05d005db3..b24ce8be3 100644 --- a/go.sum +++ b/go.sum @@ -96,7 +96,6 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fd/go-nat v1.0.0/go.mod h1:BTBu/CKvMmOMUPkKVef1pngt2WFH/lg7E6yQnulfp6E= @@ -122,7 +121,6 @@ github.com/filecoin-project/go-fil-markets v0.0.0-20200304003055-d449a980d4bd h1 github.com/filecoin-project/go-fil-markets v0.0.0-20200304003055-d449a980d4bd/go.mod h1:rfRwhd3ujcCXnD4N9oEM2wjh8GRZGoeNXME+UPG/9ts= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 h1:92PET+sx1Hb4W/8CgFwGuxaKbttwY+UNspYZTvXY0vs= github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE= -github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU= github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 h1:eYxi6vI5CyeXD15X1bB3bledDXbqKxqf0wQzTLgwYwA= github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f56/go.mod h1:tzTc9BxxSbjlIzhFwm5h9oBkXKkRuLxeiWspntwnKyw= diff --git a/node/builder.go b/node/builder.go index 7ec2b3943..1c7b5838b 100644 --- a/node/builder.go +++ b/node/builder.go @@ -16,6 +16,7 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" record "github.com/libp2p/go-libp2p-record" "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" "go.uber.org/fx" "golang.org/x/xerrors" @@ -318,15 +319,22 @@ func StorageMiner(out *api.StorageMiner) Option { func ConfigCommon(cfg *config.Common) Option { return Options( func(s *Settings) error { s.Config = true; return nil }, - - Override(SetApiEndpointKey, func(lr repo.LockedRepo) error { - apima, err := multiaddr.NewMultiaddr(cfg.API.ListenAddress) - if err != nil { - return err - } - return lr.SetAPIEndpoint(apima) + Override(new(dtypes.APIEndpoint), func() (dtypes.APIEndpoint, error) { + return multiaddr.NewMultiaddr(cfg.API.ListenAddress) }), + Override(SetApiEndpointKey, func(lr repo.LockedRepo, e dtypes.APIEndpoint) error { + return lr.SetAPIEndpoint(e) + }), + Override(new(advmgr.URLs), func(e dtypes.APIEndpoint) (advmgr.URLs, error) { + _, ip, err := manet.DialArgs(e) + if err != nil { + return nil, xerrors.Errorf("getting api endpoint dial args: %w", err) + } + var urls advmgr.URLs + urls = append(urls, "http://"+ip+"/remote") // TODO: This makes assumptions, and probably bad ones too + return urls, nil + }), ApplyIf(func(s *Settings) bool { return s.Online }, Override(StartListeningKey, lp2p.StartListening(cfg.Libp2p.ListenAddresses)), Override(ConnectionManagerKey, lp2p.ConnectionManager( diff --git a/node/modules/dtypes/api.go b/node/modules/dtypes/api.go index c0e57d911..d57b05cfa 100644 --- a/node/modules/dtypes/api.go +++ b/node/modules/dtypes/api.go @@ -1,5 +1,10 @@ package dtypes -import "github.com/gbrlsnchs/jwt/v3" +import ( + "github.com/gbrlsnchs/jwt/v3" + "github.com/multiformats/go-multiaddr" +) type APIAlg jwt.HMACSHA + +type APIEndpoint multiaddr.Multiaddr diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index f7360a3c7..b2eee7d61 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -24,6 +24,8 @@ import ( var log = logging.Logger("advmgr") +type URLs []string + type SectorIDCounter interface { Next() (abi.SectorNumber, error) } @@ -47,12 +49,16 @@ type Manager struct { storage2.Prover } -func New(ls stores.LocalStorage, cfg *sectorbuilder.Config, sc SectorIDCounter) (*Manager, error) { +func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, sc SectorIDCounter, urls URLs) (*Manager, error) { stor, err := stores.NewLocal(ls) if err != nil { return nil, err } + if err := stores.DeclareLocalStorage(context.TODO(), si, stor, urls, 10); err != nil { + log.Errorf("Declaring local storage failed: %+v") + } + mid, err := address.IDFromAddress(cfg.Miner) if err != nil { return nil, xerrors.Errorf("getting miner id: %w", err) diff --git a/storage/sealmgr/stores/index.go b/storage/sealmgr/stores/index.go index 68fdccfcb..157e2ccf2 100644 --- a/storage/sealmgr/stores/index.go +++ b/storage/sealmgr/stores/index.go @@ -34,7 +34,7 @@ type SectorIndex interface { // part of storage-miner api StorageFindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]StorageInfo, error) } -type decl struct { +type Decl struct { abi.SectorID sectorbuilder.SectorFileType } @@ -42,17 +42,42 @@ type decl struct { type Index struct { lk sync.Mutex - sectors map[decl][]ID + sectors map[Decl][]ID stores map[ID]*StorageInfo } func NewIndex() *Index { return &Index{ - sectors: map[decl][]ID{}, + sectors: map[Decl][]ID{}, stores: map[ID]*StorageInfo{}, } } +func (i *Index) StorageList(ctx context.Context) (map[ID][]Decl, error) { + byID := map[ID]map[abi.SectorID]sectorbuilder.SectorFileType{} + + for id := range i.stores { + byID[id] = map[abi.SectorID]sectorbuilder.SectorFileType{} + } + for decl, ids := range i.sectors { + for _, id := range ids { + byID[id][decl.SectorID] |= decl.SectorFileType + } + } + + out := map[ID][]Decl{} + for id, m := range byID { + for sectorID, fileType := range m { + out[id] = append(out[id], Decl{ + SectorID: sectorID, + SectorFileType: fileType, + }) + } + } + + return out, nil +} + func (i *Index) StorageAttach(ctx context.Context, si StorageInfo) error { i.lk.Lock() defer i.lk.Unlock() @@ -80,7 +105,7 @@ func (i *Index) StorageDeclareSector(ctx context.Context, storageId ID, s abi.Se continue } - d := decl{s, fileType} + d := Decl{s, fileType} for _, sid := range i.sectors[d] { if sid == storageId { @@ -99,7 +124,7 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sector i.lk.Lock() defer i.lk.Unlock() - storageIDs := i.sectors[decl{s, ft}] + storageIDs := i.sectors[Decl{s, ft}] out := make([]StorageInfo, len(storageIDs)) for j, id := range storageIDs { @@ -132,4 +157,37 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sector return out, nil } +func (i *Index) StorageInfo(ctx context.Context, id ID) (StorageInfo, error) { + si, found := i.stores[id] + if !found { + return StorageInfo{}, xerrors.Errorf("sector store not found") + } + + return *si, nil +} + +func DeclareLocalStorage(ctx context.Context, idx SectorIndex, localStore *Local, urls []string, cost int) error { + for _, path := range localStore.Local() { + err := idx.StorageAttach(ctx, StorageInfo{ + ID: path.ID, + URLs: urls, + Cost: cost, + CanSeal: path.CanSeal, + CanStore: path.CanStore, + }) + if err != nil { + log.Errorf("attaching local storage to remote: %+v") + continue + } + + for id, fileType := range localStore.List(path.ID) { + if err := idx.StorageDeclareSector(ctx, path.ID, id, fileType); err != nil { + log.Errorf("declaring sector: %+v") + } + } + } + + return nil +} + var _ SectorIndex = &Index{} diff --git a/storage/sealmgr/stores/local.go b/storage/sealmgr/stores/local.go index 26e1896a1..efac144a1 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sealmgr/stores/local.go @@ -274,3 +274,17 @@ func (st *Local) Local() []StoragePath { return out } + +func (st *Local) List(id ID) map[abi.SectorID]sectorbuilder.SectorFileType { + out := map[abi.SectorID]sectorbuilder.SectorFileType{} + for _, p := range st.paths { + if p.meta.ID != id { // TODO: not very efficient + continue + } + + for id, fileType := range p.sectors { + out[id] |= fileType + } + } + return out +} From de1c984b481f4bba585eedc13d9e4bedff3071e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 16 Mar 2020 19:46:02 +0100 Subject: [PATCH 11/45] workers: Handle init on first run --- api/apistruct/struct.go | 4 +-- cmd/lotus-seal-worker/main.go | 50 ++++++++++++++++++++++++++++++++- storage/sealmgr/stores/index.go | 2 +- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 6cca7fecb..48e83d78c 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -182,7 +182,7 @@ type StorageMinerStruct struct { SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error `perm:"write"` WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm - WorkerAttachStorage func(context.Context, stores.StorageInfo) error `perm:"admin"` + StorageAttach func(context.Context, stores.StorageInfo) error `perm:"admin"` StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, sectorbuilder.SectorFileType) error `perm:"admin"` StorageFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]stores.StorageInfo, error) `perm:"admin"` StorageList func(ctx context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` @@ -655,7 +655,7 @@ func (c *StorageMinerStruct) WorkerConnect(ctx context.Context, url string) erro } func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo) error { - return c.Internal.WorkerAttachStorage(ctx, si) + return c.Internal.StorageAttach(ctx, si) } func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error { diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 2c55af1ec..ba431f22e 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -2,12 +2,16 @@ package main import ( "context" + "encoding/json" + "io/ioutil" "net" "net/http" "os" "os/signal" + "path/filepath" "syscall" + "github.com/google/uuid" "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" "golang.org/x/xerrors" @@ -21,6 +25,7 @@ import ( "github.com/filecoin-project/lotus/lib/auth" "github.com/filecoin-project/lotus/lib/jsonrpc" "github.com/filecoin-project/lotus/lib/lotuslog" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" "github.com/filecoin-project/lotus/storage/sealmgr/stores" @@ -80,6 +85,10 @@ var runCmd = &cli.Command{ Name: "address", Usage: "Locally reachable address", }, + &cli.BoolFlag{ + Name: "no-local-storage", + Usage: "don't use storageminer repo for sector storage", + }, }, Action: func(cctx *cli.Context) error { if !cctx.Bool("enable-gpu-proving") { @@ -141,7 +150,46 @@ var runCmd = &cli.Command{ return err } if !ok { - return xerrors.Errorf("repo at '%s' is not initialized, run 'lotus-seal-worker init' to set it up", repoPath) + if err := r.Init(repo.Worker); err != nil { + return err + } + + lr, err := r.Lock(repo.Worker) + if err != nil { + return err + } + + var localPaths []config.LocalPath + + if !cctx.Bool("no-local-storage") { + b, err := json.MarshalIndent(&stores.StorageMeta{ + ID: stores.ID(uuid.New().String()), + Weight: 10, + CanSeal: true, + CanStore: false, + }, "", " ") + if err != nil { + return xerrors.Errorf("marshaling storage config: %w", err) + } + + if err := ioutil.WriteFile(filepath.Join(lr.Path(), "sectorstore.json"), b, 0644); err != nil { + return xerrors.Errorf("persisting storage metadata (%s): %w", filepath.Join(lr.Path(), "sectorstore.json"), err) + } + + localPaths = append(localPaths, config.LocalPath{ + Path: lr.Path(), + }) + } + + if err := lr.SetStorage(func(sc *config.StorageConfig) { + sc.StoragePaths = append(sc.StoragePaths, localPaths...) + }); err != nil { + return xerrors.Errorf("set storage config: %w", err) + } + + if err := lr.Close(); err != nil { + return xerrors.Errorf("close repo: %w", err) + } } lr, err := r.Lock(repo.Worker) diff --git a/storage/sealmgr/stores/index.go b/storage/sealmgr/stores/index.go index 157e2ccf2..df370741c 100644 --- a/storage/sealmgr/stores/index.go +++ b/storage/sealmgr/stores/index.go @@ -176,7 +176,7 @@ func DeclareLocalStorage(ctx context.Context, idx SectorIndex, localStore *Local CanStore: path.CanStore, }) if err != nil { - log.Errorf("attaching local storage to remote: %+v") + log.Errorf("attaching local storage to remote: %+v", err) continue } From d8ce5078d9d404bb73aa601dea5ce7688e895cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 17 Mar 2020 21:19:52 +0100 Subject: [PATCH 12/45] Use new specs-storage interface --- api/apistruct/struct.go | 30 +++---- cmd/lotus-seed/seed/seed.go | 19 +++-- cmd/lotus-storage-miner/init.go | 34 ++++---- go.mod | 4 +- go.sum | 4 + markets/retrievaladapter/provider.go | 12 ++- node/builder.go | 2 +- node/impl/storminer.go | 2 +- node/modules/storageminer.go | 4 +- storage/fpost_run.go | 14 +++- storage/miner.go | 9 +- storage/sbmock/sbmock.go | 54 +++++++----- storage/sealing.go | 5 ++ storage/sealing/garbage.go | 11 ++- storage/sealing/sealing.go | 32 +++++++- storage/sealing/states.go | 21 +++-- storage/sealmgr/advmgr/local.go | 118 +++++++++++++-------------- storage/sealmgr/advmgr/manager.go | 70 ++++++---------- storage/sealmgr/advmgr/remote.go | 6 +- storage/sealmgr/advmgr/roprov.go | 8 +- storage/sealmgr/simple.go | 56 ++++++------- storage/sealmgr/stores/local.go | 9 +- storage/sealmgr/types.go | 6 +- 23 files changed, 283 insertions(+), 247 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 48e83d78c..f5f20b41c 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -204,11 +204,11 @@ type WorkerStruct struct { TaskTypes func(context.Context) (map[sealmgr.TaskType]struct{}, error) `perm:"admin"` Paths func(context.Context) ([]stores.StoragePath, error) `perm:"admin"` - SealPreCommit1 func(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"` - SealPreCommit2 func(context.Context, abi.SectorNumber, storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) `perm:"admin"` - SealCommit1 func(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (storage.Commit1Out, error) `perm:"admin"` - SealCommit2 func(context.Context, abi.SectorNumber, storage.Commit1Out) (storage.Proof, error) `perm:"admin"` - FinalizeSector func(context.Context, abi.SectorNumber) error `perm:"admin"` + SealPreCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"` + SealPreCommit2 func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error) `perm:"admin"` + SealCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) `perm:"admin"` + SealCommit2 func(context.Context, abi.SectorID, storage.Commit1Out) (storage.Proof, error) `perm:"admin"` + FinalizeSector func(context.Context, abi.SectorID) error `perm:"admin"` } } @@ -714,24 +714,24 @@ func (w *WorkerStruct) Paths(ctx context.Context) ([]stores.StoragePath, error) return w.Internal.Paths(ctx) } -func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) { - return w.Internal.SealPreCommit1(ctx, sectorNum, ticket, pieces) +func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) { + return w.Internal.SealPreCommit1(ctx, sector, ticket, pieces) } -func (w *WorkerStruct) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, p1o storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { - return w.Internal.SealPreCommit2(ctx, sectorNum, p1o) +func (w *WorkerStruct) SealPreCommit2(ctx context.Context, sector abi.SectorID, p1o storage.PreCommit1Out) (storage.SectorCids, error) { + return w.Internal.SealPreCommit2(ctx, sector, p1o) } -func (w *WorkerStruct) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (storage.Commit1Out, error) { - return w.Internal.SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID) +func (w *WorkerStruct) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) { + return w.Internal.SealCommit1(ctx, sector, ticket, seed, pieces, cids) } -func (w *WorkerStruct) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, c1o storage.Commit1Out) (storage.Proof, error) { - return w.Internal.SealCommit2(ctx, sectorNum, c1o) +func (w *WorkerStruct) SealCommit2(ctx context.Context, sector abi.SectorID, c1o storage.Commit1Out) (storage.Proof, error) { + return w.Internal.SealCommit2(ctx, sector, c1o) } -func (w *WorkerStruct) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { - return w.Internal.FinalizeSector(ctx, sectorNum) +func (w *WorkerStruct) FinalizeSector(ctx context.Context, sector abi.SectorID) error { + return w.Internal.FinalizeSector(ctx, sector) } var _ api.Common = &CommonStruct{} diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index f18a9433f..89e310eb5 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -42,8 +42,12 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum return nil, nil, err } + mid, err := address.IDFromAddress(maddr) + if err != nil { + return nil, nil, err + } + cfg := §orbuilder.Config{ - Miner: maddr, SealProofType: spt, PoStProofType: ppt, } @@ -55,7 +59,6 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum next := offset sbfs := &fs.Basic{ - Miner: maddr, Root: sbroot, } @@ -71,7 +74,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum var sealedSectors []*genesis.PreSeal for i := 0; i < sectors; i++ { - sid := next + sid := abi.SectorID{Miner: abi.ActorID(mid), Number: next} next++ pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(ssize).Unpadded(), rand.Reader) @@ -89,7 +92,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum return nil, nil, xerrors.Errorf("commit: %w", err) } - scid, ucid, err := sb.SealPreCommit2(context.TODO(), sid, in2) + cids, err := sb.SealPreCommit2(context.TODO(), sid, in2) if err != nil { return nil, nil, xerrors.Errorf("commit: %w", err) } @@ -98,11 +101,11 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum return nil, nil, xerrors.Errorf("trim cache: %w", err) } - log.Warn("PreCommitOutput: ", sid, scid, ucid) + log.Warn("PreCommitOutput: ", sid, cids.Sealed, cids.Unsealed) sealedSectors = append(sealedSectors, &genesis.PreSeal{ - CommR: scid, - CommD: ucid, - SectorID: sid, + CommR: cids.Sealed, + CommD: cids.Unsealed, + SectorID: sid.Number, ProofType: pt, }) } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 2c91ebbab..e15a8c79e 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -7,24 +7,12 @@ import ( "encoding/binary" "encoding/json" "fmt" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" "io/ioutil" "os" "path/filepath" "strconv" - "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/market" - miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/specs-actors/actors/builtin/power" - crypto2 "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/google/uuid" - - "github.com/filecoin-project/go-address" - cborutil "github.com/filecoin-project/go-cbor-util" - paramfetch "github.com/filecoin-project/go-paramfetch" - "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" @@ -32,6 +20,17 @@ import ( "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" + "github.com/filecoin-project/go-address" + cborutil "github.com/filecoin-project/go-cbor-util" + paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + miner2 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/filecoin-project/specs-actors/actors/builtin/power" + crypto2 "github.com/filecoin-project/specs-actors/actors/crypto" + lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" @@ -45,6 +44,7 @@ import ( "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sealing" "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" ) var initCmd = &cli.Command{ @@ -392,15 +392,19 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return err } + mid, err := address.IDFromAddress(a) + if err != nil { + return xerrors.Errorf("getting id address: %w", err) + } + smgr, err := advmgr.New(lr, stores.NewIndex(), §orbuilder.Config{ SealProofType: spt, PoStProofType: ppt, - Miner: a, - }, nil, nil) + }, nil) if err != nil { return err } - epp := storage.NewElectionPoStProver(smgr) + epp := storage.NewElectionPoStProver(smgr, abi.ActorID(mid)) m := miner.NewMiner(api, epp) { diff --git a/go.mod b/go.mod index e860fa423..f7d665963 100644 --- a/go.mod +++ b/go.mod @@ -22,11 +22,11 @@ require ( github.com/filecoin-project/go-fil-markets v0.0.0-20200304003055-d449a980d4bd github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 - github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200314022627-38af9db49ba2 + github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317165603-bd9e7cb04d81 github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/specs-actors v0.0.0-20200311215506-e95895452888 - github.com/filecoin-project/specs-storage v0.0.0-20200303233430-1a5a408f7513 + github.com/filecoin-project/specs-storage v0.0.0-20200317133846-063ba163b217 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 github.com/go-ole/go-ole v1.2.4 // indirect github.com/google/uuid v1.1.1 diff --git a/go.sum b/go.sum index b24ce8be3..dbd585bd2 100644 --- a/go.sum +++ b/go.sum @@ -126,6 +126,8 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/g github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200226210935-4739f8749f56/go.mod h1:tzTc9BxxSbjlIzhFwm5h9oBkXKkRuLxeiWspntwnKyw= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200314022627-38af9db49ba2 h1:4RjDynwobd/UYlZUprRg/GMEsMP6fAfVRTXgFs4XNfo= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200314022627-38af9db49ba2/go.mod h1:NcE+iL0bbYnamGmYQgCPVGbSaf8VF2/CLra/61B3I3I= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317165603-bd9e7cb04d81 h1:W5yekTpVTUiB86rSDiZo6rTI3lrLKrsrdY0tx/IqgJA= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317165603-bd9e7cb04d81/go.mod h1:3c3MEU9GHLlau37+MmefFNunTo9sVEKfjaJuHBgksdY= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= @@ -138,6 +140,8 @@ github.com/filecoin-project/specs-actors v0.0.0-20200311215506-e95895452888 h1:V github.com/filecoin-project/specs-actors v0.0.0-20200311215506-e95895452888/go.mod h1:5WngRgTN5Eo4+0SjCBqLzEr2l6Mj45DrP2606gBhqI0= github.com/filecoin-project/specs-storage v0.0.0-20200303233430-1a5a408f7513 h1:okBx3lPomwDxlPmRvyP078BwivDfdxNUlpCDhDD0ia8= github.com/filecoin-project/specs-storage v0.0.0-20200303233430-1a5a408f7513/go.mod h1:sC2Ck2l1G8hXI5Do/3sp0yxbMRMnukbFwP9KF1CRFLw= +github.com/filecoin-project/specs-storage v0.0.0-20200317133846-063ba163b217 h1:doPA79fSLg5TnY2rJhXs5dIZHP3IoCcIiCLKFGfgrY8= +github.com/filecoin-project/specs-storage v0.0.0-20200317133846-063ba163b217/go.mod h1:dUmzHS7izOD6HW3/JpzFrjxnptxbsHXBlO8puK2UzBk= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0= diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index 487fe2466..d5e60f056 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -38,7 +38,17 @@ func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID uin if err != nil { return nil, err } - return rpn.sealer.ReadPieceFromSealedSector(ctx, abi.SectorNumber(sectorID), sectorbuilder.UnpaddedByteIndex(offset), abi.UnpaddedPieceSize(length), si.Ticket.Value, *si.CommD) + + mid, err := address.IDFromAddress(rpn.miner.Address()) + if err != nil { + panic(err) + } + + sid := abi.SectorID{ + Miner: abi.ActorID(mid), + Number: abi.SectorNumber(sectorID), + } + return rpn.sealer.ReadPieceFromSealedSector(ctx, sid, sectorbuilder.UnpaddedByteIndex(offset), abi.UnpaddedPieceSize(length), si.Ticket.Value, *si.CommD) } func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *paych.SignedVoucher, proof []byte, expectedAmount abi.TokenAmount) (abi.TokenAmount, error) { diff --git a/node/builder.go b/node/builder.go index 1c7b5838b..1941ef2e3 100644 --- a/node/builder.go +++ b/node/builder.go @@ -262,7 +262,7 @@ func Online() Option { Override(new(*stores.Index), stores.NewIndex()), Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig), Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), - Override(new(advmgr.SectorIDCounter), modules.SectorIDCounter), + Override(new(sealing.SectorIDCounter), modules.SectorIDCounter), Override(new(*advmgr.Manager), advmgr.New), Override(new(sealmgr.Manager), From(new(*advmgr.Manager))), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index ac4e3a282..6ec6d9acb 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -59,7 +59,7 @@ func (sm *StorageMinerAPI) WorkerStats(context.Context) (sectorbuilder.WorkerSta }*/ func (sm *StorageMinerAPI) ActorAddress(context.Context) (address.Address, error) { - return sm.SectorBuilderConfig.Miner, nil + return sm.Miner.Address(), nil } func (sm *StorageMinerAPI) ActorSectorSize(ctx context.Context, addr address.Address) (abi.SectorSize, error) { diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index c127db179..fa46c9336 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -49,7 +49,6 @@ import ( "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sealing" "github.com/filecoin-project/lotus/storage/sealmgr" - "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { @@ -91,7 +90,6 @@ func SectorBuilderConfig(ds dtypes.MetadataDS, fnapi lapi.FullNode) (*sectorbuil } sb := §orbuilder.Config{ - Miner: minerAddr, SealProofType: spt, PoStProofType: ppt, } @@ -108,7 +106,7 @@ func (s *sidsc) Next() (abi.SectorNumber, error) { return abi.SectorNumber(i), err } -func SectorIDCounter(ds dtypes.MetadataDS) advmgr.SectorIDCounter { +func SectorIDCounter(ds dtypes.MetadataDS) sealing.SectorIDCounter { sc := storedcounter.New(ds, datastore.NewKey("/storage/nextid")) return &sidsc{sc} } diff --git a/storage/fpost_run.go b/storage/fpost_run.go index dc38d7f4b..b745a3cb1 100644 --- a/storage/fpost_run.go +++ b/storage/fpost_run.go @@ -3,6 +3,7 @@ package storage import ( "bytes" "context" + "github.com/filecoin-project/go-address" "time" "github.com/filecoin-project/specs-actors/actors/crypto" @@ -187,19 +188,24 @@ func (s *FPoStScheduler) runPost(ctx context.Context, eps abi.ChainEpoch, ts *ty "sectors", len(ssi), "faults", len(faults)) - scandidates, proof, err := s.sb.GenerateFallbackPoSt(ssi, abi.PoStRandomness(rand), faults) + mid, err := address.IDFromAddress(s.actor) + if err != nil { + return nil, err + } + + postOut, err := s.sb.GenerateFallbackPoSt(ctx, abi.ActorID(mid), ssi, abi.PoStRandomness(rand), faults) if err != nil { return nil, xerrors.Errorf("running post failed: %w", err) } - if len(scandidates) == 0 { + if len(postOut.PoStInputs) == 0 { return nil, xerrors.Errorf("received zero candidates back from generate fallback post") } // TODO: until we figure out how fallback post is really supposed to work, // let's just pass a single candidate... - scandidates = scandidates[:1] - proof = proof[:1] + scandidates := postOut.PoStInputs[:1] + proof := postOut.Proof[:1] elapsed := time.Since(tsStart) log.Infow("submitting PoSt", "pLen", len(proof), "elapsed", elapsed) diff --git a/storage/miner.go b/storage/miner.go index 1b535d786..ce6bbdc14 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -121,10 +121,11 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error { type SectorBuilderEpp struct { prover storage.Prover + miner abi.ActorID } -func NewElectionPoStProver(sb storage.Prover) *SectorBuilderEpp { - return &SectorBuilderEpp{sb} +func NewElectionPoStProver(sb storage.Prover, miner abi.ActorID) *SectorBuilderEpp { + return &SectorBuilderEpp{sb, miner} } var _ gen.ElectionPoStProver = (*SectorBuilderEpp)(nil) @@ -133,7 +134,7 @@ func (epp *SectorBuilderEpp) GenerateCandidates(ctx context.Context, ssi []abi.S start := time.Now() var faults []abi.SectorNumber // TODO - cds, err := epp.prover.GenerateEPostCandidates(ssi, rand, faults) + cds, err := epp.prover.GenerateEPostCandidates(ctx, epp.miner, ssi, rand, faults) if err != nil { return nil, xerrors.Errorf("failed to generate candidates: %w", err) } @@ -153,7 +154,7 @@ func (epp *SectorBuilderEpp) ComputeProof(ctx context.Context, ssi []abi.SectorI } start := time.Now() - proof, err := epp.prover.ComputeElectionPoSt(ssi, rand, owins) + proof, err := epp.prover.ComputeElectionPoSt(ctx, epp.miner, ssi, rand, owins) if err != nil { return nil, err } diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go index a7e7adf51..f93029022 100644 --- a/storage/sbmock/sbmock.go +++ b/storage/sbmock/sbmock.go @@ -24,7 +24,7 @@ import ( var log = logging.Logger("sbmock") type SBMock struct { - sectors map[abi.SectorNumber]*sectorState + sectors map[abi.SectorID]*sectorState sectorSize abi.SectorSize nextSectorID abi.SectorNumber rateLimit chan struct{} @@ -42,7 +42,7 @@ func NewMockSectorBuilder(threads int, ssize abi.SectorSize) *SBMock { } return &SBMock{ - sectors: make(map[abi.SectorNumber]*sectorState), + sectors: make(map[abi.SectorID]*sectorState), sectorSize: ssize, nextSectorID: 5, rateLimit: make(chan struct{}, threads), @@ -74,7 +74,11 @@ func (sb *SBMock) RateLimit() func() { } } -func (sb *SBMock) AddPiece(ctx context.Context, sectorId abi.SectorNumber, existingPieces []abi.UnpaddedPieceSize, size abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { +func (sb *SBMock) NewSector(ctx context.Context, sector abi.SectorID) error { + return nil +} + +func (sb *SBMock) AddPiece(ctx context.Context, sectorId abi.SectorID, existingPieces []abi.UnpaddedPieceSize, size abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { log.Warn("Add piece: ", sectorId, size, sb.proofType) sb.lk.Lock() ss, ok := sb.sectors[sectorId] @@ -114,11 +118,7 @@ func (sb *SBMock) AcquireSectorNumber() (abi.SectorNumber, error) { return id, nil } -func (sb *SBMock) GenerateFallbackPoSt([]abi.SectorInfo, abi.PoStRandomness, []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, []abi.PoStProof, error) { - panic("NYI") -} - -func (sb *SBMock) SealPreCommit1(ctx context.Context, sid abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { +func (sb *SBMock) SealPreCommit1(ctx context.Context, sid abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { sb.lk.Lock() ss, ok := sb.sectors[sid] sb.lk.Unlock() @@ -173,7 +173,7 @@ func (sb *SBMock) SealPreCommit1(ctx context.Context, sid abi.SectorNumber, tick return cc, nil } -func (sb *SBMock) SealPreCommit2(ctx context.Context, sid abi.SectorNumber, phase1Out storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { +func (sb *SBMock) SealPreCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.PreCommit1Out) (cids storage.SectorCids, err error) { db := []byte(string(phase1Out)) db[0] ^= 'd' @@ -186,10 +186,13 @@ func (sb *SBMock) SealPreCommit2(ctx context.Context, sid abi.SectorNumber, phas commR := commcid.DataCommitmentV1ToCID(commr) - return commR, d, nil + return storage.SectorCids{ + Unsealed: d, + Sealed: commR, + }, nil } -func (sb *SBMock) SealCommit1(ctx context.Context, sid abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCid cid.Cid, unsealed cid.Cid) (output storage.Commit1Out, err error) { +func (sb *SBMock) SealCommit1(ctx context.Context, sid abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (output storage.Commit1Out, err error) { sb.lk.Lock() ss, ok := sb.sectors[sid] sb.lk.Unlock() @@ -211,16 +214,16 @@ func (sb *SBMock) SealCommit1(ctx context.Context, sid abi.SectorNumber, ticket var out [32]byte for i := range out { - out[i] = unsealed.Bytes()[i] + sealedCid.Bytes()[31-i] - ticket[i]*seed[i] ^ byte(sid&0xff) + out[i] = cids.Unsealed.Bytes()[i] + cids.Sealed.Bytes()[31-i] - ticket[i]*seed[i] ^ byte(sid.Number&0xff) } return out[:], nil } -func (sb *SBMock) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage.Commit1Out) (proof storage.Proof, err error) { +func (sb *SBMock) SealCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.Commit1Out) (proof storage.Proof, err error) { var out [32]byte for i := range out { - out[i] = phase1Out[i] ^ byte(sectorNum&0xff) + out[i] = phase1Out[i] ^ byte(sid.Number&0xff) } return out[:], nil @@ -228,7 +231,7 @@ func (sb *SBMock) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, p // Test Instrumentation Methods -func (sb *SBMock) FailSector(sid abi.SectorNumber) error { +func (sb *SBMock) FailSector(sid abi.SectorID) error { sb.lk.Lock() defer sb.lk.Unlock() ss, ok := sb.sectors[sid] @@ -256,11 +259,15 @@ func AddOpFinish(ctx context.Context) (context.Context, func()) { } } -func (sb *SBMock) ComputeElectionPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) { +func (sb *SBMock) GenerateFallbackPoSt(context.Context, abi.ActorID, []abi.SectorInfo, abi.PoStRandomness, []abi.SectorNumber) (storage.FallbackPostOut, error) { panic("implement me") } -func (sb *SBMock) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) { +func (sb *SBMock) ComputeElectionPoSt(ctx context.Context, mid abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) { + panic("implement me") +} + +func (sb *SBMock) GenerateEPostCandidates(ctx context.Context, mid abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) { if len(faults) > 0 { panic("todo") } @@ -280,7 +287,7 @@ func (sb *SBMock) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challenge Candidate: abi.PoStCandidate{ SectorID: abi.SectorID{ Number: abi.SectorNumber((int(start) + i) % len(sectorInfo)), - Miner: 1125125, //TODO + Miner: mid, }, PartialTicket: abi.PartialTicket(challengeSeed), }, @@ -290,14 +297,14 @@ func (sb *SBMock) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challenge return out, nil } -func (sb *SBMock) ReadPieceFromSealedSector(ctx context.Context, sectorID abi.SectorNumber, offset sectorbuilder.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, commD cid.Cid) (io.ReadCloser, error) { +func (sb *SBMock) ReadPieceFromSealedSector(ctx context.Context, sectorID abi.SectorID, offset sectorbuilder.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, commD cid.Cid) (io.ReadCloser, error) { if len(sb.sectors[sectorID].pieces) > 1 { panic("implme") } return ioutil.NopCloser(io.LimitReader(bytes.NewReader(sb.sectors[sectorID].pieces[0].Bytes()[offset:]), int64(size))), nil } -func (sb *SBMock) StageFakeData() (abi.SectorNumber, []abi.PieceInfo, error) { +func (sb *SBMock) StageFakeData(mid abi.ActorID) (abi.SectorNumber, []abi.PieceInfo, error) { usize := abi.PaddedPieceSize(sb.sectorSize).Unpadded() sid, err := sb.AcquireSectorNumber() if err != nil { @@ -307,7 +314,10 @@ func (sb *SBMock) StageFakeData() (abi.SectorNumber, []abi.PieceInfo, error) { buf := make([]byte, usize) rand.Read(buf) - pi, err := sb.AddPiece(context.TODO(), sid, nil, usize, bytes.NewReader(buf)) + pi, err := sb.AddPiece(context.TODO(), abi.SectorID{ + Miner: mid, + Number: sid, + }, nil, usize, bytes.NewReader(buf)) if err != nil { return 0, nil, err } @@ -315,7 +325,7 @@ func (sb *SBMock) StageFakeData() (abi.SectorNumber, []abi.PieceInfo, error) { return sid, []abi.PieceInfo{pi}, nil } -func (sb *SBMock) FinalizeSector(context.Context, abi.SectorNumber) error { +func (sb *SBMock) FinalizeSector(context.Context, abi.SectorID) error { return nil } diff --git a/storage/sealing.go b/storage/sealing.go index cb7cfb5a3..f2db4d166 100644 --- a/storage/sealing.go +++ b/storage/sealing.go @@ -2,6 +2,7 @@ package storage import ( "context" + "github.com/filecoin-project/go-address" "io" "github.com/filecoin-project/specs-actors/actors/abi" @@ -12,6 +13,10 @@ import ( // TODO: refactor this to be direct somehow +func (m *Miner) Address() address.Address { + return m.sealing.Address() +} + func (m *Miner) AllocatePiece(size abi.UnpaddedPieceSize) (sectorID abi.SectorNumber, offset uint64, err error) { return m.sealing.AllocatePiece(size) } diff --git a/storage/sealing/garbage.go b/storage/sealing/garbage.go index 445ba8969..4903abf25 100644 --- a/storage/sealing/garbage.go +++ b/storage/sealing/garbage.go @@ -16,7 +16,7 @@ func (m *Sealing) pledgeReader(size abi.UnpaddedPieceSize) io.Reader { return io.LimitReader(&nullreader.Reader{}, int64(size)) } -func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, existingPieceSizes []abi.UnpaddedPieceSize, sizes ...abi.UnpaddedPieceSize) ([]Piece, error) { +func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorID, existingPieceSizes []abi.UnpaddedPieceSize, sizes ...abi.UnpaddedPieceSize) ([]Piece, error) { if len(sizes) == 0 { return nil, nil } @@ -55,13 +55,18 @@ func (m *Sealing) PledgeSector() error { return } - sid, err := m.sealer.NewSector() + sid, err := m.sc.Next() + if err != nil { + log.Errorf("%+v", err) + return + } + err = m.sealer.NewSector(ctx, m.minerSector(sid)) if err != nil { log.Errorf("%+v", err) return } - pieces, err := m.pledgeSector(ctx, sid, []abi.UnpaddedPieceSize{}, size) + pieces, err := m.pledgeSector(ctx, m.minerSector(sid), []abi.UnpaddedPieceSize{}, size) if err != nil { log.Errorf("%+v", err) return diff --git a/storage/sealing/sealing.go b/storage/sealing/sealing.go index b74d34db1..ed793b0ae 100644 --- a/storage/sealing/sealing.go +++ b/storage/sealing/sealing.go @@ -30,6 +30,10 @@ var log = logging.Logger("sectors") type TicketFn func(context.Context) (*api.SealTicket, error) +type SectorIDCounter interface { + Next() (abi.SectorNumber, error) +} + type sealingApi interface { // TODO: trim down // Call a read only method on actors (no interaction with the chain required) StateCall(context.Context, *types.Message, types.TipSetKey) (*api.InvocResult, error) @@ -68,6 +72,7 @@ type Sealing struct { sealer sealmgr.Manager sectors *statemachine.StateGroup tktFn TicketFn + sc SectorIDCounter } func New(api sealingApi, events *events.Events, maddr address.Address, worker address.Address, ds datastore.Batching, sealer sealmgr.Manager, tktFn TicketFn) *Sealing { @@ -104,9 +109,14 @@ func (m *Sealing) AllocatePiece(size abi.UnpaddedPieceSize) (sectorID abi.Sector return 0, 0, xerrors.Errorf("cannot allocate unpadded piece") } - sid, err := m.sealer.NewSector() // TODO: Put more than one thing in a sector + sid, err := m.sc.Next() if err != nil { - return 0, 0, xerrors.Errorf("acquiring sector ID: %w", err) + return 0, 0, xerrors.Errorf("getting sector number: %w", err) + } + + err = m.sealer.NewSector(context.TODO(), m.minerSector(sid)) // TODO: Put more than one thing in a sector + if err != nil { + return 0, 0, xerrors.Errorf("initializing sector: %w", err) } // offset hard-coded to 0 since we only put one thing in a sector for now @@ -116,7 +126,7 @@ func (m *Sealing) AllocatePiece(size abi.UnpaddedPieceSize) (sectorID abi.Sector func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, sectorID abi.SectorNumber, dealID abi.DealID) error { log.Infof("Seal piece for deal %d", dealID) - ppi, err := m.sealer.AddPiece(ctx, sectorID, []abi.UnpaddedPieceSize{}, size, r) + ppi, err := m.sealer.AddPiece(ctx, m.minerSector(sectorID), []abi.UnpaddedPieceSize{}, size, r) if err != nil { return xerrors.Errorf("adding piece to sector: %w", err) } @@ -144,3 +154,19 @@ func (m *Sealing) newSector(sid abi.SectorNumber, rt abi.RegisteredProof, pieces sectorType: rt, }) } + +func (m *Sealing) minerSector(num abi.SectorNumber) abi.SectorID { + mid, err := address.IDFromAddress(m.maddr) + if err != nil { + panic(err) + } + + return abi.SectorID{ + Number: num, + Miner: abi.ActorID(mid), + } +} + +func (m *Sealing) Address() address.Address { + return m.maddr +} diff --git a/storage/sealing/states.go b/storage/sealing/states.go index e12860447..ca02e90f4 100644 --- a/storage/sealing/states.go +++ b/storage/sealing/states.go @@ -2,6 +2,7 @@ package sealing import ( "context" + "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/specs-actors/actors/crypto" @@ -40,7 +41,7 @@ func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) err log.Warnf("Creating %d filler pieces for sector %d", len(fillerSizes), sector.SectorID) } - pieces, err := m.pledgeSector(ctx.Context(), sector.SectorID, sector.existingPieces(), fillerSizes...) + pieces, err := m.pledgeSector(ctx.Context(), m.minerSector(sector.SectorID), sector.existingPieces(), fillerSizes...) if err != nil { return xerrors.Errorf("filling up the sector (%v): %w", fillerSizes, err) } @@ -69,19 +70,19 @@ func (m *Sealing) handleUnsealed(ctx statemachine.Context, sector SectorInfo) er return ctx.Send(SectorSealFailed{xerrors.Errorf("getting ticket failed: %w", err)}) } - pc1o, err := m.sealer.SealPreCommit1(ctx.Context(), sector.SectorID, ticket.Value, sector.pieceInfos()) + pc1o, err := m.sealer.SealPreCommit1(ctx.Context(), m.minerSector(sector.SectorID), ticket.Value, sector.pieceInfos()) if err != nil { return ctx.Send(SectorSealFailed{xerrors.Errorf("seal pre commit failed: %w", err)}) } - sealed, unsealed, err := m.sealer.SealPreCommit2(ctx.Context(), sector.SectorID, pc1o) + cids, err := m.sealer.SealPreCommit2(ctx.Context(), m.minerSector(sector.SectorID), pc1o) if err != nil { return ctx.Send(SectorSealFailed{xerrors.Errorf("seal pre commit failed: %w", err)}) } return ctx.Send(SectorSealed{ - commD: unsealed, - commR: sealed, + commD: cids.Unsealed, + commR: cids.Sealed, ticket: *ticket, }) } @@ -184,12 +185,16 @@ func (m *Sealing) handleCommitting(ctx statemachine.Context, sector SectorInfo) log.Infof("KOMIT %d %x(%d); %x(%d); %v; r:%x; d:%x", sector.SectorID, sector.Ticket.Value, sector.Ticket.Epoch, sector.Seed.Value, sector.Seed.Epoch, sector.pieceInfos(), sector.CommR, sector.CommD) - c2in, err := m.sealer.SealCommit1(ctx.Context(), sector.SectorID, sector.Ticket.Value, sector.Seed.Value, sector.pieceInfos(), *sector.CommR, *sector.CommD) + cids := storage.SectorCids{ + Unsealed: *sector.CommD, + Sealed: *sector.CommR, + } + c2in, err := m.sealer.SealCommit1(ctx.Context(), m.minerSector(sector.SectorID), sector.Ticket.Value, sector.Seed.Value, sector.pieceInfos(), cids) if err != nil { return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed: %w", err)}) } - proof, err := m.sealer.SealCommit2(ctx.Context(), sector.SectorID, c2in) + proof, err := m.sealer.SealCommit2(ctx.Context(), m.minerSector(sector.SectorID), c2in) if err != nil { return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed: %w", err)}) } @@ -250,7 +255,7 @@ func (m *Sealing) handleCommitWait(ctx statemachine.Context, sector SectorInfo) func (m *Sealing) handleFinalizeSector(ctx statemachine.Context, sector SectorInfo) error { // TODO: Maybe wait for some finality - if err := m.sealer.FinalizeSector(ctx.Context(), sector.SectorID); err != nil { + if err := m.sealer.FinalizeSector(ctx.Context(), m.minerSector(sector.SectorID)); err != nil { return ctx.Send(SectorFinalizeFailed{xerrors.Errorf("finalize sector: %w", err)}) } diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go index 7d6b5f779..38dcf38f7 100644 --- a/storage/sealmgr/advmgr/local.go +++ b/storage/sealmgr/advmgr/local.go @@ -4,9 +4,6 @@ import ( "context" "io" - "github.com/ipfs/go-cid" - "golang.org/x/xerrors" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" @@ -35,7 +32,6 @@ func NewLocalWorker(ma address.Address, spt abi.RegisteredProof, store stores.St scfg: §orbuilder.Config{ SealProofType: spt, PoStProofType: ppt, - Miner: ma, }, storage: store, localStore: local, @@ -47,17 +43,7 @@ type localWorkerPathProvider struct { w *LocalWorker } -func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { - mid, err := address.IDFromAddress(l.w.scfg.Miner) - if err != nil { - return sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("get miner ID: %w", err) - } - - sector := abi.SectorID{ - Miner: abi.ActorID(mid), - Number: id, - } - +func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, sector abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { paths, storageIDs, done, err := l.w.storage.AcquireSector(ctx, sector, existing, allocate, sealing) if err != nil { return sectorbuilder.SectorPaths{}, nil, err @@ -84,58 +70,68 @@ func (l *LocalWorker) sb() (sectorbuilder.Basic, error) { return sectorbuilder.New(&localWorkerPathProvider{w: l}, l.scfg) } -func (l *LocalWorker) AddPiece(ctx context.Context, sn abi.SectorNumber, epcs []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { - sb, err := l.sb() - if err != nil { - return abi.PieceInfo{}, err - } - return sb.AddPiece(ctx, sn, epcs, sz, r) -} - -func (l *LocalWorker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) { - sb, err := l.sb() - if err != nil { - return nil, err - } - - return sb.SealPreCommit1(ctx, sectorNum, ticket, pieces) -} - -func (l *LocalWorker) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { - sb, err := l.sb() - if err != nil { - return cid.Undef, cid.Undef, err - } - - return sb.SealPreCommit2(ctx, sectorNum, phase1Out) -} - -func (l *LocalWorker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output storage2.Commit1Out, err error) { - sb, err := l.sb() - if err != nil { - return nil, err - } - - return sb.SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID) -} - -func (l *LocalWorker) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) { - sb, err := l.sb() - if err != nil { - return nil, err - } - - return sb.SealCommit2(ctx, sectorNum, phase1Out) -} - -func (l *LocalWorker) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { +func (l *LocalWorker) NewSector(ctx context.Context, sector abi.SectorID) error { sb, err := l.sb() if err != nil { return err } - return sb.FinalizeSector(ctx, sectorNum) + return sb.NewSector(ctx, sector) +} + +func (l *LocalWorker) AddPiece(ctx context.Context, sector abi.SectorID, epcs []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { + sb, err := l.sb() + if err != nil { + return abi.PieceInfo{}, err + } + + return sb.AddPiece(ctx, sector, epcs, sz, r) +} + +func (l *LocalWorker) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) { + sb, err := l.sb() + if err != nil { + return nil, err + } + + return sb.SealPreCommit1(ctx, sector, ticket, pieces) +} + +func (l *LocalWorker) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.PreCommit1Out) (cids storage2.SectorCids, err error) { + sb, err := l.sb() + if err != nil { + return storage2.SectorCids{}, err + } + + return sb.SealPreCommit2(ctx, sector, phase1Out) +} + +func (l *LocalWorker) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage2.SectorCids) (output storage2.Commit1Out, err error) { + sb, err := l.sb() + if err != nil { + return nil, err + } + + return sb.SealCommit1(ctx, sector, ticket, seed, pieces, cids) +} + +func (l *LocalWorker) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) { + sb, err := l.sb() + if err != nil { + return nil, err + } + + return sb.SealCommit2(ctx, sector, phase1Out) +} + +func (l *LocalWorker) FinalizeSector(ctx context.Context, sector abi.SectorID) error { + sb, err := l.sb() + if err != nil { + return err + } + + return sb.FinalizeSector(ctx, sector) } func (l *LocalWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) { diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index b2eee7d61..52e5463b2 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -10,8 +10,6 @@ import ( "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/storage/sealmgr/stores" @@ -26,10 +24,6 @@ var log = logging.Logger("advmgr") type URLs []string -type SectorIDCounter interface { - Next() (abi.SectorNumber, error) -} - type Worker interface { sectorbuilder.Sealer @@ -40,7 +34,6 @@ type Worker interface { type Manager struct { workers []Worker scfg *sectorbuilder.Config - sc SectorIDCounter ls stores.LocalStorage storage *stores.Local @@ -49,7 +42,7 @@ type Manager struct { storage2.Prover } -func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, sc SectorIDCounter, urls URLs) (*Manager, error) { +func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, urls URLs) (*Manager, error) { stor, err := stores.NewLocal(ls) if err != nil { return nil, err @@ -59,12 +52,7 @@ func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, sc log.Errorf("Declaring local storage failed: %+v") } - mid, err := address.IDFromAddress(cfg.Miner) - if err != nil { - return nil, xerrors.Errorf("getting miner id: %w", err) - } - - prover, err := sectorbuilder.New(&readonlyProvider{stor: stor, miner: abi.ActorID(mid)}, cfg) + prover, err := sectorbuilder.New(&readonlyProvider{stor: stor}, cfg) if err != nil { return nil, xerrors.Errorf("creating prover instance: %w", err) } @@ -74,7 +62,6 @@ func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, sc &LocalWorker{scfg: cfg, storage: stor}, }, scfg: cfg, - sc: sc, ls: ls, storage: stor, @@ -113,11 +100,7 @@ func (m *Manager) SectorSize() abi.SectorSize { return sz } -func (m *Manager) NewSector() (abi.SectorNumber, error) { - return m.sc.Next() -} - -func (m *Manager) ReadPieceFromSealedSector(context.Context, abi.SectorNumber, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) { +func (m *Manager) ReadPieceFromSealedSector(context.Context, abi.SectorID, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) { panic("implement me") } @@ -166,7 +149,12 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.Stor return workers, paths } -func (m *Manager) AddPiece(ctx context.Context, sn abi.SectorNumber, existingPieces []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { +func (m *Manager) NewSector(ctx context.Context, sector abi.SectorID) error { + log.Warnf("stub NewSector") + return nil +} + +func (m *Manager) AddPiece(ctx context.Context, sector abi.SectorID, existingPieces []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { // TODO: consider multiple paths vs workers when initially allocating var best []stores.StorageMeta @@ -174,7 +162,7 @@ func (m *Manager) AddPiece(ctx context.Context, sn abi.SectorNumber, existingPie if len(existingPieces) == 0 { // new best, err = m.storage.FindBestAllocStorage(sectorbuilder.FTUnsealed, true) } else { // append to existing - best, err = m.storage.FindSector(m.minerID(), sn, sectorbuilder.FTUnsealed) + best, err = m.storage.FindSector(sector, sectorbuilder.FTUnsealed) } if err != nil { return abi.PieceInfo{}, xerrors.Errorf("finding sector path: %w", err) @@ -188,10 +176,10 @@ func (m *Manager) AddPiece(ctx context.Context, sn abi.SectorNumber, existingPie // TODO: select(candidateWorkers, ...) // TODO: remove the sectorbuilder abstraction, pass path directly - return candidateWorkers[0].AddPiece(ctx, sn, existingPieces, sz, r) + return candidateWorkers[0].AddPiece(ctx, sector, existingPieces, sz, r) } -func (m *Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) { +func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) { // TODO: also consider where the unsealed data sits best, err := m.storage.FindBestAllocStorage(sectorbuilder.FTCache|sectorbuilder.FTSealed, true) @@ -203,26 +191,26 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber // TODO: select(candidateWorkers, ...) // TODO: remove the sectorbuilder abstraction, pass path directly - return candidateWorkers[0].SealPreCommit1(ctx, sectorNum, ticket, pieces) + return candidateWorkers[0].SealPreCommit1(ctx, sector, ticket, pieces) } -func (m *Manager) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { +func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.PreCommit1Out) (cids storage2.SectorCids, err error) { // TODO: allow workers to fetch the sectors - best, err := m.storage.FindSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed) + best, err := m.storage.FindSector(sector, sectorbuilder.FTCache|sectorbuilder.FTSealed) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("finding path for sector sealing: %w", err) + return storage2.SectorCids{}, xerrors.Errorf("finding path for sector sealing: %w", err) } candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit2, best) // TODO: select(candidateWorkers, ...) // TODO: remove the sectorbuilder abstraction, pass path directly - return candidateWorkers[0].SealPreCommit2(ctx, sectorNum, phase1Out) + return candidateWorkers[0].SealPreCommit2(ctx, sector, phase1Out) } -func (m *Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output storage2.Commit1Out, err error) { - best, err := m.storage.FindSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed) +func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage2.SectorCids) (output storage2.Commit1Out, err error) { + best, err := m.storage.FindSector(sector, sectorbuilder.FTCache|sectorbuilder.FTSealed) if err != nil { return nil, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -231,10 +219,10 @@ func (m *Manager) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, t // TODO: select(candidateWorkers, ...) // TODO: remove the sectorbuilder abstraction, pass path directly - return candidateWorkers[0].SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID) + return candidateWorkers[0].SealCommit1(ctx, sector, ticket, seed, pieces, cids) } -func (m *Manager) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) { +func (m *Manager) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) { for _, worker := range m.workers { tt, err := worker.TaskTypes(context.TODO()) if err != nil { @@ -245,14 +233,14 @@ func (m *Manager) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, p continue } - return worker.SealCommit2(ctx, sectorNum, phase1Out) + return worker.SealCommit2(ctx, sector, phase1Out) } return nil, xerrors.New("no worker found") } -func (m *Manager) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { - best, err := m.storage.FindSector(m.minerID(), sectorNum, sectorbuilder.FTCache|sectorbuilder.FTSealed|sectorbuilder.FTUnsealed) +func (m *Manager) FinalizeSector(ctx context.Context, sector abi.SectorID) error { + best, err := m.storage.FindSector(sector, sectorbuilder.FTCache|sectorbuilder.FTSealed|sectorbuilder.FTUnsealed) if err != nil { return xerrors.Errorf("finding sealed sector: %w", err) } @@ -260,15 +248,7 @@ func (m *Manager) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit2, best) // find last worker with the sector // TODO: Move the sector to long-term storage - return candidateWorkers[0].FinalizeSector(ctx, sectorNum) -} - -func (m *Manager) minerID() abi.ActorID { - mid, err := address.IDFromAddress(m.scfg.Miner) - if err != nil { - panic(err) - } - return abi.ActorID(mid) + return candidateWorkers[0].FinalizeSector(ctx, sector) } var _ sealmgr.Manager = &Manager{} diff --git a/storage/sealmgr/advmgr/remote.go b/storage/sealmgr/advmgr/remote.go index 066f6af1a..27699434e 100644 --- a/storage/sealmgr/advmgr/remote.go +++ b/storage/sealmgr/advmgr/remote.go @@ -16,7 +16,11 @@ type remote struct { api.WorkerApi } -func (r *remote) AddPiece(ctx context.Context, sector abi.SectorNumber, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage2.Data) (abi.PieceInfo, error) { +func (r *remote) NewSector(ctx context.Context, sector abi.SectorID) error { + return xerrors.New("unsupported") +} + +func (r *remote) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData storage2.Data) (abi.PieceInfo, error) { return abi.PieceInfo{}, xerrors.New("unsupported") } diff --git a/storage/sealmgr/advmgr/roprov.go b/storage/sealmgr/advmgr/roprov.go index ce40d2ad3..dd549041f 100644 --- a/storage/sealmgr/advmgr/roprov.go +++ b/storage/sealmgr/advmgr/roprov.go @@ -11,19 +11,15 @@ import ( ) type readonlyProvider struct { - miner abi.ActorID stor *stores.Local } -func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { +func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { if allocate != 0 { return sectorbuilder.SectorPaths{}, nil, xerrors.New("read-only storage") } - p, _, done, err := l.stor.AcquireSector(ctx, abi.SectorID{ - Miner: l.miner, - Number: id, - }, existing, allocate, sealing) + p, _, done, err := l.stor.AcquireSector(ctx, id, existing, allocate, sealing) return p, done, err } diff --git a/storage/sealmgr/simple.go b/storage/sealmgr/simple.go index 0bb4fb161..7cf8c98c9 100644 --- a/storage/sealmgr/simple.go +++ b/storage/sealmgr/simple.go @@ -5,14 +5,11 @@ import ( "io" "sync" - "github.com/ipfs/go-cid" - "golang.org/x/xerrors" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-markets/storedcounter" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-storage/storage" + "github.com/ipfs/go-cid" ) type LocalWorker struct { @@ -24,7 +21,6 @@ var _ Worker = &LocalWorker{} // Simple implements a very basic storage manager which has one local worker, // running one thing locally type Simple struct { - sc *storedcounter.StoredCounter maddr address.Address rateLimiter sync.Mutex @@ -39,82 +35,76 @@ func (s *Simple) SectorSize() abi.SectorSize { return s.worker.(sszgetter).SectorSize() } -func NewSimpleManager(sc *storedcounter.StoredCounter, maddr address.Address, sb sectorbuilder.Basic) (*Simple, error) { +func NewSimpleManager(maddr address.Address, sb sectorbuilder.Basic) (*Simple, error) { w := &LocalWorker{ sb, } return &Simple{ - sc: sc, maddr: maddr, worker: w, }, nil } -func (s *Simple) NewSector() (abi.SectorNumber, error) { - n, err := s.sc.Next() - if err != nil { - return 0, xerrors.Errorf("acquire sector number: %w", err) - } - - return abi.SectorNumber(n), nil +func (s *Simple) NewSector(ctx context.Context, id abi.SectorID) error { + return s.worker.NewSector(ctx, id) } -func (s *Simple) AddPiece(ctx context.Context, sectorNum abi.SectorNumber, existingPieces []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r storage.Data) (abi.PieceInfo, error) { +func (s *Simple) AddPiece(ctx context.Context, id abi.SectorID, existingPieces []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r storage.Data) (abi.PieceInfo, error) { s.rateLimiter.Lock() defer s.rateLimiter.Unlock() - return s.worker.AddPiece(ctx, sectorNum, existingPieces, sz, r) + return s.worker.AddPiece(ctx, id, existingPieces, sz, r) } -func (s *Simple) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { +func (s *Simple) SealPreCommit1(ctx context.Context, id abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { s.rateLimiter.Lock() defer s.rateLimiter.Unlock() - return s.worker.SealPreCommit1(ctx, sectorNum, ticket, pieces) + return s.worker.SealPreCommit1(ctx, id, ticket, pieces) } -func (s *Simple) SealPreCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { +func (s *Simple) SealPreCommit2(ctx context.Context, id abi.SectorID, phase1Out storage.PreCommit1Out) (cids storage.SectorCids, err error) { s.rateLimiter.Lock() defer s.rateLimiter.Unlock() - return s.worker.SealPreCommit2(ctx, sectorNum, phase1Out) + return s.worker.SealPreCommit2(ctx, id, phase1Out) } -func (s *Simple) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (output storage.Commit1Out, err error) { +func (s *Simple) SealCommit1(ctx context.Context, id abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (output storage.Commit1Out, err error) { s.rateLimiter.Lock() defer s.rateLimiter.Unlock() - return s.worker.SealCommit1(ctx, sectorNum, ticket, seed, pieces, sealedCID, unsealedCID) + return s.worker.SealCommit1(ctx, id, ticket, seed, pieces, cids) } -func (s *Simple) SealCommit2(ctx context.Context, sectorNum abi.SectorNumber, phase1Out storage.Commit1Out) (proof storage.Proof, err error) { +func (s *Simple) SealCommit2(ctx context.Context, id abi.SectorID, phase1Out storage.Commit1Out) (proof storage.Proof, err error) { s.rateLimiter.Lock() defer s.rateLimiter.Unlock() - return s.worker.SealCommit2(ctx, sectorNum, phase1Out) + return s.worker.SealCommit2(ctx, id, phase1Out) } -func (s *Simple) FinalizeSector(ctx context.Context, sectorNum abi.SectorNumber) error { +func (s *Simple) FinalizeSector(ctx context.Context, id abi.SectorID) error { s.rateLimiter.Lock() defer s.rateLimiter.Unlock() - return s.worker.FinalizeSector(ctx, sectorNum) + return s.worker.FinalizeSector(ctx, id) } -func (s *Simple) GenerateEPostCandidates(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) { - return s.worker.GenerateEPostCandidates(sectorInfo, challengeSeed, faults) +func (s *Simple) GenerateEPostCandidates(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) { + return s.worker.GenerateEPostCandidates(ctx, miner, sectorInfo, challengeSeed, faults) } -func (s *Simple) GenerateFallbackPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, []abi.PoStProof, error) { - return s.worker.GenerateFallbackPoSt(sectorInfo, challengeSeed, faults) +func (s *Simple) GenerateFallbackPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) (storage.FallbackPostOut, error) { + return s.worker.GenerateFallbackPoSt(ctx, miner, sectorInfo, challengeSeed, faults) } -func (s *Simple) ComputeElectionPoSt(sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) { - return s.worker.ComputeElectionPoSt(sectorInfo, challengeSeed, winners) +func (s *Simple) ComputeElectionPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) { + return s.worker.ComputeElectionPoSt(ctx, miner, sectorInfo, challengeSeed, winners) } -func (s *Simple) ReadPieceFromSealedSector(context.Context, abi.SectorNumber, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) { +func (s *Simple) ReadPieceFromSealedSector(context.Context, abi.SectorID, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) { panic("todo") } diff --git a/storage/sealmgr/stores/local.go b/storage/sealmgr/stores/local.go index efac144a1..4b9a0ed71 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sealmgr/stores/local.go @@ -235,14 +235,11 @@ func (st *Local) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sea return out, nil } -func (st *Local) FindSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbuilder.SectorFileType) ([]StorageMeta, error) { +func (st *Local) FindSector(id abi.SectorID, typ sectorbuilder.SectorFileType) ([]StorageMeta, error) { var out []StorageMeta for _, p := range st.paths { p.lk.Lock() - t := p.sectors[abi.SectorID{ - Miner: mid, - Number: sn, - }] + t := p.sectors[id] if t|typ == 0 { continue } @@ -250,7 +247,7 @@ func (st *Local) FindSector(mid abi.ActorID, sn abi.SectorNumber, typ sectorbuil out = append(out, p.meta) } if len(out) == 0 { - return nil, xerrors.Errorf("sector %s/s-t0%d-%d not found", typ, mid, sn) + return nil, xerrors.Errorf("sector %s/s-t0%d-%d not found", typ, id.Miner, id.Number) } return out, nil diff --git a/storage/sealmgr/types.go b/storage/sealmgr/types.go index c7827f028..c9f11d38b 100644 --- a/storage/sealmgr/types.go +++ b/storage/sealmgr/types.go @@ -19,15 +19,11 @@ type Worker interface { type Manager interface { SectorSize() abi.SectorSize - // NewSector allocates staging area for data - // Storage manager forwards proof-related calls - NewSector() (abi.SectorNumber, error) - // TODO: Can[Pre]Commit[1,2] // TODO: Scrub() []Faults // TODO: Separate iface - ReadPieceFromSealedSector(context.Context, abi.SectorNumber, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) + ReadPieceFromSealedSector(context.Context, abi.SectorID, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) sectorbuilder.Sealer storage.Prover From 5f2caffb517f9951ca35565e58db08ffd154beb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 18 Mar 2020 02:08:11 +0100 Subject: [PATCH 13/45] Fix tests after specs-storage changes --- api/apistruct/struct.go | 8 +++--- cmd/lotus-bench/main.go | 40 ++++++++++++++++------------ cmd/lotus-seed/seed/seed.go | 2 +- cmd/lotus-storage-miner/init.go | 4 +-- go.mod | 2 +- go.sum | 2 ++ markets/retrievaladapter/provider.go | 2 +- node/builder.go | 3 +++ node/modules/dtypes/miner.go | 9 +++++++ node/modules/storageminer.go | 21 +++++++++------ node/node_test.go | 5 ++-- storage/miner.go | 13 +++++---- storage/sbmock/sbmock.go | 16 ++++++----- storage/sbmock/sbmock_test.go | 2 +- storage/sealing/sealing.go | 5 ++-- storage/sealmgr/advmgr/local.go | 1 - storage/sealmgr/advmgr/manager.go | 4 +-- storage/sealmgr/advmgr/roprov.go | 2 +- storage/sealmgr/simple.go | 4 +++ 19 files changed, 90 insertions(+), 55 deletions(-) create mode 100644 node/modules/dtypes/miner.go diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index f5f20b41c..6a048ee03 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -204,11 +204,11 @@ type WorkerStruct struct { TaskTypes func(context.Context) (map[sealmgr.TaskType]struct{}, error) `perm:"admin"` Paths func(context.Context) ([]stores.StoragePath, error) `perm:"admin"` - SealPreCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"` - SealPreCommit2 func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error) `perm:"admin"` + SealPreCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"` + SealPreCommit2 func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error) `perm:"admin"` SealCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) `perm:"admin"` - SealCommit2 func(context.Context, abi.SectorID, storage.Commit1Out) (storage.Proof, error) `perm:"admin"` - FinalizeSector func(context.Context, abi.SectorID) error `perm:"admin"` + SealCommit2 func(context.Context, abi.SectorID, storage.Commit1Out) (storage.Proof, error) `perm:"admin"` + FinalizeSector func(context.Context, abi.SectorID) error `perm:"admin"` } } diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 20ca2416b..54a8a7c36 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -168,7 +168,6 @@ func main() { } cfg := §orbuilder.Config{ - Miner: maddr, SealProofType: spt, PoStProofType: ppt, } @@ -186,8 +185,7 @@ func main() { } sbfs := &fs.Basic{ - Miner: maddr, - Root: sbdir, + Root: sbdir, } sb, err := sectorbuilder.New(sbfs, cfg) @@ -206,12 +204,17 @@ func main() { var sealedSectors []abi.SectorInfo numSectors := abi.SectorNumber(1) for i := abi.SectorNumber(1); i <= numSectors && robench == ""; i++ { + sid := abi.SectorID{ + Miner: mid, + Number: i, + } + start := time.Now() log.Info("Writing piece into sector...") r := rand.New(rand.NewSource(100 + int64(i))) - pi, err := sb.AddPiece(context.TODO(), i, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), r) + pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), r) if err != nil { return err } @@ -223,7 +226,7 @@ func main() { log.Info("Running replication(1)...") pieces := []abi.PieceInfo{pi} - pc1o, err := sb.SealPreCommit1(context.TODO(), i, ticket, pieces) + pc1o, err := sb.SealPreCommit1(context.TODO(), sid, ticket, pieces) if err != nil { return xerrors.Errorf("commit: %w", err) } @@ -231,7 +234,7 @@ func main() { precommit1 := time.Now() log.Info("Running replication(2)...") - commR, commD, err := sb.SealPreCommit2(context.TODO(), i, pc1o) + cids, err := sb.SealPreCommit2(context.TODO(), sid, pc1o) if err != nil { return xerrors.Errorf("commit: %w", err) } @@ -241,7 +244,7 @@ func main() { sealedSectors = append(sealedSectors, abi.SectorInfo{ RegisteredProof: spt, SectorNumber: i, - SealedCID: commR, + SealedCID: cids.Sealed, }) seed := lapi.SealSeed{ @@ -250,7 +253,7 @@ func main() { } log.Info("Generating PoRep for sector (1)") - c1o, err := sb.SealCommit1(context.TODO(), i, ticket, seed.Value, pieces, commR, commD) + c1o, err := sb.SealCommit1(context.TODO(), sid, ticket, seed.Value, pieces, cids) if err != nil { return err } @@ -278,7 +281,7 @@ func main() { var proof storage.Proof if !c.Bool("skip-commit2") { - proof, err = sb.SealCommit2(context.TODO(), i, c1o) + proof, err = sb.SealCommit2(context.TODO(), sid, c1o) if err != nil { return err } @@ -291,7 +294,7 @@ func main() { svi := abi.SealVerifyInfo{ SectorID: abi.SectorID{Miner: mid, Number: i}, OnChain: abi.OnChainSealVerifyInfo{ - SealedCID: commR, + SealedCID: cids.Sealed, InteractiveEpoch: seed.Epoch, RegisteredProof: spt, Proof: proof, @@ -301,7 +304,7 @@ func main() { }, Randomness: ticket, InteractiveRandomness: seed.Value, - UnsealedCID: commD, + UnsealedCID: cids.Unsealed, } ok, err := sectorbuilder.ProofVerifier.VerifySeal(svi) @@ -318,7 +321,7 @@ func main() { if !c.Bool("skip-unseal") { log.Info("Unsealing sector") // TODO: RM unsealed sector first - rc, err := sb.ReadPieceFromSealedSector(context.TODO(), 1, 0, abi.UnpaddedPieceSize(sectorSize), ticket, commD) + rc, err := sb.ReadPieceFromSealedSector(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, 0, abi.UnpaddedPieceSize(sectorSize), ticket, cids.Unsealed) if err != nil { return err } @@ -380,7 +383,7 @@ func main() { if !c.Bool("skip-commit2") { log.Info("generating election post candidates") - fcandidates, err := sb.GenerateEPostCandidates(sealedSectors, challenge[:], []abi.SectorNumber{}) + fcandidates, err := sb.GenerateEPostCandidates(context.TODO(), mid, sealedSectors, challenge[:], []abi.SectorNumber{}) if err != nil { return err } @@ -394,7 +397,7 @@ func main() { gencandidates := time.Now() log.Info("computing election post snark (cold)") - proof1, err := sb.ComputeElectionPoSt(sealedSectors, challenge[:], candidates[:1]) + proof1, err := sb.ComputeElectionPoSt(context.TODO(), mid, sealedSectors, challenge[:], candidates[:1]) if err != nil { return err } @@ -402,7 +405,7 @@ func main() { epost1 := time.Now() log.Info("computing election post snark (hot)") - proof2, err := sb.ComputeElectionPoSt(sealedSectors, challenge[:], candidates[:1]) + proof2, err := sb.ComputeElectionPoSt(context.TODO(), mid, sealedSectors, challenge[:], candidates[:1]) if err != nil { return err } @@ -528,6 +531,10 @@ var proveCmd = &cli.Command{ if err != nil { return err } + mid, err := address.IDFromAddress(maddr) + if err != nil { + return err + } ppt, spt, err := lapi.ProofTypeFromSectorSize(abi.SectorSize(c2in.SectorSize)) if err != nil { @@ -535,7 +542,6 @@ var proveCmd = &cli.Command{ } cfg := §orbuilder.Config{ - Miner: maddr, SealProofType: spt, PoStProofType: ppt, } @@ -547,7 +553,7 @@ var proveCmd = &cli.Command{ start := time.Now() - proof, err := sb.SealCommit2(context.TODO(), abi.SectorNumber(c2in.SectorNum), c2in.Phase1Out) + proof, err := sb.SealCommit2(context.TODO(), abi.SectorID{Miner: abi.ActorID(mid), Number: abi.SectorNumber(c2in.SectorNum)}, c2in.Phase1Out) if err != nil { return err } diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 89e310eb5..9f94e212d 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -59,7 +59,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum next := offset sbfs := &fs.Basic{ - Root: sbroot, + Root: sbroot, } sb, err := sectorbuilder.New(sbfs, cfg) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index e15a8c79e..2b49bc16a 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -400,11 +400,11 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, smgr, err := advmgr.New(lr, stores.NewIndex(), §orbuilder.Config{ SealProofType: spt, PoStProofType: ppt, - }, nil) + }, nil, stores.NewIndex()) if err != nil { return err } - epp := storage.NewElectionPoStProver(smgr, abi.ActorID(mid)) + epp := storage.NewElectionPoStProver(smgr, dtypes.MinerID(mid)) m := miner.NewMiner(api, epp) { diff --git a/go.mod b/go.mod index f7d665963..1f47eab00 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/filecoin-project/go-fil-markets v0.0.0-20200304003055-d449a980d4bd github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 - github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317165603-bd9e7cb04d81 + github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317221918-42574fc2aab9 github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 github.com/filecoin-project/go-statestore v0.1.0 github.com/filecoin-project/specs-actors v0.0.0-20200311215506-e95895452888 diff --git a/go.sum b/go.sum index dbd585bd2..8683dc335 100644 --- a/go.sum +++ b/go.sum @@ -128,6 +128,8 @@ github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200314022627-38af9db49ba github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200314022627-38af9db49ba2/go.mod h1:NcE+iL0bbYnamGmYQgCPVGbSaf8VF2/CLra/61B3I3I= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317165603-bd9e7cb04d81 h1:W5yekTpVTUiB86rSDiZo6rTI3lrLKrsrdY0tx/IqgJA= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317165603-bd9e7cb04d81/go.mod h1:3c3MEU9GHLlau37+MmefFNunTo9sVEKfjaJuHBgksdY= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317221918-42574fc2aab9 h1:ROfxm5X9dMATYk6MvNe8WBcL2yaelpgGEvve5CnZ1+g= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317221918-42574fc2aab9/go.mod h1:3c3MEU9GHLlau37+MmefFNunTo9sVEKfjaJuHBgksdY= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ= diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index d5e60f056..8f598be05 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -45,7 +45,7 @@ func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID uin } sid := abi.SectorID{ - Miner: abi.ActorID(mid), + Miner: abi.ActorID(mid), Number: abi.SectorNumber(sectorID), } return rpn.sealer.ReadPieceFromSealedSector(ctx, sid, sectorbuilder.UnpaddedByteIndex(offset), abi.UnpaddedPieceSize(length), si.Ticket.Value, *si.CommD) diff --git a/node/builder.go b/node/builder.go index 1941ef2e3..9470eca26 100644 --- a/node/builder.go +++ b/node/builder.go @@ -260,6 +260,9 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, Override(new(*stores.Index), stores.NewIndex()), + Override(new(stores.SectorIndex), From(new(*stores.Index))), + Override(new(dtypes.MinerID), modules.MinerID), + Override(new(dtypes.MinerAddress), modules.MinerAddress), Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig), Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), Override(new(sealing.SectorIDCounter), modules.SectorIDCounter), diff --git a/node/modules/dtypes/miner.go b/node/modules/dtypes/miner.go new file mode 100644 index 000000000..c872fdf69 --- /dev/null +++ b/node/modules/dtypes/miner.go @@ -0,0 +1,9 @@ +package dtypes + +import ( + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/abi" +) + +type MinerAddress address.Address +type MinerID abi.ActorID diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index fa46c9336..3a62b5887 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -73,13 +73,18 @@ func GetParams(sbc *sectorbuilder.Config) error { return nil } -func SectorBuilderConfig(ds dtypes.MetadataDS, fnapi lapi.FullNode) (*sectorbuilder.Config, error) { - minerAddr, err := minerAddrFromDS(ds) - if err != nil { - return nil, err - } +func MinerAddress(ds dtypes.MetadataDS) (dtypes.MinerAddress, error) { + ma, err := minerAddrFromDS(ds) + return dtypes.MinerAddress(ma), err +} - ssize, err := fnapi.StateMinerSectorSize(context.TODO(), minerAddr, types.EmptyTSK) +func MinerID(ma dtypes.MinerAddress) (dtypes.MinerID, error) { + id, err := address.IDFromAddress(address.Address(ma)) + return dtypes.MinerID(id), err +} + +func SectorBuilderConfig(maddr dtypes.MinerAddress, fnapi lapi.FullNode) (*sectorbuilder.Config, error) { + ssize, err := fnapi.StateMinerSectorSize(context.TODO(), address.Address(maddr), types.EmptyTSK) if err != nil { return nil, err } @@ -111,7 +116,7 @@ func SectorIDCounter(ds dtypes.MetadataDS) sealing.SectorIDCounter { return &sidsc{sc} } -func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sealmgr.Manager, tktFn sealing.TicketFn) (*storage.Miner, error) { +func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sealmgr.Manager, sc sealing.SectorIDCounter, tktFn sealing.TicketFn) (*storage.Miner, error) { maddr, err := minerAddrFromDS(ds) if err != nil { return nil, err @@ -131,7 +136,7 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h fps := storage.NewFPoStScheduler(api, sealer, maddr, worker, ppt) - sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, tktFn) + sm, err := storage.NewMiner(api, maddr, worker, h, ds, sealer, sc, tktFn) if err != nil { return nil, err } diff --git a/node/node_test.go b/node/node_test.go index c915c9328..de495017b 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -80,6 +80,7 @@ func testStorageNode(ctx context.Context, t *testing.T, waddr address.Address, a for i := 0; i < nPreseal; i++ { nic.Next() } + nic.Next() err = lr.Close() require.NoError(t, err) @@ -256,7 +257,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options()) if err := storers[i].StorageAddLocal(ctx, presealDirs[i]); err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) } /* sma := storers[i].StorageMiner.(*impl.StorageMinerAPI) @@ -386,7 +387,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options( node.Override(new(sealmgr.Manager), func() (sealmgr.Manager, error) { - return sealmgr.NewSimpleManager(storedcounter.New(datastore.NewMapDatastore(), datastore.NewKey("/potato")), genMiner, sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])) + return sealmgr.NewSimpleManager(genMiner, sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])) }), node.Unset(new(*advmgr.Manager)), )) diff --git a/storage/miner.go b/storage/miner.go index ce6bbdc14..9fea5f042 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -3,6 +3,7 @@ package storage import ( "context" "errors" + "github.com/filecoin-project/lotus/node/modules/dtypes" "time" "github.com/filecoin-project/go-address" @@ -36,6 +37,7 @@ type Miner struct { sealer sealmgr.Manager ds datastore.Batching tktFn sealing.TicketFn + sc sealing.SectorIDCounter maddr address.Address worker address.Address @@ -72,13 +74,14 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sealmgr.Manager, tktFn sealing.TicketFn) (*Miner, error) { +func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sealmgr.Manager, sc sealing.SectorIDCounter, tktFn sealing.TicketFn) (*Miner, error) { m := &Miner{ api: api, h: h, sealer: sealer, ds: ds, tktFn: tktFn, + sc: sc, maddr: maddr, worker: worker, @@ -93,7 +96,7 @@ func (m *Miner) Run(ctx context.Context) error { } evts := events.NewEvents(ctx, m.api) - m.sealing = sealing.New(m.api, evts, m.maddr, m.worker, m.ds, m.sealer, m.tktFn) + m.sealing = sealing.New(m.api, evts, m.maddr, m.worker, m.ds, m.sealer, m.sc, m.tktFn) go m.sealing.Run(ctx) @@ -121,11 +124,11 @@ func (m *Miner) runPreflightChecks(ctx context.Context) error { type SectorBuilderEpp struct { prover storage.Prover - miner abi.ActorID + miner abi.ActorID } -func NewElectionPoStProver(sb storage.Prover, miner abi.ActorID) *SectorBuilderEpp { - return &SectorBuilderEpp{sb, miner} +func NewElectionPoStProver(sb storage.Prover, miner dtypes.MinerID) *SectorBuilderEpp { + return &SectorBuilderEpp{sb, abi.ActorID(miner)} } var _ gen.ElectionPoStProver = (*SectorBuilderEpp)(nil) diff --git a/storage/sbmock/sbmock.go b/storage/sbmock/sbmock.go index f93029022..7708578bb 100644 --- a/storage/sbmock/sbmock.go +++ b/storage/sbmock/sbmock.go @@ -304,25 +304,27 @@ func (sb *SBMock) ReadPieceFromSealedSector(ctx context.Context, sectorID abi.Se return ioutil.NopCloser(io.LimitReader(bytes.NewReader(sb.sectors[sectorID].pieces[0].Bytes()[offset:]), int64(size))), nil } -func (sb *SBMock) StageFakeData(mid abi.ActorID) (abi.SectorNumber, []abi.PieceInfo, error) { +func (sb *SBMock) StageFakeData(mid abi.ActorID) (abi.SectorID, []abi.PieceInfo, error) { usize := abi.PaddedPieceSize(sb.sectorSize).Unpadded() sid, err := sb.AcquireSectorNumber() if err != nil { - return 0, nil, err + return abi.SectorID{}, nil, err } buf := make([]byte, usize) rand.Read(buf) - pi, err := sb.AddPiece(context.TODO(), abi.SectorID{ + id := abi.SectorID{ Miner: mid, Number: sid, - }, nil, usize, bytes.NewReader(buf)) - if err != nil { - return 0, nil, err } - return sid, []abi.PieceInfo{pi}, nil + pi, err := sb.AddPiece(context.TODO(), id, nil, usize, bytes.NewReader(buf)) + if err != nil { + return abi.SectorID{}, nil, err + } + + return id, []abi.PieceInfo{pi}, nil } func (sb *SBMock) FinalizeSector(context.Context, abi.SectorID) error { diff --git a/storage/sbmock/sbmock_test.go b/storage/sbmock/sbmock_test.go index 6bfe9e062..1d6b1a4f6 100644 --- a/storage/sbmock/sbmock_test.go +++ b/storage/sbmock/sbmock_test.go @@ -11,7 +11,7 @@ import ( func TestOpFinish(t *testing.T) { sb := NewMockSectorBuilder(1, 2048) - sid, pieces, err := sb.StageFakeData() + sid, pieces, err := sb.StageFakeData(123) if err != nil { t.Fatal(err) } diff --git a/storage/sealing/sealing.go b/storage/sealing/sealing.go index ed793b0ae..3c50a1507 100644 --- a/storage/sealing/sealing.go +++ b/storage/sealing/sealing.go @@ -75,7 +75,7 @@ type Sealing struct { sc SectorIDCounter } -func New(api sealingApi, events *events.Events, maddr address.Address, worker address.Address, ds datastore.Batching, sealer sealmgr.Manager, tktFn TicketFn) *Sealing { +func New(api sealingApi, events *events.Events, maddr address.Address, worker address.Address, ds datastore.Batching, sealer sealmgr.Manager, sc SectorIDCounter, tktFn TicketFn) *Sealing { s := &Sealing{ api: api, events: events, @@ -84,6 +84,7 @@ func New(api sealingApi, events *events.Events, maddr address.Address, worker ad worker: worker, sealer: sealer, tktFn: tktFn, + sc: sc, } s.sectors = statemachine.New(namespace.Wrap(ds, datastore.NewKey(SectorStorePrefix)), s, SectorInfo{}) @@ -163,7 +164,7 @@ func (m *Sealing) minerSector(num abi.SectorNumber) abi.SectorID { return abi.SectorID{ Number: num, - Miner: abi.ActorID(mid), + Miner: abi.ActorID(mid), } } diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/local.go index 38dcf38f7..358ee33d2 100644 --- a/storage/sealmgr/advmgr/local.go +++ b/storage/sealmgr/advmgr/local.go @@ -70,7 +70,6 @@ func (l *LocalWorker) sb() (sectorbuilder.Basic, error) { return sectorbuilder.New(&localWorkerPathProvider{w: l}, l.scfg) } - func (l *LocalWorker) NewSector(ctx context.Context, sector abi.SectorID) error { sb, err := l.sb() if err != nil { diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 52e5463b2..89518ec69 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -42,7 +42,7 @@ type Manager struct { storage2.Prover } -func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, urls URLs) (*Manager, error) { +func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, urls URLs, sindex stores.SectorIndex) (*Manager, error) { stor, err := stores.NewLocal(ls) if err != nil { return nil, err @@ -59,7 +59,7 @@ func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, ur m := &Manager{ workers: []Worker{ - &LocalWorker{scfg: cfg, storage: stor}, + &LocalWorker{scfg: cfg, localStore: stor, storage: stor, sindex: sindex}, }, scfg: cfg, diff --git a/storage/sealmgr/advmgr/roprov.go b/storage/sealmgr/advmgr/roprov.go index dd549041f..32e0c9ba4 100644 --- a/storage/sealmgr/advmgr/roprov.go +++ b/storage/sealmgr/advmgr/roprov.go @@ -11,7 +11,7 @@ import ( ) type readonlyProvider struct { - stor *stores.Local + stor *stores.Local } func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { diff --git a/storage/sealmgr/simple.go b/storage/sealmgr/simple.go index 7cf8c98c9..1dc47b64e 100644 --- a/storage/sealmgr/simple.go +++ b/storage/sealmgr/simple.go @@ -31,6 +31,10 @@ type sszgetter interface { SectorSize() abi.SectorSize } +func (s *LocalWorker) SectorSize() abi.SectorSize { + return s.Basic.(sszgetter).SectorSize() +} + func (s *Simple) SectorSize() abi.SectorSize { return s.worker.(sszgetter).SectorSize() } From d8c8832a37f61f560d8aa2558a36fe6b1a80be14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 18 Mar 2020 05:40:25 +0100 Subject: [PATCH 14/45] stores: fix StorageList --- cmd/lotus-seal-worker/main.go | 20 +++++++++----------- storage/sealmgr/stores/index.go | 3 +++ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index ba431f22e..8968abf40 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -7,9 +7,7 @@ import ( "net" "net/http" "os" - "os/signal" "path/filepath" - "syscall" "github.com/google/uuid" "github.com/gorilla/mux" @@ -117,11 +115,6 @@ var runCmd = &cli.Command{ } log.Infof("Remote version %s", v) - go func() { - <-ctx.Done() - log.Warn("Shutting down..") - }() - // Check params act, err := nodeApi.ActorAddress(ctx) @@ -202,6 +195,8 @@ var runCmd = &cli.Command{ return err } + log.Info("Connecting local storage to master") + if err := stores.DeclareLocalStorage( ctx, nodeApi, @@ -231,6 +226,8 @@ var runCmd = &cli.Command{ mux := mux.NewRouter() + log.Info("Setting up control endpoint at " + "http://" + cctx.String("address")) + rpcServer := jsonrpc.NewServer() rpcServer.Register("Filecoin", apistruct.PermissionedWorkerAPI(workerApi)) @@ -245,9 +242,8 @@ var runCmd = &cli.Command{ srv := &http.Server{Handler: ah} - sigChan := make(chan os.Signal, 2) go func() { - <-sigChan + <-ctx.Done() log.Warn("Shutting down..") if err := srv.Shutdown(context.TODO()); err != nil { log.Errorf("shutting down RPC server failed: %s", err) @@ -255,13 +251,15 @@ var runCmd = &cli.Command{ log.Warn("Graceful shutdown successful") }() - signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT) - nl, err := net.Listen("tcp4", cctx.String("address")) if err != nil { return err } + log.Info("Waiting for tasks") + + // todo go register + return srv.Serve(nl) }, } diff --git a/storage/sealmgr/stores/index.go b/storage/sealmgr/stores/index.go index df370741c..db4769989 100644 --- a/storage/sealmgr/stores/index.go +++ b/storage/sealmgr/stores/index.go @@ -67,6 +67,7 @@ func (i *Index) StorageList(ctx context.Context) (map[ID][]Decl, error) { out := map[ID][]Decl{} for id, m := range byID { + out[id] = []Decl{} for sectorID, fileType := range m { out[id] = append(out[id], Decl{ SectorID: sectorID, @@ -82,6 +83,8 @@ func (i *Index) StorageAttach(ctx context.Context, si StorageInfo) error { i.lk.Lock() defer i.lk.Unlock() + log.Infof("New sector storage: %s", si.ID) + if _, ok := i.stores[si.ID]; ok { for _, u := range si.URLs { if _, err := url.Parse(u); err != nil { From 94ebbd9d900a438dc672c5dda50ccccb17d84225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 19 Mar 2020 00:23:28 +0100 Subject: [PATCH 15/45] workers: Actually register in miner --- cmd/lotus-seal-worker/main.go | 24 ++++++++++++++++++- lib/lotuslog/levels.go | 2 ++ node/impl/storminer.go | 8 ++++--- storage/sealmgr/advmgr/manager.go | 17 ++++++++++++- .../advmgr/{local.go => worker_local.go} | 0 .../advmgr/{remote.go => worker_remote.go} | 2 +- storage/sealmgr/stores/local.go | 11 +++++++++ 7 files changed, 58 insertions(+), 6 deletions(-) rename storage/sealmgr/advmgr/{local.go => worker_local.go} (100%) rename storage/sealmgr/advmgr/{remote.go => worker_remote.go} (92%) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 8968abf40..f80adbef7 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -105,6 +105,8 @@ var runCmd = &cli.Command{ } defer closer() ctx := lcli.ReqContext(cctx) + ctx, cancel := context.WithCancel(ctx) + defer cancel() v, err := nodeApi.Version(ctx) if err != nil { @@ -180,6 +182,13 @@ var runCmd = &cli.Command{ return xerrors.Errorf("set storage config: %w", err) } + { + // init datastore for r.Exists + _, err := lr.Datastore("/") + if err != nil { + return err + } + } if err := lr.Close(); err != nil { return xerrors.Errorf("close repo: %w", err) } @@ -240,7 +249,12 @@ var runCmd = &cli.Command{ Next: mux.ServeHTTP, } - srv := &http.Server{Handler: ah} + srv := &http.Server{ + Handler: ah, + BaseContext: func(listener net.Listener) context.Context { + return ctx + }, + } go func() { <-ctx.Done() @@ -258,6 +272,14 @@ var runCmd = &cli.Command{ log.Info("Waiting for tasks") + go func() { + if err := nodeApi.WorkerConnect(ctx, "ws://"+cctx.String("address")+"/rpc/v0"); err != nil { + log.Errorf("Registering worker failed: %+v", err) + cancel() + return + } + }() + // todo go register return srv.Serve(nl) diff --git a/lib/lotuslog/levels.go b/lib/lotuslog/levels.go index e49c46006..5f92ccc65 100644 --- a/lib/lotuslog/levels.go +++ b/lib/lotuslog/levels.go @@ -14,5 +14,7 @@ func SetupLogLevels() { logging.SetLogLevel("bitswap", "WARN") //logging.SetLogLevel("pubsub", "WARN") logging.SetLogLevel("connmgr", "WARN") + logging.SetLogLevel("advmgr", "DEBUG") + logging.SetLogLevel("stores", "DEBUG") } } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 6ec6d9acb..3227a975a 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -145,12 +145,14 @@ func (sm *StorageMinerAPI) SectorsUpdate(ctx context.Context, id abi.SectorNumbe } func (sm *StorageMinerAPI) WorkerConnect(ctx context.Context, url string) error { - _, err := advmgr.ConnectRemote(ctx, sm.Full, url) + w, err := advmgr.ConnectRemote(ctx, sm, url) if err != nil { - return err + return xerrors.Errorf("connecting remote storage failed: %w", err) } - panic("todo register ") + log.Infof("Connected to a remote worker at %s", url) + + return sm.StorageMgr.AddWorker(w) } func (sm *StorageMinerAPI) MarketImportDealData(ctx context.Context, propCid cid.Cid, path string) error { diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 89518ec69..87c56e8a1 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -4,6 +4,7 @@ import ( "context" "io" "net/http" + "sync" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" @@ -40,6 +41,8 @@ type Manager struct { remoteHnd *stores.FetchHandler storage2.Prover + + lk sync.Mutex } func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, urls URLs, sindex stores.SectorIndex) (*Manager, error) { @@ -59,7 +62,7 @@ func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, ur m := &Manager{ workers: []Worker{ - &LocalWorker{scfg: cfg, localStore: stor, storage: stor, sindex: sindex}, + //&LocalWorker{scfg: cfg, localStore: stor, storage: stor, sindex: sindex}, }, scfg: cfg, @@ -91,6 +94,14 @@ func (m *Manager) AddLocalStorage(path string) error { return nil } +func (m *Manager) AddWorker(w Worker) error { + m.lk.Lock() + defer m.lk.Unlock() + + m.workers = append(m.workers, w) + return nil +} + func (m *Manager) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.remoteHnd.ServeHTTP(w, r) } @@ -115,6 +126,7 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.Stor continue } if _, ok := tt[task]; !ok { + log.Debugf("dropping worker %d; task %s not supported (supports %v)", i, task, tt) continue } @@ -139,6 +151,8 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.Stor } } if st == nil { + log.Debugf("skipping worker %d; doesn't have any of %v", i, inPaths) + log.Debugf("skipping worker %d; only has %v", i, phs) continue } @@ -168,6 +182,7 @@ func (m *Manager) AddPiece(ctx context.Context, sector abi.SectorID, existingPie return abi.PieceInfo{}, xerrors.Errorf("finding sector path: %w", err) } + log.Debugf("find workers for %v", best) candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTAddPiece, best) if len(candidateWorkers) == 0 { diff --git a/storage/sealmgr/advmgr/local.go b/storage/sealmgr/advmgr/worker_local.go similarity index 100% rename from storage/sealmgr/advmgr/local.go rename to storage/sealmgr/advmgr/worker_local.go diff --git a/storage/sealmgr/advmgr/remote.go b/storage/sealmgr/advmgr/worker_remote.go similarity index 92% rename from storage/sealmgr/advmgr/remote.go rename to storage/sealmgr/advmgr/worker_remote.go index 27699434e..734e14ebc 100644 --- a/storage/sealmgr/advmgr/remote.go +++ b/storage/sealmgr/advmgr/worker_remote.go @@ -24,7 +24,7 @@ func (r *remote) AddPiece(ctx context.Context, sector abi.SectorID, pieceSizes [ return abi.PieceInfo{}, xerrors.New("unsupported") } -func ConnectRemote(ctx context.Context, fa api.FullNode, url string) (*remote, error) { +func ConnectRemote(ctx context.Context, fa api.Common, url string) (*remote, error) { token, err := fa.AuthNew(ctx, []api.Permission{"admin"}) if err != nil { return nil, xerrors.Errorf("creating auth token for remote connection: %w", err) diff --git a/storage/sealmgr/stores/local.go b/storage/sealmgr/stores/local.go index 4b9a0ed71..4cf3cf565 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sealmgr/stores/local.go @@ -212,13 +212,18 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing s } func (st *Local) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sealing bool) ([]StorageMeta, error) { + st.localLk.RLock() + defer st.localLk.RUnlock() + var out []StorageMeta for _, p := range st.paths { if sealing && !p.meta.CanSeal { + log.Debugf("alloc: not considering %s; can't seal", p.meta.ID) continue } if !sealing && !p.meta.CanStore { + log.Debugf("alloc: not considering %s; can't store", p.meta.ID) continue } @@ -236,6 +241,9 @@ func (st *Local) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sea } func (st *Local) FindSector(id abi.SectorID, typ sectorbuilder.SectorFileType) ([]StorageMeta, error) { + st.localLk.RLock() + defer st.localLk.RUnlock() + var out []StorageMeta for _, p := range st.paths { p.lk.Lock() @@ -254,6 +262,9 @@ func (st *Local) FindSector(id abi.SectorID, typ sectorbuilder.SectorFileType) ( } func (st *Local) Local() []StoragePath { + st.localLk.RLock() + defer st.localLk.RUnlock() + var out []StoragePath for _, p := range st.paths { if p.local == "" { From 2d1e30ae3b0065ac6c06561bd7046638bdfe5057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 19 Mar 2020 03:23:21 +0100 Subject: [PATCH 16/45] workers: Call find* on remote --- cmd/lotus-storage-miner/init.go | 2 +- storage/sealmgr/advmgr/manager.go | 38 +++++++++++++++++++------------ storage/sealmgr/stores/remote.go | 8 +++++++ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 2b49bc16a..f24f5afd6 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -400,7 +400,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, smgr, err := advmgr.New(lr, stores.NewIndex(), §orbuilder.Config{ SealProofType: spt, PoStProofType: ppt, - }, nil, stores.NewIndex()) + }, nil) if err != nil { return err } diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 87c56e8a1..6bbf72ae6 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -12,13 +12,14 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" - "github.com/filecoin-project/specs-actors/actors/abi" storage2 "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" ) var log = logging.Logger("advmgr") @@ -29,6 +30,8 @@ type Worker interface { sectorbuilder.Sealer TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) + + // Returns paths accessible to the worker Paths(context.Context) ([]stores.StoragePath, error) } @@ -36,39 +39,46 @@ type Manager struct { workers []Worker scfg *sectorbuilder.Config - ls stores.LocalStorage - storage *stores.Local - remoteHnd *stores.FetchHandler + ls stores.LocalStorage + storage *stores.Remote + localStore *stores.Local + remoteHnd *stores.FetchHandler storage2.Prover lk sync.Mutex } -func New(ls stores.LocalStorage, si *stores.Index, cfg *sectorbuilder.Config, urls URLs, sindex stores.SectorIndex) (*Manager, error) { - stor, err := stores.NewLocal(ls) +func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls URLs, ca impl.CommonAPI) (*Manager, error) { + lstor, err := stores.NewLocal(ls) if err != nil { return nil, err } - if err := stores.DeclareLocalStorage(context.TODO(), si, stor, urls, 10); err != nil { + if err := stores.DeclareLocalStorage(context.TODO(), si, lstor, urls, 10); err != nil { log.Errorf("Declaring local storage failed: %+v") } - prover, err := sectorbuilder.New(&readonlyProvider{stor: stor}, cfg) + prover, err := sectorbuilder.New(&readonlyProvider{stor: lstor}, cfg) if err != nil { return nil, xerrors.Errorf("creating prover instance: %w", err) } + token, err := ca.AuthNew(context.TODO(), []api.Permission{"admin"}) + headers := http.Header{} + headers.Add("Authorization", "Bearer "+string(token)) + stor := stores.NewRemote(lstor, si, headers) + m := &Manager{ workers: []Worker{ - //&LocalWorker{scfg: cfg, localStore: stor, storage: stor, sindex: sindex}, + //&LocalWorker{scfg: cfg, localStore: lstor, storage: lstor, sindex: sindex}, }, scfg: cfg, - ls: ls, - storage: stor, - remoteHnd: &stores.FetchHandler{Store: stor}, + ls: ls, + storage: stor, + localStore: lstor, + remoteHnd: &stores.FetchHandler{Store: lstor}, Prover: prover, } @@ -82,7 +92,7 @@ func (m *Manager) AddLocalStorage(path string) error { return xerrors.Errorf("expanding local path: %w", err) } - if err := m.storage.OpenPath(path); err != nil { + if err := m.localStore.OpenPath(path); err != nil { return xerrors.Errorf("opening local path: %w", err) } diff --git a/storage/sealmgr/stores/remote.go b/storage/sealmgr/stores/remote.go index 04272b450..ad11879c9 100644 --- a/storage/sealmgr/stores/remote.go +++ b/storage/sealmgr/stores/remote.go @@ -163,6 +163,14 @@ func (r *Remote) fetch(url, outname string) error { } +func (r *Remote) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sealing bool) ([]StorageMeta, error) { + panic("todo") +} + +func (r *Remote) FindSector(id abi.SectorID, typ sectorbuilder.SectorFileType) ([]StorageMeta, error) { + panic("todo") +} + func mergeDone(a func(), b func()) func() { return func() { a() From 2fa3f2578fda352a7b1b537f6782c9724a1ae5b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 19 Mar 2020 03:31:53 +0100 Subject: [PATCH 17/45] impl: Move CommonAPI into a separate pkg --- node/impl/{ => common}/common.go | 2 +- node/impl/full.go | 8 ++++---- node/impl/storminer.go | 7 +++---- storage/sealmgr/advmgr/manager.go | 4 ++-- 4 files changed, 10 insertions(+), 11 deletions(-) rename node/impl/{ => common}/common.go (99%) diff --git a/node/impl/common.go b/node/impl/common/common.go similarity index 99% rename from node/impl/common.go rename to node/impl/common/common.go index 359a64543..38f5587ad 100644 --- a/node/impl/common.go +++ b/node/impl/common/common.go @@ -1,4 +1,4 @@ -package impl +package common import ( "context" diff --git a/node/impl/full.go b/node/impl/full.go index 8e7456604..1329e956f 100644 --- a/node/impl/full.go +++ b/node/impl/full.go @@ -3,18 +3,18 @@ package impl import ( logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/impl/client" + "github.com/filecoin-project/lotus/node/impl/common" + "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/market" "github.com/filecoin-project/lotus/node/impl/paych" - - "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/node/impl/full" ) var log = logging.Logger("node") type FullNodeAPI struct { - CommonAPI + common.CommonAPI full.ChainAPI client.API full.MpoolAPI diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 3227a975a..502938fde 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -3,7 +3,6 @@ package impl import ( "context" "encoding/json" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" "net/http" "os" "strconv" @@ -13,22 +12,22 @@ import ( "github.com/filecoin-project/go-address" storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" - "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/miner" + "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" "github.com/filecoin-project/lotus/storage/sectorblocks" ) type StorageMinerAPI struct { - CommonAPI + common.CommonAPI SectorBuilderConfig *sectorbuilder.Config //SectorBuilder sectorbuilder.Interface diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 6bbf72ae6..c4f47b2d4 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/config" - "github.com/filecoin-project/lotus/node/impl" + "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/storage/sealmgr" "github.com/filecoin-project/lotus/storage/sealmgr/stores" ) @@ -49,7 +49,7 @@ type Manager struct { lk sync.Mutex } -func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls URLs, ca impl.CommonAPI) (*Manager, error) { +func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls URLs, ca common.CommonAPI) (*Manager, error) { lstor, err := stores.NewLocal(ls) if err != nil { return nil, err From d87b7c264d2e20fe7135a43c316374f90c77e27a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 19 Mar 2020 16:10:19 +0100 Subject: [PATCH 18/45] stores: Use index in local store --- api/api_storage.go | 2 - api/apistruct/struct.go | 21 ++- cmd/lotus-seal-worker/main.go | 20 +-- cmd/lotus-seed/seed/seed.go | 2 +- cmd/lotus-storage-miner/init.go | 4 +- cmd/lotus-storage-miner/storage.go | 4 +- node/impl/storminer.go | 2 +- node/repo/memrepo.go | 2 +- storage/sealmgr/advmgr/manager.go | 37 +++-- storage/sealmgr/advmgr/worker_local.go | 4 +- storage/sealmgr/stores/index.go | 118 +++++++++----- storage/sealmgr/stores/interface.go | 20 +++ storage/sealmgr/stores/local.go | 203 +++++++++++-------------- storage/sealmgr/stores/remote.go | 10 +- 14 files changed, 243 insertions(+), 206 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index fe7e4eb51..872a8631b 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -110,8 +110,6 @@ type StorageMiner interface { StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) - StorageInfo(context.Context, stores.ID) (stores.StorageInfo, error) - // WorkerConnect tells the node to connect to workers RPC WorkerConnect(context.Context, string) error stores.SectorIndex diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 6a048ee03..e75a85ad5 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -181,12 +181,13 @@ type StorageMinerStruct struct { SectorsRefs func(context.Context) (map[string][]api.SealedRef, error) `perm:"read"` SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error `perm:"write"` - WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm - StorageAttach func(context.Context, stores.StorageInfo) error `perm:"admin"` - StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, sectorbuilder.SectorFileType) error `perm:"admin"` - StorageFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]stores.StorageInfo, error) `perm:"admin"` - StorageList func(ctx context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` - StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` + WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm + StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` + StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, sectorbuilder.SectorFileType) error `perm:"admin"` + StorageFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]stores.StorageInfo, error) `perm:"admin"` + StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` + StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` + StorageBestAlloc func(ctx context.Context, allocate sectorbuilder.SectorFileType, sealing bool) ([]stores.StorageInfo, error) `perm:"admin"` DealsImportData func(ctx context.Context, dealPropCid cid.Cid, file string) error `perm:"write"` DealsList func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` @@ -654,8 +655,8 @@ func (c *StorageMinerStruct) WorkerConnect(ctx context.Context, url string) erro return c.Internal.WorkerConnect(ctx, url) } -func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo) error { - return c.Internal.StorageAttach(ctx, si) +func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo, st stores.FsStat) error { + return c.Internal.StorageAttach(ctx, si, st) } func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error { @@ -674,6 +675,10 @@ func (c *StorageMinerStruct) StorageInfo(ctx context.Context, id stores.ID) (sto return c.Internal.StorageInfo(ctx, id) } +func (c *StorageMinerStruct) StorageBestAlloc(ctx context.Context, allocate sectorbuilder.SectorFileType, sealing bool) ([]stores.StorageInfo, error) { + return c.Internal.StorageBestAlloc(ctx, allocate, sealing) +} + func (c *StorageMinerStruct) MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error { return c.Internal.MarketImportDealData(ctx, propcid, path) } diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index f80adbef7..15cdd31d7 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -157,7 +157,7 @@ var runCmd = &cli.Command{ var localPaths []config.LocalPath if !cctx.Bool("no-local-storage") { - b, err := json.MarshalIndent(&stores.StorageMeta{ + b, err := json.MarshalIndent(&stores.LocalStorageMeta{ ID: stores.ID(uuid.New().String()), Weight: 10, CanSeal: true, @@ -199,21 +199,13 @@ var runCmd = &cli.Command{ return err } - localStore, err := stores.NewLocal(lr) + log.Info("Opening local storage; connecting to master") + + localStore, err := stores.NewLocal(ctx, lr, nodeApi, []string{"http://" + cctx.String("address") + "/remote"}) if err != nil { return err } - log.Info("Connecting local storage to master") - - if err := stores.DeclareLocalStorage( - ctx, - nodeApi, - localStore, - []string{"http://" + cctx.String("address") + "/remote"}, // TODO: Less hardcoded - 1); err != nil { - return err - } // Setup remote sector store _, spt, err := api.ProofTypeFromSectorSize(ssize) if err != nil { @@ -230,12 +222,12 @@ var runCmd = &cli.Command{ // Create / expose the worker workerApi := &worker{ - LocalWorker: advmgr.NewLocalWorker(act, spt, remote, localStore, stores.NewIndex()), + LocalWorker: advmgr.NewLocalWorker(act, spt, remote, localStore, nodeApi), } mux := mux.NewRouter() - log.Info("Setting up control endpoint at " + "http://" + cctx.String("address")) + log.Info("Setting up control endpoint at " + cctx.String("address")) rpcServer := jsonrpc.NewServer() rpcServer.Register("Filecoin", apistruct.PermissionedWorkerAPI(workerApi)) diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 9f94e212d..8ad1896e1 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -137,7 +137,7 @@ func PreSeal(maddr address.Address, pt abi.RegisteredProof, offset abi.SectorNum } { - b, err := json.MarshalIndent(&stores.StorageMeta{ + b, err := json.MarshalIndent(&stores.LocalStorageMeta{ ID: stores.ID(uuid.New().String()), Weight: 0, // read-only CanSeal: false, diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 13faa8857..145487d58 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -188,7 +188,7 @@ var initCmd = &cli.Command{ } if !cctx.Bool("no-local-storage") { - b, err := json.MarshalIndent(&stores.StorageMeta{ + b, err := json.MarshalIndent(&stores.LocalStorageMeta{ ID: stores.ID(uuid.New().String()), Weight: 10, CanSeal: true, @@ -400,7 +400,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, smgr, err := advmgr.New(lr, stores.NewIndex(), §orbuilder.Config{ SealProofType: spt, PoStProofType: ppt, - }, nil) + }, nil, api) if err != nil { return err } diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index cf07a5ece..a72da5b52 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -83,7 +83,7 @@ var storageAttachCmd = &cli.Command{ return err } - cfg := &stores.StorageMeta{ + cfg := &stores.LocalStorageMeta{ ID: stores.ID(uuid.New().String()), Weight: cctx.Uint64("weight"), CanSeal: cctx.Bool("seal"), @@ -144,7 +144,7 @@ var storageListCmd = &cli.Command{ if err != nil { return err } - fmt.Printf("\tSeal: %t; Store: %t; Cost: %d\n", si.CanSeal, si.CanStore, si.Cost) + fmt.Printf("\tSeal: %t; Store: %t; Weight: %d\n", si.CanSeal, si.CanStore, si.Weight) for _, l := range si.URLs { fmt.Printf("\tReachable %s\n", l) // TODO; try pinging maybe?? print latency? } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 502938fde..7162022b9 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -195,7 +195,7 @@ func (sm *StorageMinerAPI) StorageAddLocal(ctx context.Context, path string) err return xerrors.Errorf("no storage manager") } - return sm.StorageMgr.AddLocalStorage(path) + return sm.StorageMgr.AddLocalStorage(ctx, path) } var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 016b0f21e..61141f115 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -82,7 +82,7 @@ func (lmem *lockedMemRepo) Path() string { panic(err) } - b, err := json.MarshalIndent(&stores.StorageMeta{ + b, err := json.MarshalIndent(&stores.LocalStorageMeta{ ID: stores.ID(uuid.New().String()), Weight: 10, CanSeal: true, diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index c4f47b2d4..e3e19fabe 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -17,7 +17,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/config" - "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/storage/sealmgr" "github.com/filecoin-project/lotus/storage/sealmgr/stores" ) @@ -43,22 +42,21 @@ type Manager struct { storage *stores.Remote localStore *stores.Local remoteHnd *stores.FetchHandler + index stores.SectorIndex storage2.Prover lk sync.Mutex } -func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls URLs, ca common.CommonAPI) (*Manager, error) { - lstor, err := stores.NewLocal(ls) +func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls URLs, ca api.Common) (*Manager, error) { + ctx := context.TODO() + + lstor, err := stores.NewLocal(ctx, ls, si, urls) if err != nil { return nil, err } - if err := stores.DeclareLocalStorage(context.TODO(), si, lstor, urls, 10); err != nil { - log.Errorf("Declaring local storage failed: %+v") - } - prover, err := sectorbuilder.New(&readonlyProvider{stor: lstor}, cfg) if err != nil { return nil, xerrors.Errorf("creating prover instance: %w", err) @@ -79,6 +77,7 @@ func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Confi storage: stor, localStore: lstor, remoteHnd: &stores.FetchHandler{Store: lstor}, + index: si, Prover: prover, } @@ -86,13 +85,13 @@ func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Confi return m, nil } -func (m *Manager) AddLocalStorage(path string) error { +func (m *Manager) AddLocalStorage(ctx context.Context, path string) error { path, err := homedir.Expand(path) if err != nil { return xerrors.Errorf("expanding local path: %w", err) } - if err := m.localStore.OpenPath(path); err != nil { + if err := m.localStore.OpenPath(ctx, path); err != nil { return xerrors.Errorf("opening local path: %w", err) } @@ -125,9 +124,9 @@ func (m *Manager) ReadPieceFromSealedSector(context.Context, abi.SectorID, secto panic("implement me") } -func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.StorageMeta) ([]Worker, map[int]stores.StorageMeta) { +func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.StorageInfo) ([]Worker, map[int]stores.StorageInfo) { var workers []Worker - paths := map[int]stores.StorageMeta{} + paths := map[int]stores.StorageInfo{} for i, worker := range m.workers { tt, err := worker.TaskTypes(context.TODO()) @@ -147,7 +146,7 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.Stor } // check if the worker has access to the path we selected - var st *stores.StorageMeta + var st *stores.StorageInfo for _, p := range phs { for _, meta := range inPaths { if p.ID == meta.ID { @@ -181,12 +180,12 @@ func (m *Manager) NewSector(ctx context.Context, sector abi.SectorID) error { func (m *Manager) AddPiece(ctx context.Context, sector abi.SectorID, existingPieces []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { // TODO: consider multiple paths vs workers when initially allocating - var best []stores.StorageMeta + var best []stores.StorageInfo var err error if len(existingPieces) == 0 { // new - best, err = m.storage.FindBestAllocStorage(sectorbuilder.FTUnsealed, true) + best, err = m.index.StorageBestAlloc(ctx, sectorbuilder.FTUnsealed, true) } else { // append to existing - best, err = m.storage.FindSector(sector, sectorbuilder.FTUnsealed) + best, err = m.index.StorageFindSector(ctx, sector, sectorbuilder.FTUnsealed) } if err != nil { return abi.PieceInfo{}, xerrors.Errorf("finding sector path: %w", err) @@ -207,7 +206,7 @@ func (m *Manager) AddPiece(ctx context.Context, sector abi.SectorID, existingPie func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) { // TODO: also consider where the unsealed data sits - best, err := m.storage.FindBestAllocStorage(sectorbuilder.FTCache|sectorbuilder.FTSealed, true) + best, err := m.index.StorageBestAlloc(ctx, sectorbuilder.FTCache|sectorbuilder.FTSealed, true) if err != nil { return nil, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -222,7 +221,7 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticke func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.PreCommit1Out) (cids storage2.SectorCids, err error) { // TODO: allow workers to fetch the sectors - best, err := m.storage.FindSector(sector, sectorbuilder.FTCache|sectorbuilder.FTSealed) + best, err := m.index.StorageFindSector(ctx, sector, sectorbuilder.FTCache|sectorbuilder.FTSealed) if err != nil { return storage2.SectorCids{}, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -235,7 +234,7 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase } func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage2.SectorCids) (output storage2.Commit1Out, err error) { - best, err := m.storage.FindSector(sector, sectorbuilder.FTCache|sectorbuilder.FTSealed) + best, err := m.index.StorageFindSector(ctx, sector, sectorbuilder.FTCache|sectorbuilder.FTSealed) if err != nil { return nil, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -265,7 +264,7 @@ func (m *Manager) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Ou } func (m *Manager) FinalizeSector(ctx context.Context, sector abi.SectorID) error { - best, err := m.storage.FindSector(sector, sectorbuilder.FTCache|sectorbuilder.FTSealed|sectorbuilder.FTUnsealed) + best, err := m.index.StorageFindSector(ctx, sector, sectorbuilder.FTCache|sectorbuilder.FTSealed|sectorbuilder.FTUnsealed) if err != nil { return xerrors.Errorf("finding sealed sector: %w", err) } diff --git a/storage/sealmgr/advmgr/worker_local.go b/storage/sealmgr/advmgr/worker_local.go index 358ee33d2..1579f9b07 100644 --- a/storage/sealmgr/advmgr/worker_local.go +++ b/storage/sealmgr/advmgr/worker_local.go @@ -142,8 +142,8 @@ func (l *LocalWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, }, nil } -func (l *LocalWorker) Paths(context.Context) ([]stores.StoragePath, error) { - return l.localStore.Local(), nil +func (l *LocalWorker) Paths(ctx context.Context) ([]stores.StoragePath, error) { + return l.localStore.Local(ctx) } var _ Worker = &LocalWorker{} diff --git a/storage/sealmgr/stores/index.go b/storage/sealmgr/stores/index.go index db4769989..d594cf82e 100644 --- a/storage/sealmgr/stores/index.go +++ b/storage/sealmgr/stores/index.go @@ -2,14 +2,17 @@ package stores import ( "context" + "math/bits" "net/url" gopath "path" + "sort" "sync" "golang.org/x/xerrors" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" ) @@ -19,19 +22,23 @@ import ( type ID string type StorageInfo struct { - ID ID - URLs []string // TODO: Support non-http transports - Cost int + ID ID + URLs []string // TODO: Support non-http transports + Weight uint64 CanSeal bool CanStore bool } type SectorIndex interface { // part of storage-miner api - StorageAttach(context.Context, StorageInfo) error + StorageAttach(context.Context, StorageInfo, FsStat) error + StorageInfo(context.Context, ID) (StorageInfo, error) + // TODO: StorageUpdateStats(FsStat) StorageDeclareSector(ctx context.Context, storageId ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error StorageFindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]StorageInfo, error) + + StorageBestAlloc(ctx context.Context, allocate sectorbuilder.SectorFileType, sealing bool) ([]StorageInfo, error) } type Decl struct { @@ -39,17 +46,22 @@ type Decl struct { sectorbuilder.SectorFileType } +type storageEntry struct { + info *StorageInfo + fsi FsStat +} + type Index struct { - lk sync.Mutex + lk sync.RWMutex sectors map[Decl][]ID - stores map[ID]*StorageInfo + stores map[ID]*storageEntry } func NewIndex() *Index { return &Index{ sectors: map[Decl][]ID{}, - stores: map[ID]*StorageInfo{}, + stores: map[ID]*storageEntry{}, } } @@ -79,7 +91,7 @@ func (i *Index) StorageList(ctx context.Context) (map[ID][]Decl, error) { return out, nil } -func (i *Index) StorageAttach(ctx context.Context, si StorageInfo) error { +func (i *Index) StorageAttach(ctx context.Context, si StorageInfo, st FsStat) error { i.lk.Lock() defer i.lk.Unlock() @@ -92,10 +104,13 @@ func (i *Index) StorageAttach(ctx context.Context, si StorageInfo) error { } } - i.stores[si.ID].URLs = append(i.stores[si.ID].URLs, si.URLs...) + i.stores[si.ID].info.URLs = append(i.stores[si.ID].info.URLs, si.URLs...) return nil } - i.stores[si.ID] = &si + i.stores[si.ID] = &storageEntry{ + info: &si, + fsi: st, + } return nil } @@ -124,8 +139,12 @@ func (i *Index) StorageDeclareSector(ctx context.Context, storageId ID, s abi.Se } func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sectorbuilder.SectorFileType) ([]StorageInfo, error) { - i.lk.Lock() - defer i.lk.Unlock() + if bits.OnesCount(uint(ft)) != 1 { + return nil, xerrors.Errorf("findSector only works for a single file type") + } + + i.lk.RLock() + defer i.lk.RUnlock() storageIDs := i.sectors[Decl{s, ft}] out := make([]StorageInfo, len(storageIDs)) @@ -137,8 +156,8 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sector continue } - urls := make([]string, len(st.URLs)) - for k, u := range st.URLs { + urls := make([]string, len(st.info.URLs)) + for k, u := range st.info.URLs { rl, err := url.Parse(u) if err != nil { return nil, xerrors.Errorf("failed to parse url: %w", err) @@ -151,9 +170,9 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sector out[j] = StorageInfo{ ID: id, URLs: nil, - Cost: st.Cost, - CanSeal: st.CanSeal, - CanStore: st.CanStore, + Weight: st.info.Weight, + CanSeal: st.info.CanSeal, + CanStore: st.info.CanStore, } } @@ -161,36 +180,65 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sector } func (i *Index) StorageInfo(ctx context.Context, id ID) (StorageInfo, error) { + i.lk.RLock() + defer i.lk.RUnlock() + si, found := i.stores[id] if !found { return StorageInfo{}, xerrors.Errorf("sector store not found") } - return *si, nil + return *si.info, nil } -func DeclareLocalStorage(ctx context.Context, idx SectorIndex, localStore *Local, urls []string, cost int) error { - for _, path := range localStore.Local() { - err := idx.StorageAttach(ctx, StorageInfo{ - ID: path.ID, - URLs: urls, - Cost: cost, - CanSeal: path.CanSeal, - CanStore: path.CanStore, - }) - if err != nil { - log.Errorf("attaching local storage to remote: %+v", err) +func (i *Index) StorageBestAlloc(ctx context.Context, allocate sectorbuilder.SectorFileType, sealing bool) ([]StorageInfo, error) { + i.lk.RLock() + defer i.lk.RUnlock() + + var candidates []storageEntry + + for _, p := range i.stores { + if sealing && !p.info.CanSeal { + log.Debugf("alloc: not considering %s; can't seal", p.info.ID) + continue + } + if !sealing && !p.info.CanStore { + log.Debugf("alloc: not considering %s; can't store", p.info.ID) continue } - for id, fileType := range localStore.List(path.ID) { - if err := idx.StorageDeclareSector(ctx, path.ID, id, fileType); err != nil { - log.Errorf("declaring sector: %+v") - } - } + // TODO: filter out of space + + candidates = append(candidates, *p) } - return nil + if len(candidates) == 0 { + return nil, xerrors.New("no good path found") + } + + sort.Slice(candidates, func(i, j int) bool { + iw := big.Mul(big.NewInt(int64(candidates[i].fsi.Free)), big.NewInt(int64(candidates[i].info.Weight))) + jw := big.Mul(big.NewInt(int64(candidates[j].fsi.Free)), big.NewInt(int64(candidates[j].info.Weight))) + + return iw.GreaterThan(jw) + }) + + out := make([]StorageInfo, len(candidates)) + for i, candidate := range candidates { + out[i] = *candidate.info + } + + return out, nil +} + +func (i *Index) FindSector(id abi.SectorID, typ sectorbuilder.SectorFileType) ([]ID, error) { + i.lk.RLock() + defer i.lk.RUnlock() + + return i.sectors[Decl{ + SectorID: id, + SectorFileType: typ, + }], nil } var _ SectorIndex = &Index{} diff --git a/storage/sealmgr/stores/interface.go b/storage/sealmgr/stores/interface.go index 6e855307d..67c18b16e 100644 --- a/storage/sealmgr/stores/interface.go +++ b/storage/sealmgr/stores/interface.go @@ -2,6 +2,9 @@ package stores import ( "context" + "syscall" + + "golang.org/x/xerrors" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" @@ -10,3 +13,20 @@ import ( type Store interface { AcquireSector(ctx context.Context, s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (paths sectorbuilder.SectorPaths, stores sectorbuilder.SectorPaths, done func(), err error) } + +type FsStat struct { + Capacity uint64 + Free uint64 // Free to use for sector storage +} + +func Stat(path string) (FsStat, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + return FsStat{}, xerrors.Errorf("statfs: %w", err) + } + + return FsStat{ + Capacity: stat.Blocks * uint64(stat.Bsize), + Free: stat.Bavail * uint64(stat.Bsize), + }, nil +} diff --git a/storage/sealmgr/stores/local.go b/storage/sealmgr/stores/local.go index 4cf3cf565..11f398b19 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sealmgr/stores/local.go @@ -27,7 +27,7 @@ type StoragePath struct { } // [path]/sectorstore.json -type StorageMeta struct { +type LocalStorageMeta struct { ID ID Weight uint64 // 0 = readonly @@ -45,35 +45,40 @@ const MetaFile = "sectorstore.json" var pathTypes = []sectorbuilder.SectorFileType{sectorbuilder.FTUnsealed, sectorbuilder.FTSealed, sectorbuilder.FTCache} type Local struct { - localLk sync.RWMutex localStorage LocalStorage + index SectorIndex + urls []string - paths []*path + paths map[ID]*path + + localLk sync.RWMutex } type path struct { - lk sync.Mutex - - meta StorageMeta - local string - - sectors map[abi.SectorID]sectorbuilder.SectorFileType + local string // absolute local path } -func NewLocal(ls LocalStorage) (*Local, error) { +func NewLocal(ctx context.Context, ls LocalStorage, index SectorIndex, urls []string) (*Local, error) { l := &Local{ localStorage: ls, + index: index, + urls: urls, + + paths: map[ID]*path{}, } - return l, l.open() + return l, l.open(ctx) } -func (st *Local) OpenPath(p string) error { +func (st *Local) OpenPath(ctx context.Context, p string) error { + st.localLk.Lock() + defer st.localLk.Unlock() + mb, err := ioutil.ReadFile(filepath.Join(p, MetaFile)) if err != nil { return xerrors.Errorf("reading storage metadata for %s: %w", p, err) } - var meta StorageMeta + var meta LocalStorageMeta if err := json.Unmarshal(mb, &meta); err != nil { return xerrors.Errorf("unmarshalling storage metadata for %s: %w", p, err) } @@ -81,9 +86,23 @@ func (st *Local) OpenPath(p string) error { // TODO: Check existing / dedupe out := &path{ - meta: meta, - local: p, - sectors: map[abi.SectorID]sectorbuilder.SectorFileType{}, + local: p, + } + + fst, err := Stat(p) + if err != nil { + return err + } + + err = st.index.StorageAttach(ctx, StorageInfo{ + ID: meta.ID, + URLs: st.urls, + Weight: meta.Weight, + CanSeal: meta.CanSeal, + CanStore: meta.CanStore, + }, fst) + if err != nil { + return xerrors.Errorf("declaring storage in index: %w", err) } for _, t := range pathTypes { @@ -105,26 +124,25 @@ func (st *Local) OpenPath(p string) error { return xerrors.Errorf("parse sector id %s: %w", ent.Name(), err) } - out.sectors[sid] |= t + if err := st.index.StorageDeclareSector(ctx, meta.ID, sid, t); err != nil { + return xerrors.Errorf("declare sector %d(t:%d) -> %s: %w", sid, t, meta.ID, err) + } } } - st.paths = append(st.paths, out) + st.paths[meta.ID] = out return nil } -func (st *Local) open() error { - st.localLk.Lock() - defer st.localLk.Unlock() - +func (st *Local) open(ctx context.Context) error { cfg, err := st.localStorage.GetStorage() if err != nil { return xerrors.Errorf("getting local storage config: %w", err) } for _, path := range cfg.StoragePaths { - err := st.OpenPath(path.Path) + err := st.OpenPath(ctx, path.Path) if err != nil { return xerrors.Errorf("opening path %s: %w", path.Path, err) } @@ -148,23 +166,25 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing s continue } - for _, p := range st.paths { - p.lk.Lock() - s, ok := p.sectors[sid] - p.lk.Unlock() + si, err := st.index.StorageFindSector(ctx, sid, fileType) + if err != nil { + log.Warnf("finding existing sector %d(t:%d) failed: %+v", sid, fileType, err) + continue + } + + for _, info := range si { + p, ok := st.paths[info.ID] if !ok { continue } - if s&fileType == 0 { - continue - } - if p.local == "" { + + if p.local == "" { // TODO: can that even be the case? continue } spath := filepath.Join(p.local, fileType.String(), sectorutil.SectorName(sid)) sectorutil.SetPathByType(&out, fileType, spath) - sectorutil.SetPathByType(&storageIDs, fileType, string(p.meta.ID)) + sectorutil.SetPathByType(&storageIDs, fileType, string(info.ID)) existing ^= fileType } @@ -175,27 +195,37 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing s continue } + sis, err := st.index.StorageBestAlloc(ctx, fileType, sealing) + if err != nil { + st.localLk.RUnlock() + return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("finding best storage for allocating : %w", err) + } + var best string var bestID ID - for _, p := range st.paths { - if sealing && !p.meta.CanSeal { - continue - } - if !sealing && !p.meta.CanStore { + for _, si := range sis { + p, ok := st.paths[si.ID] + if !ok { continue } - p.lk.Lock() - p.sectors[sid] |= fileType - p.lk.Unlock() + if p.local == "" { // TODO: can that even be the case? + continue + } + + if sealing && !si.CanSeal { + continue + } + + if !sealing && !si.CanStore { + continue + } // TODO: Check free space - // TODO: Calc weights best = filepath.Join(p.local, fileType.String(), sectorutil.SectorName(sid)) - bestID = p.meta.ID - break // todo: the first path won't always be the best + bestID = si.ID } if best == "" { @@ -211,88 +241,41 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing s return out, storageIDs, st.localLk.RUnlock, nil } -func (st *Local) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sealing bool) ([]StorageMeta, error) { - st.localLk.RLock() - defer st.localLk.RUnlock() - - var out []StorageMeta - - for _, p := range st.paths { - if sealing && !p.meta.CanSeal { - log.Debugf("alloc: not considering %s; can't seal", p.meta.ID) - continue - } - if !sealing && !p.meta.CanStore { - log.Debugf("alloc: not considering %s; can't store", p.meta.ID) - continue - } - - // TODO: filter out of space - - out = append(out, p.meta) - } - - if len(out) == 0 { - return nil, xerrors.New("no good path found") - } - - // todo: sort by some kind of preference - return out, nil -} - -func (st *Local) FindSector(id abi.SectorID, typ sectorbuilder.SectorFileType) ([]StorageMeta, error) { - st.localLk.RLock() - defer st.localLk.RUnlock() - - var out []StorageMeta - for _, p := range st.paths { - p.lk.Lock() - t := p.sectors[id] - if t|typ == 0 { - continue - } - p.lk.Unlock() - out = append(out, p.meta) - } - if len(out) == 0 { - return nil, xerrors.Errorf("sector %s/s-t0%d-%d not found", typ, id.Miner, id.Number) - } - - return out, nil -} - -func (st *Local) Local() []StoragePath { +func (st *Local) Local(ctx context.Context) ([]StoragePath, error) { st.localLk.RLock() defer st.localLk.RUnlock() var out []StoragePath - for _, p := range st.paths { + for id, p := range st.paths { if p.local == "" { continue } + si, err := st.index.StorageInfo(ctx, id) + if err != nil { + return nil, xerrors.Errorf("get storage info for %s: %w", id, err) + } + out = append(out, StoragePath{ - ID: p.meta.ID, - Weight: p.meta.Weight, + ID: id, + Weight: si.Weight, LocalPath: p.local, - CanSeal: p.meta.CanSeal, - CanStore: p.meta.CanStore, + CanSeal: si.CanSeal, + CanStore: si.CanStore, }) } - return out + return out, nil } -func (st *Local) List(id ID) map[abi.SectorID]sectorbuilder.SectorFileType { - out := map[abi.SectorID]sectorbuilder.SectorFileType{} - for _, p := range st.paths { - if p.meta.ID != id { // TODO: not very efficient - continue - } +func (st *Local) FsStat(id ID) (FsStat, error) { + st.localLk.RLock() + defer st.localLk.RUnlock() - for id, fileType := range p.sectors { - out[id] |= fileType - } + p, ok := st.paths[id] + if !ok { + return FsStat{}, xerrors.Errorf("fsstat: path not found") } - return out + + return Stat(p.local) } diff --git a/storage/sealmgr/stores/remote.go b/storage/sealmgr/stores/remote.go index ad11879c9..fdedb4d52 100644 --- a/storage/sealmgr/stores/remote.go +++ b/storage/sealmgr/stores/remote.go @@ -84,7 +84,7 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType } sort.Slice(si, func(i, j int) bool { - return si[i].Cost < si[j].Cost + return si[i].Weight < si[j].Weight }) apaths, ids, done, err := r.local.AcquireSector(ctx, s, 0, fileType, sealing) @@ -163,14 +163,6 @@ func (r *Remote) fetch(url, outname string) error { } -func (r *Remote) FindBestAllocStorage(allocate sectorbuilder.SectorFileType, sealing bool) ([]StorageMeta, error) { - panic("todo") -} - -func (r *Remote) FindSector(id abi.SectorID, typ sectorbuilder.SectorFileType) ([]StorageMeta, error) { - panic("todo") -} - func mergeDone(a func(), b func()) func() { return func() { a() From 80cca91584de9ef2dd95ba812cbbd8dd6abb331b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 19 Mar 2020 18:11:45 +0100 Subject: [PATCH 19/45] workers: get to executing tasks remotely! --- cmd/lotus-seal-worker/main.go | 6 ++++- node/builder.go | 3 +++ node/options.go | 11 ++++++++- storage/sealmgr/advmgr/manager.go | 14 +++++++++++- storage/sealmgr/advmgr/worker_local.go | 31 +++++++++++++++++--------- storage/sealmgr/stores/http_handler.go | 4 ++-- storage/sealmgr/stores/index.go | 2 +- storage/sealmgr/stores/remote.go | 20 ++++++++--------- 8 files changed, 65 insertions(+), 26 deletions(-) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 15cdd31d7..1fc5cbbfa 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -25,6 +25,7 @@ import ( "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/sealmgr" "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" "github.com/filecoin-project/lotus/storage/sealmgr/stores" ) @@ -222,7 +223,10 @@ var runCmd = &cli.Command{ // Create / expose the worker workerApi := &worker{ - LocalWorker: advmgr.NewLocalWorker(act, spt, remote, localStore, nodeApi), + LocalWorker: advmgr.NewLocalWorker(advmgr.WorkerConfig{ + SealProof: spt, + TaskTypes: []sealmgr.TaskType{sealmgr.TTPreCommit1, sealmgr.TTPreCommit2, sealmgr.TTCommit2}, + }, remote, localStore, nodeApi), } mux := mux.NewRouter() diff --git a/node/builder.go b/node/builder.go index fdd7f2b6f..5804f68b7 100644 --- a/node/builder.go +++ b/node/builder.go @@ -51,6 +51,7 @@ import ( "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/hello" "github.com/filecoin-project/lotus/node/impl" + "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" @@ -263,6 +264,8 @@ func Online() Option { // Storage miner ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner }, + Override(new(api.Common), From(new(common.CommonAPI))), + Override(new(*stores.Index), stores.NewIndex()), Override(new(stores.SectorIndex), From(new(*stores.Index))), Override(new(dtypes.MinerID), modules.MinerID), diff --git a/node/options.go b/node/options.go index a99dc5bbc..c92e209be 100644 --- a/node/options.go +++ b/node/options.go @@ -138,8 +138,17 @@ func as(in interface{}, as interface{}) interface{} { return reflect.MakeFunc(ctype, func(args []reflect.Value) (results []reflect.Value) { outs := reflect.ValueOf(in).Call(args) + out := reflect.New(outType.Elem()) - out.Elem().Set(outs[0]) + if outs[0].Type().AssignableTo(outType.Elem()) { + // Out: Iface = In: *Struct; Out: Iface = In: OtherIface + out.Elem().Set(outs[0]) + } else { + // Out: Iface = &(In: Struct) + t := reflect.New(outs[0].Type()) + t.Elem().Set(outs[0]) + out.Elem().Set(t) + } outs[0] = out.Elem() return outs diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index e3e19fabe..0b9ced284 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -69,7 +69,10 @@ func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Confi m := &Manager{ workers: []Worker{ - //&LocalWorker{scfg: cfg, localStore: lstor, storage: lstor, sindex: sindex}, + NewLocalWorker(WorkerConfig{ + SealProof: cfg.SealProofType, + TaskTypes: []sealmgr.TaskType{sealmgr.TTAddPiece}, + }, stor, lstor, si), }, scfg: cfg, @@ -212,6 +215,9 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticke } candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit1, best) + if len(candidateWorkers) == 0 { + return nil, xerrors.New("no suitable workers found") // TODO: wait? + } // TODO: select(candidateWorkers, ...) // TODO: remove the sectorbuilder abstraction, pass path directly @@ -227,6 +233,9 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase } candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit2, best) + if len(candidateWorkers) == 0 { + return storage2.SectorCids{}, xerrors.New("no suitable workers found") // TODO: wait? + } // TODO: select(candidateWorkers, ...) // TODO: remove the sectorbuilder abstraction, pass path directly @@ -240,6 +249,9 @@ func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket a } candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit2, best) + if len(candidateWorkers) == 0 { + return nil, xerrors.New("no suitable workers found") // TODO: wait? + } // TODO: select(candidateWorkers, ...) // TODO: remove the sectorbuilder abstraction, pass path directly diff --git a/storage/sealmgr/advmgr/worker_local.go b/storage/sealmgr/advmgr/worker_local.go index 1579f9b07..4c7ebbc26 100644 --- a/storage/sealmgr/advmgr/worker_local.go +++ b/storage/sealmgr/advmgr/worker_local.go @@ -4,7 +4,6 @@ import ( "context" "io" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" storage2 "github.com/filecoin-project/specs-storage/storage" @@ -16,26 +15,41 @@ import ( var pathTypes = []sectorbuilder.SectorFileType{sectorbuilder.FTUnsealed, sectorbuilder.FTSealed, sectorbuilder.FTCache} +type WorkerConfig struct { + SealProof abi.RegisteredProof + TaskTypes []sealmgr.TaskType +} + type LocalWorker struct { scfg *sectorbuilder.Config storage stores.Store localStore *stores.Local sindex stores.SectorIndex + + acceptTasks map[sealmgr.TaskType]struct{} } -func NewLocalWorker(ma address.Address, spt abi.RegisteredProof, store stores.Store, local *stores.Local, sindex stores.SectorIndex) *LocalWorker { - ppt, err := spt.RegisteredPoStProof() +func NewLocalWorker(wcfg WorkerConfig, store stores.Store, local *stores.Local, sindex stores.SectorIndex) *LocalWorker { + ppt, err := wcfg.SealProof.RegisteredPoStProof() if err != nil { panic(err) } + + acceptTasks := map[sealmgr.TaskType]struct{}{} + for _, taskType := range wcfg.TaskTypes { + acceptTasks[taskType] = struct{}{} + } + return &LocalWorker{ scfg: §orbuilder.Config{ - SealProofType: spt, + SealProofType: wcfg.SealProof, PoStProofType: ppt, }, storage: store, localStore: local, sindex: sindex, + + acceptTasks: acceptTasks, } } @@ -49,6 +63,8 @@ func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, sector abi. return sectorbuilder.SectorPaths{}, nil, err } + log.Debugf("acquired sector %d (e:%d; a:%d): %v", sector, existing, allocate, paths) + return paths, func() { done() @@ -134,12 +150,7 @@ func (l *LocalWorker) FinalizeSector(ctx context.Context, sector abi.SectorID) e } func (l *LocalWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) { - return map[sealmgr.TaskType]struct{}{ - sealmgr.TTAddPiece: {}, - sealmgr.TTPreCommit1: {}, - sealmgr.TTPreCommit2: {}, - sealmgr.TTCommit2: {}, - }, nil + return l.acceptTasks, nil } func (l *LocalWorker) Paths(ctx context.Context) ([]stores.StoragePath, error) { diff --git a/storage/sealmgr/stores/http_handler.go b/storage/sealmgr/stores/http_handler.go index f1fa42c8e..e50c85e38 100644 --- a/storage/sealmgr/stores/http_handler.go +++ b/storage/sealmgr/stores/http_handler.go @@ -20,10 +20,10 @@ type FetchHandler struct { Store } -func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // /storage/ +func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // /remote/ mux := mux.NewRouter() - mux.HandleFunc("/{type}/{id}", handler.remoteGetSector).Methods("GET") + mux.HandleFunc("/remote/{type}/{id}", handler.remoteGetSector).Methods("GET") log.Infof("SERVEGETREMOTE %s", r.URL) diff --git a/storage/sealmgr/stores/index.go b/storage/sealmgr/stores/index.go index d594cf82e..cccf9134b 100644 --- a/storage/sealmgr/stores/index.go +++ b/storage/sealmgr/stores/index.go @@ -169,7 +169,7 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sector out[j] = StorageInfo{ ID: id, - URLs: nil, + URLs: urls, Weight: st.info.Weight, CanSeal: st.info.CanSeal, CanStore: st.info.CanStore, diff --git a/storage/sealmgr/stores/remote.go b/storage/sealmgr/stores/remote.go index fdedb4d52..b2f7e5f21 100644 --- a/storage/sealmgr/stores/remote.go +++ b/storage/sealmgr/stores/remote.go @@ -20,20 +20,20 @@ import ( ) type Remote struct { - local *Local - remote SectorIndex - auth http.Header + local *Local + index SectorIndex + auth http.Header fetchLk sync.Mutex // TODO: this can be much smarter // TODO: allow multiple parallel fetches // (make sure to not fetch the same sector data twice) } -func NewRemote(local *Local, remote SectorIndex, auth http.Header) *Remote { +func NewRemote(local *Local, index SectorIndex, auth http.Header) *Remote { return &Remote{ - local: local, - remote: remote, - auth: auth, + local: local, + index: index, + auth: auth, } } @@ -69,7 +69,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec sectorutil.SetPathByType(&paths, fileType, ap) sectorutil.SetPathByType(&stores, fileType, string(storageID)) - if err := r.remote.StorageDeclareSector(ctx, storageID, s, fileType); err != nil { + if err := r.index.StorageDeclareSector(ctx, storageID, s, fileType); err != nil { log.Warnf("declaring sector %v in %s failed: %+v", s, storageID, err) } } @@ -78,7 +78,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec } func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, ID, func(), error) { - si, err := r.remote.StorageFindSector(ctx, s, fileType) + si, err := r.index.StorageFindSector(ctx, s, fileType) if err != nil { return "", "", nil, err } @@ -111,7 +111,7 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType } done() - return "", "", nil, xerrors.Errorf("failed to acquire sector %v from remote: %w", s, merr) + return "", "", nil, xerrors.Errorf("failed to acquire sector %v from remote (tried %v): %w", s, si, merr) } func (r *Remote) fetch(url, outname string) error { From 314fba049f7a884d2843154f5ba8c5efe86a9925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 19 Mar 2020 20:17:43 +0100 Subject: [PATCH 20/45] workers: get sectors back to miner process after precommit --- storage/sealing/states.go | 4 ++-- storage/sealmgr/advmgr/manager.go | 7 ++++--- storage/sealmgr/stores/index.go | 28 +++++++++++++++++----------- storage/sealmgr/stores/remote.go | 2 +- storage/sealmgr/task.go | 5 ++++- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/storage/sealing/states.go b/storage/sealing/states.go index a1babd17d..24f44fdf0 100644 --- a/storage/sealing/states.go +++ b/storage/sealing/states.go @@ -72,12 +72,12 @@ func (m *Sealing) handleUnsealed(ctx statemachine.Context, sector SectorInfo) er pc1o, err := m.sealer.SealPreCommit1(ctx.Context(), m.minerSector(sector.SectorID), ticket.Value, sector.pieceInfos()) if err != nil { - return ctx.Send(SectorSealFailed{xerrors.Errorf("seal pre commit failed: %w", err)}) + return ctx.Send(SectorSealFailed{xerrors.Errorf("seal pre commit(1) failed: %w", err)}) } cids, err := m.sealer.SealPreCommit2(ctx.Context(), m.minerSector(sector.SectorID), pc1o) if err != nil { - return ctx.Send(SectorSealFailed{xerrors.Errorf("seal pre commit failed: %w", err)}) + return ctx.Send(SectorSealFailed{xerrors.Errorf("seal pre commit(2) failed: %w", err)}) } return ctx.Send(SectorSealed{ diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 0b9ced284..03a1bc139 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -71,7 +71,7 @@ func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Confi workers: []Worker{ NewLocalWorker(WorkerConfig{ SealProof: cfg.SealProofType, - TaskTypes: []sealmgr.TaskType{sealmgr.TTAddPiece}, + TaskTypes: []sealmgr.TaskType{sealmgr.TTAddPiece, sealmgr.TTCommit1, sealmgr.TTFinalize}, }, stor, lstor, si), }, scfg: cfg, @@ -248,7 +248,7 @@ func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket a return nil, xerrors.Errorf("finding path for sector sealing: %w", err) } - candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit2, best) + candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTCommit1, best) if len(candidateWorkers) == 0 { return nil, xerrors.New("no suitable workers found") // TODO: wait? } @@ -281,8 +281,9 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector abi.SectorID) error return xerrors.Errorf("finding sealed sector: %w", err) } - candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit2, best) // find last worker with the sector + candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTCommit1, best) + // TODO: Remove sector from sealing stores // TODO: Move the sector to long-term storage return candidateWorkers[0].FinalizeSector(ctx, sector) } diff --git a/storage/sealmgr/stores/index.go b/storage/sealmgr/stores/index.go index cccf9134b..054bb9533 100644 --- a/storage/sealmgr/stores/index.go +++ b/storage/sealmgr/stores/index.go @@ -2,7 +2,6 @@ package stores import ( "context" - "math/bits" "net/url" gopath "path" "sort" @@ -139,17 +138,24 @@ func (i *Index) StorageDeclareSector(ctx context.Context, storageId ID, s abi.Se } func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sectorbuilder.SectorFileType) ([]StorageInfo, error) { - if bits.OnesCount(uint(ft)) != 1 { - return nil, xerrors.Errorf("findSector only works for a single file type") - } - i.lk.RLock() defer i.lk.RUnlock() - storageIDs := i.sectors[Decl{s, ft}] - out := make([]StorageInfo, len(storageIDs)) + storageIDs := map[ID]uint64{} - for j, id := range storageIDs { + for _, pathType := range pathTypes { + if ft&pathType == 0 { + continue + } + + for _, id := range i.sectors[Decl{s, ft}] { + storageIDs[id]++ + } + } + + out := make([]StorageInfo, 0, len(storageIDs)) + + for id, n := range storageIDs { st, ok := i.stores[id] if !ok { log.Warnf("storage %s is not present in sector index (referenced by sector %v)", id, s) @@ -167,13 +173,13 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sector urls[k] = rl.String() } - out[j] = StorageInfo{ + out = append(out, StorageInfo{ ID: id, URLs: urls, - Weight: st.info.Weight, + Weight: st.info.Weight * n, // storage with more sector types is better CanSeal: st.info.CanSeal, CanStore: st.info.CanStore, - } + }) } return out, nil diff --git a/storage/sealmgr/stores/remote.go b/storage/sealmgr/stores/remote.go index b2f7e5f21..ecb907cb1 100644 --- a/storage/sealmgr/stores/remote.go +++ b/storage/sealmgr/stores/remote.go @@ -47,7 +47,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec paths, stores, done, err := r.local.AcquireSector(ctx, s, existing, allocate, sealing) if err != nil { - return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, err + return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, xerrors.Errorf("local acquire error: %w", err) } for _, fileType := range pathTypes { diff --git a/storage/sealmgr/task.go b/storage/sealmgr/task.go index 10b804724..fd35a1204 100644 --- a/storage/sealmgr/task.go +++ b/storage/sealmgr/task.go @@ -5,6 +5,9 @@ type TaskType string const ( TTAddPiece TaskType = "seal/v0/addpiece" TTPreCommit1 TaskType = "seal/v0/precommit/1" - TTPreCommit2 TaskType = "seal/v0/precommit/2" // Commit1 is called here too + TTPreCommit2 TaskType = "seal/v0/precommit/2" + TTCommit1 TaskType = "seal/v0/commit/1" // NOTE: We use this to transfer the sector into miner-local storage for now; Don't use on workers! TTCommit2 TaskType = "seal/v0/commit/2" + + TTFinalize TaskType = "seal/v0/finalize" ) From 978a718159655ebbb06fd865230ea04d63e2907a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 19 Mar 2020 20:51:33 +0100 Subject: [PATCH 21/45] storage: storage find command --- api/api_storage.go | 1 + api/apistruct/struct.go | 7 +- cmd/lotus-storage-miner/storage.go | 125 ++++++++++++++++++++++++++++- node/impl/storminer.go | 4 + storage/sealmgr/advmgr/manager.go | 14 ++++ 5 files changed, 148 insertions(+), 3 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 872a8631b..863feed1b 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -109,6 +109,7 @@ type StorageMiner interface { SectorsUpdate(context.Context, abi.SectorNumber, SectorState) error StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) + StorageLocal(ctx context.Context) (map[stores.ID]string, error) // WorkerConnect tells the node to connect to workers RPC WorkerConnect(context.Context, string) error diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index e75a85ad5..132ff0170 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -182,10 +182,11 @@ type StorageMinerStruct struct { SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error `perm:"write"` WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm + StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` + StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, sectorbuilder.SectorFileType) error `perm:"admin"` StorageFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]stores.StorageInfo, error) `perm:"admin"` - StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` StorageBestAlloc func(ctx context.Context, allocate sectorbuilder.SectorFileType, sealing bool) ([]stores.StorageInfo, error) `perm:"admin"` @@ -671,6 +672,10 @@ func (c *StorageMinerStruct) StorageList(ctx context.Context) (map[stores.ID][]s return c.Internal.StorageList(ctx) } +func (c *StorageMinerStruct) StorageLocal(ctx context.Context) (map[stores.ID]string, error) { + return c.Internal.StorageLocal(ctx) +} + func (c *StorageMinerStruct) StorageInfo(ctx context.Context, id stores.ID) (stores.StorageInfo, error) { return c.Internal.StorageInfo(ctx, id) } diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index a72da5b52..d022e3929 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -3,15 +3,18 @@ package main import ( "encoding/json" "fmt" + "github.com/filecoin-project/specs-actors/actors/abi" "io/ioutil" "os" "path/filepath" + "strconv" "github.com/google/uuid" "github.com/mitchellh/go-homedir" "golang.org/x/xerrors" "gopkg.in/urfave/cli.v2" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" lcli "github.com/filecoin-project/lotus/cli" @@ -26,6 +29,7 @@ var storageCmd = &cli.Command{ Subcommands: []*cli.Command{ storageAttachCmd, storageListCmd, + storageFindCmd, }, } @@ -109,7 +113,8 @@ var storageAttachCmd = &cli.Command{ } var storageListCmd = &cli.Command{ - Name: "list", + Name: "list", + Usage: "list local storage paths", Action: func(cctx *cli.Context) error { nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { @@ -123,6 +128,11 @@ var storageListCmd = &cli.Command{ return err } + local, err := nodeApi.StorageLocal(ctx) + if err != nil { + return err + } + for id, sectors := range st { var u, s, c int for _, decl := range sectors { @@ -145,8 +155,119 @@ var storageListCmd = &cli.Command{ return err } fmt.Printf("\tSeal: %t; Store: %t; Weight: %d\n", si.CanSeal, si.CanStore, si.Weight) + if localPath, ok := local[id]; ok { + fmt.Printf("\tLocal: %s\n", localPath) + } for _, l := range si.URLs { - fmt.Printf("\tReachable %s\n", l) // TODO; try pinging maybe?? print latency? + fmt.Printf("\tURL: %s\n", l) // TODO; try pinging maybe?? print latency? + } + } + + return nil + }, +} + +type storedSector struct { + store stores.StorageInfo + unsealed, sealed, cache bool +} + +var storageFindCmd = &cli.Command{ + Name: "find", + Usage: "find sector in the storage system", + ArgsUsage: "[sector number]", + Action: func(cctx *cli.Context) error { + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + ctx := lcli.ReqContext(cctx) + + ma, err := nodeApi.ActorAddress(ctx) + if err != nil { + return err + } + + mid, err := address.IDFromAddress(ma) + if err != nil { + return err + } + + if !cctx.Args().Present() { + return xerrors.New("Usage: lotus-storage-miner storage find [sector number]") + } + + snum, err := strconv.ParseUint(cctx.Args().First(), 10, 64) + if err != nil { + return err + } + + sid := abi.SectorID{ + Miner: abi.ActorID(mid), + Number: abi.SectorNumber(snum), + } + + u, err := nodeApi.StorageFindSector(ctx, sid, sectorbuilder.FTUnsealed) + if err != nil { + return xerrors.Errorf("finding unsealed: %w", err) + } + + s, err := nodeApi.StorageFindSector(ctx, sid, sectorbuilder.FTSealed) + if err != nil { + return xerrors.Errorf("finding sealed: %w", err) + } + + c, err := nodeApi.StorageFindSector(ctx, sid, sectorbuilder.FTCache) + if err != nil { + return xerrors.Errorf("finding cache: %w", err) + } + + byId := map[stores.ID]*storedSector{} + for _, info := range u { + sts, ok := byId[info.ID] + if !ok { + sts = &storedSector{ + store: info, + } + byId[info.ID] = sts + } + sts.unsealed = true + } + for _, info := range s { + sts, ok := byId[info.ID] + if !ok { + sts = &storedSector{ + store: info, + } + byId[info.ID] = sts + } + sts.sealed = true + } + for _, info := range c { + sts, ok := byId[info.ID] + if !ok { + sts = &storedSector{ + store: info, + } + byId[info.ID] = sts + } + sts.cache = true + } + + local, err := nodeApi.StorageLocal(ctx) + if err != nil { + return err + } + + for id, info := range byId { + fmt.Printf("In %s (Unsealed: %t; Sealed: %t; Cache: %t)\n", id, info.unsealed, info.sealed, info.cache) + fmt.Printf("\tSealing: %t; Storage: %t\n", info.store.CanSeal, info.store.CanSeal) + if localPath, ok := local[id]; ok { + fmt.Printf("\tLocal: %s\n", localPath) + } + for _, l := range info.store.URLs { + fmt.Printf("\tURL: %s\n", l) } } diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 7162022b9..b260096b7 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -123,6 +123,10 @@ func (sm *StorageMinerAPI) SectorsList(context.Context) ([]abi.SectorNumber, err return out, nil } +func (sm *StorageMinerAPI) StorageLocal(ctx context.Context) (map[stores.ID]string, error) { + return sm.StorageMgr.StorageLocal(ctx) +} + func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.SealedRef, error) { // json can't handle cids as map keys out := map[string][]api.SealedRef{} diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 03a1bc139..8eb679bd9 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -288,4 +288,18 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector abi.SectorID) error return candidateWorkers[0].FinalizeSector(ctx, sector) } +func (m *Manager) StorageLocal(ctx context.Context) (map[stores.ID]string, error) { + l, err := m.localStore.Local(ctx) + if err != nil { + return nil, err + } + + out := map[stores.ID]string{} + for _, st := range l { + out[st.ID] = st.LocalPath + } + + return out, nil +} + var _ sealmgr.Manager = &Manager{} From e0c15b24fc70989f0fe92e252242fc23720be283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 20 Mar 2020 01:20:01 +0100 Subject: [PATCH 22/45] storageminer: Fix preseal meta import for genesis miners --- cmd/lotus-storage-miner/init.go | 38 +++++++++++++++++++++++------- cmd/lotus-storage-miner/storage.go | 6 +++-- storage/sealing/fsm.go | 2 +- storage/sealmgr/stores/index.go | 2 +- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 145487d58..94c8ee0c5 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -174,7 +174,7 @@ var initCmd = &cli.Command{ var localPaths []config.LocalPath if pssb := cctx.StringSlice("pre-sealed-sectors"); len(pssb) != 0 { - log.Infof("Setting up storage config with presealed sector: %v", pssb) + log.Infof("Setting up storage config with presealed sectors: %v", pssb) for _, psp := range pssb { psp, err := homedir.Expand(psp) @@ -249,11 +249,16 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string, return xerrors.Errorf("reading preseal metadata: %w", err) } - meta := genesis.Miner{} - if err := json.Unmarshal(b, &meta); err != nil { + psm := map[string]genesis.Miner{} + if err := json.Unmarshal(b, &psm); err != nil { return xerrors.Errorf("unmarshaling preseal metadata: %w", err) } + meta, ok := psm[maddr.String()] + if !ok { + return xerrors.Errorf("preseal file didn't contain metadata for miner %s", maddr) + } + maxSectorID := abi.SectorNumber(0) for _, sector := range meta.Sectors { sectorKey := datastore.NewKey(sealing.SectorStorePrefix).ChildString(fmt.Sprint(sector.SectorID)) @@ -327,6 +332,8 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, metadata string, }*/ } + log.Infof("Setting next sector ID to %d", maxSectorID+1) + buf := make([]byte, binary.MaxVarintLen64) size := binary.PutUvarint(buf, uint64(maxSectorID+1)) return mds.Put(datastore.NewKey("/storage/nextid"), buf[:size]) @@ -412,17 +419,30 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return xerrors.Errorf("failed to start up genesis miner: %w", err) } - defer func() { - if err := m.Unregister(ctx, a); err != nil { - log.Error("failed to shut down storage miner: ", err) - } - }() + cerr := configureStorageMiner(ctx, api, a, peerid) - if err := configureStorageMiner(ctx, api, a, peerid); err != nil { + if err := m.Unregister(ctx, a); err != nil { + log.Error("failed to shut down storage miner: ", err) + } + + if cerr != nil { return xerrors.Errorf("failed to configure storage miner: %w", err) } } + if pssb := cctx.String("pre-sealed-metadata"); pssb != "" { + pssb, err := homedir.Expand(pssb) + if err != nil { + return err + } + + log.Infof("Importing pre-sealed sector metadata for %s", a) + + if err := migratePreSealMeta(ctx, api, pssb, a, mds); err != nil { + return xerrors.Errorf("migrating presealed sector metadata: %w", err) + } + } + return nil } diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index d022e3929..15de84b7f 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "fmt" - "github.com/filecoin-project/specs-actors/actors/abi" "io/ioutil" "os" "path/filepath" @@ -16,6 +15,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/specs-actors/actors/abi" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/storage/sealmgr/stores" @@ -264,7 +264,9 @@ var storageFindCmd = &cli.Command{ fmt.Printf("In %s (Unsealed: %t; Sealed: %t; Cache: %t)\n", id, info.unsealed, info.sealed, info.cache) fmt.Printf("\tSealing: %t; Storage: %t\n", info.store.CanSeal, info.store.CanSeal) if localPath, ok := local[id]; ok { - fmt.Printf("\tLocal: %s\n", localPath) + fmt.Printf("\tLocal (%s)\n", localPath) + } else { + fmt.Printf("\tRemote\n") } for _, l := range info.store.URLs { fmt.Printf("\tURL: %s\n", l) diff --git a/storage/sealing/fsm.go b/storage/sealing/fsm.go index 7bfb0c340..bd3031aca 100644 --- a/storage/sealing/fsm.go +++ b/storage/sealing/fsm.go @@ -85,7 +85,7 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta for _, event := range events { l := Log{ Timestamp: uint64(time.Now().Unix()), - Message: fmt.Sprintf("%+v", event), + Message: fmt.Sprintf("%s", event), Kind: fmt.Sprintf("event;%T", event.User), } diff --git a/storage/sealmgr/stores/index.go b/storage/sealmgr/stores/index.go index 054bb9533..5c8fa9dfd 100644 --- a/storage/sealmgr/stores/index.go +++ b/storage/sealmgr/stores/index.go @@ -148,7 +148,7 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sector continue } - for _, id := range i.sectors[Decl{s, ft}] { + for _, id := range i.sectors[Decl{s, pathType}] { storageIDs[id]++ } } From 076cc428af17fb99c0f5097ae654e6669b39478d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 20 Mar 2020 01:55:49 +0100 Subject: [PATCH 23/45] workers: Fix moving storage around in later steps --- api/apistruct/struct.go | 6 +++--- cmd/lotus-storage-miner/storage.go | 6 +++--- storage/sealmgr/advmgr/manager.go | 8 ++++---- storage/sealmgr/stores/index.go | 31 ++++++++++++++++++++++++++++-- storage/sealmgr/stores/local.go | 2 +- storage/sealmgr/stores/remote.go | 2 +- 6 files changed, 41 insertions(+), 14 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 132ff0170..eb833db86 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -186,7 +186,7 @@ type StorageMinerStruct struct { StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, sectorbuilder.SectorFileType) error `perm:"admin"` - StorageFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]stores.StorageInfo, error) `perm:"admin"` + StorageFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType, bool) ([]stores.StorageInfo, error) `perm:"admin"` StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` StorageBestAlloc func(ctx context.Context, allocate sectorbuilder.SectorFileType, sealing bool) ([]stores.StorageInfo, error) `perm:"admin"` @@ -664,8 +664,8 @@ func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId return c.Internal.StorageDeclareSector(ctx, storageId, s, ft) } -func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType) ([]stores.StorageInfo, error) { - return c.Internal.StorageFindSector(ctx, si, types) +func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType, allowFetch bool) ([]stores.StorageInfo, error) { + return c.Internal.StorageFindSector(ctx, si, types, allowFetch) } func (c *StorageMinerStruct) StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) { diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 15de84b7f..7c84eb10c 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -208,17 +208,17 @@ var storageFindCmd = &cli.Command{ Number: abi.SectorNumber(snum), } - u, err := nodeApi.StorageFindSector(ctx, sid, sectorbuilder.FTUnsealed) + u, err := nodeApi.StorageFindSector(ctx, sid, sectorbuilder.FTUnsealed, false) if err != nil { return xerrors.Errorf("finding unsealed: %w", err) } - s, err := nodeApi.StorageFindSector(ctx, sid, sectorbuilder.FTSealed) + s, err := nodeApi.StorageFindSector(ctx, sid, sectorbuilder.FTSealed, false) if err != nil { return xerrors.Errorf("finding sealed: %w", err) } - c, err := nodeApi.StorageFindSector(ctx, sid, sectorbuilder.FTCache) + c, err := nodeApi.StorageFindSector(ctx, sid, sectorbuilder.FTCache, false) if err != nil { return xerrors.Errorf("finding cache: %w", err) } diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 8eb679bd9..b6228bf01 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -188,7 +188,7 @@ func (m *Manager) AddPiece(ctx context.Context, sector abi.SectorID, existingPie if len(existingPieces) == 0 { // new best, err = m.index.StorageBestAlloc(ctx, sectorbuilder.FTUnsealed, true) } else { // append to existing - best, err = m.index.StorageFindSector(ctx, sector, sectorbuilder.FTUnsealed) + best, err = m.index.StorageFindSector(ctx, sector, sectorbuilder.FTUnsealed, false) } if err != nil { return abi.PieceInfo{}, xerrors.Errorf("finding sector path: %w", err) @@ -227,7 +227,7 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticke func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.PreCommit1Out) (cids storage2.SectorCids, err error) { // TODO: allow workers to fetch the sectors - best, err := m.index.StorageFindSector(ctx, sector, sectorbuilder.FTCache|sectorbuilder.FTSealed) + best, err := m.index.StorageFindSector(ctx, sector, sectorbuilder.FTCache|sectorbuilder.FTSealed, true) if err != nil { return storage2.SectorCids{}, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -243,7 +243,7 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase } func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage2.SectorCids) (output storage2.Commit1Out, err error) { - best, err := m.index.StorageFindSector(ctx, sector, sectorbuilder.FTCache|sectorbuilder.FTSealed) + best, err := m.index.StorageFindSector(ctx, sector, sectorbuilder.FTCache|sectorbuilder.FTSealed, true) if err != nil { return nil, xerrors.Errorf("finding path for sector sealing: %w", err) } @@ -276,7 +276,7 @@ func (m *Manager) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Ou } func (m *Manager) FinalizeSector(ctx context.Context, sector abi.SectorID) error { - best, err := m.index.StorageFindSector(ctx, sector, sectorbuilder.FTCache|sectorbuilder.FTSealed|sectorbuilder.FTUnsealed) + best, err := m.index.StorageFindSector(ctx, sector, sectorbuilder.FTCache|sectorbuilder.FTSealed|sectorbuilder.FTUnsealed, true) if err != nil { return xerrors.Errorf("finding sealed sector: %w", err) } diff --git a/storage/sealmgr/stores/index.go b/storage/sealmgr/stores/index.go index 5c8fa9dfd..fbacf2987 100644 --- a/storage/sealmgr/stores/index.go +++ b/storage/sealmgr/stores/index.go @@ -35,7 +35,7 @@ type SectorIndex interface { // part of storage-miner api // TODO: StorageUpdateStats(FsStat) StorageDeclareSector(ctx context.Context, storageId ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error - StorageFindSector(context.Context, abi.SectorID, sectorbuilder.SectorFileType) ([]StorageInfo, error) + StorageFindSector(ctx context.Context, s abi.SectorID, ft sectorbuilder.SectorFileType, allowFetch bool) ([]StorageInfo, error) StorageBestAlloc(ctx context.Context, allocate sectorbuilder.SectorFileType, sealing bool) ([]StorageInfo, error) } @@ -137,7 +137,7 @@ func (i *Index) StorageDeclareSector(ctx context.Context, storageId ID, s abi.Se return nil } -func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sectorbuilder.SectorFileType) ([]StorageInfo, error) { +func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sectorbuilder.SectorFileType, allowFetch bool) ([]StorageInfo, error) { i.lk.RLock() defer i.lk.RUnlock() @@ -182,6 +182,33 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sector }) } + if allowFetch { + for id, st := range i.stores { + if _, ok := storageIDs[id]; ok { + continue + } + + urls := make([]string, len(st.info.URLs)) + for k, u := range st.info.URLs { + rl, err := url.Parse(u) + if err != nil { + return nil, xerrors.Errorf("failed to parse url: %w", err) + } + + rl.Path = gopath.Join(rl.Path, ft.String(), sectorutil.SectorName(s)) + urls[k] = rl.String() + } + + out = append(out, StorageInfo{ + ID: id, + URLs: urls, + Weight: st.info.Weight * 0, // TODO: something better than just '0' + CanSeal: st.info.CanSeal, + CanStore: st.info.CanStore, + }) + } + } + return out, nil } diff --git a/storage/sealmgr/stores/local.go b/storage/sealmgr/stores/local.go index 11f398b19..2395f840c 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sealmgr/stores/local.go @@ -166,7 +166,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing s continue } - si, err := st.index.StorageFindSector(ctx, sid, fileType) + si, err := st.index.StorageFindSector(ctx, sid, fileType, false) if err != nil { log.Warnf("finding existing sector %d(t:%d) failed: %+v", sid, fileType, err) continue diff --git a/storage/sealmgr/stores/remote.go b/storage/sealmgr/stores/remote.go index ecb907cb1..5866614b0 100644 --- a/storage/sealmgr/stores/remote.go +++ b/storage/sealmgr/stores/remote.go @@ -78,7 +78,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec } func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, ID, func(), error) { - si, err := r.index.StorageFindSector(ctx, s, fileType) + si, err := r.index.StorageFindSector(ctx, s, fileType, false) if err != nil { return "", "", nil, err } From c57c0e7f551a02dd39f4c35332f28cc7d6cbc1a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 20 Mar 2020 23:30:17 +0100 Subject: [PATCH 24/45] workers: Memory based scheduling --- api/api_worker.go | 18 +- api/apistruct/struct.go | 5 + cmd/lotus-bench/main.go | 4 +- go.mod | 1 + go.sum | 11 ++ node/impl/storminer.go | 2 +- storage/sealmgr/advmgr/manager.go | 158 ++++++++++++---- storage/sealmgr/advmgr/resources.go | 135 ++++++++++++++ storage/sealmgr/advmgr/sched.go | 239 +++++++++++++++++++++++++ storage/sealmgr/advmgr/worker_local.go | 38 ++++ 10 files changed, 575 insertions(+), 36 deletions(-) create mode 100644 storage/sealmgr/advmgr/resources.go create mode 100644 storage/sealmgr/advmgr/sched.go diff --git a/api/api_worker.go b/api/api_worker.go index bafd6ca83..7a0004656 100644 --- a/api/api_worker.go +++ b/api/api_worker.go @@ -2,12 +2,12 @@ package api import ( "context" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sealmgr/stores" ) type WorkerApi interface { @@ -16,6 +16,22 @@ type WorkerApi interface { TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) // TaskType -> Weight Paths(context.Context) ([]stores.StoragePath, error) + Info(context.Context) (WorkerInfo, error) storage.Sealer } + +type WorkerResources struct { + MemPhysical uint64 + MemSwap uint64 + + MemReserved uint64 // Used by system / other processes + + GPUs []string +} + +type WorkerInfo struct { + Hostname string + + Resources WorkerResources +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index eb833db86..059f2e79d 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -205,6 +205,7 @@ type WorkerStruct struct { TaskTypes func(context.Context) (map[sealmgr.TaskType]struct{}, error) `perm:"admin"` Paths func(context.Context) ([]stores.StoragePath, error) `perm:"admin"` + Info func(context.Context) (api.WorkerInfo, error) `perm:"admin"` SealPreCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"` SealPreCommit2 func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error) `perm:"admin"` @@ -724,6 +725,10 @@ func (w *WorkerStruct) Paths(ctx context.Context) ([]stores.StoragePath, error) return w.Internal.Paths(ctx) } +func (w *WorkerStruct) Info(ctx context.Context) (api.WorkerInfo, error) { + return w.Internal.Info(ctx) +} + func (w *WorkerStruct) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) { return w.Internal.SealPreCommit1(ctx, sector, ticket, pieces) } diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 54a8a7c36..a4e52a2df 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -137,11 +137,11 @@ func main() { if err != nil { return err } - defer func() { + /*defer func() { if err := os.RemoveAll(tsdir); err != nil { log.Warn("remove all: ", err) } - }() + }()*/ sbdir = tsdir } else { exp, err := homedir.Expand(robench) diff --git a/go.mod b/go.mod index bfa28557b..1233a04d9 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/coreos/go-systemd/v22 v22.0.0 github.com/docker/go-units v0.4.0 + github.com/elastic/go-sysinfo v1.3.0 github.com/filecoin-project/chain-validation v0.0.6-0.20200318065243-0ccb5ec3afc5 github.com/filecoin-project/filecoin-ffi v0.0.0-20200304181354-4446ff8a1bb9 github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be diff --git a/go.sum b/go.sum index 7e4c8b895..2b1d2a531 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,10 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/elastic/go-sysinfo v1.3.0 h1:eb2XFGTMlSwG/yyU9Y8jVAYLIzU2sFzWXwo2gmetyrE= +github.com/elastic/go-sysinfo v1.3.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= +github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7aSY= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/fatih/color v1.8.0 h1:5bzFgL+oy7JITMTxUPJ00n7VxmYd/PdMp5mHFX40/RY= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/filecoin-project/chain-validation v0.0.6-0.20200318065243-0ccb5ec3afc5 h1:cr9+8iX+u9fDV53MWqqZw820EyeWVX+h/HCz56JUWb0= @@ -363,6 +367,8 @@ github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -702,6 +708,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0 h1:c8R11WC8m7KNMkTv/0+Be8vvwo4I3/Ut9AC2FW8fX3U= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= @@ -887,6 +895,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 h1:gSbV7h1NRL2G1xTg/owz62CST1oJBmxy4QpMMregXVQ= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -955,5 +964,7 @@ honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= diff --git a/node/impl/storminer.go b/node/impl/storminer.go index b260096b7..45fb726e2 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -155,7 +155,7 @@ func (sm *StorageMinerAPI) WorkerConnect(ctx context.Context, url string) error log.Infof("Connected to a remote worker at %s", url) - return sm.StorageMgr.AddWorker(w) + return sm.StorageMgr.AddWorker(ctx, w) } func (sm *StorageMinerAPI) MarketImportDealData(ctx context.Context, propCid cid.Cid, path string) error { diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index b6228bf01..3846baa42 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -1,6 +1,7 @@ package advmgr import ( + "container/list" "context" "io" "net/http" @@ -32,11 +33,14 @@ type Worker interface { // Returns paths accessible to the worker Paths(context.Context) ([]stores.StoragePath, error) + + Info(context.Context) (api.WorkerInfo, error) } +type workerID uint64 + type Manager struct { - workers []Worker - scfg *sectorbuilder.Config + scfg *sectorbuilder.Config ls stores.LocalStorage storage *stores.Remote @@ -46,7 +50,16 @@ type Manager struct { storage2.Prover - lk sync.Mutex + workersLk sync.Mutex + nextWorker workerID + workers map[workerID]*workerHandle + + newWorkers chan *workerHandle + schedule chan *workerRequest + workerFree chan workerID + closing chan struct{} + + schedQueue list.List // List[*workerRequest] } func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls URLs, ca api.Common) (*Manager, error) { @@ -68,12 +81,6 @@ func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Confi stor := stores.NewRemote(lstor, si, headers) m := &Manager{ - workers: []Worker{ - NewLocalWorker(WorkerConfig{ - SealProof: cfg.SealProofType, - TaskTypes: []sealmgr.TaskType{sealmgr.TTAddPiece, sealmgr.TTCommit1, sealmgr.TTFinalize}, - }, stor, lstor, si), - }, scfg: cfg, ls: ls, @@ -82,9 +89,27 @@ func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Confi remoteHnd: &stores.FetchHandler{Store: lstor}, index: si, + nextWorker: 0, + workers: map[workerID]*workerHandle{}, + + newWorkers: make(chan *workerHandle), + schedule: make(chan *workerRequest), + workerFree: make(chan workerID), + closing: make(chan struct{}), + Prover: prover, } + go m.runSched() + + err = m.AddWorker(ctx, NewLocalWorker(WorkerConfig{ + SealProof: cfg.SealProofType, + TaskTypes: []sealmgr.TaskType{sealmgr.TTAddPiece, sealmgr.TTCommit1, sealmgr.TTFinalize}, + }, stor, lstor, si)) + if err != nil { + return nil, xerrors.Errorf("adding local worker: %w", err) + } + return m, nil } @@ -106,11 +131,16 @@ func (m *Manager) AddLocalStorage(ctx context.Context, path string) error { return nil } -func (m *Manager) AddWorker(w Worker) error { - m.lk.Lock() - defer m.lk.Unlock() +func (m *Manager) AddWorker(ctx context.Context, w Worker) error { + info, err := w.Info(ctx) + if err != nil { + return xerrors.Errorf("getting worker info: %w", err) + } - m.workers = append(m.workers, w) + m.newWorkers <- &workerHandle{ + w: w, + info: info, + } return nil } @@ -127,12 +157,15 @@ func (m *Manager) ReadPieceFromSealedSector(context.Context, abi.SectorID, secto panic("implement me") } -func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.StorageInfo) ([]Worker, map[int]stores.StorageInfo) { - var workers []Worker - paths := map[int]stores.StorageInfo{} +func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.StorageInfo) ([]workerID, map[workerID]stores.StorageInfo) { + m.workersLk.Lock() + defer m.workersLk.Unlock() + + var workers []workerID + paths := map[workerID]stores.StorageInfo{} for i, worker := range m.workers { - tt, err := worker.TaskTypes(context.TODO()) + tt, err := worker.w.TaskTypes(context.TODO()) if err != nil { log.Errorf("error getting supported worker task types: %+v", err) continue @@ -142,7 +175,7 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.Stor continue } - phs, err := worker.Paths(context.TODO()) + phs, err := worker.w.Paths(context.TODO()) if err != nil { log.Errorf("error getting worker paths: %+v", err) continue @@ -169,12 +202,39 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.Stor } paths[i] = *st - workers = append(workers, worker) + workers = append(workers, i) } return workers, paths } +func (m *Manager) getWorker(ctx context.Context, taskType sealmgr.TaskType, accept []workerID) (Worker, func(), error) { + ret := make(chan workerResponse) + + select { + case m.schedule <- &workerRequest{ + taskType: taskType, + accept: accept, + + cancel: ctx.Done(), + ret: ret, + }: + case <-m.closing: + return nil, nil, xerrors.New("closing") + case <-ctx.Done(): + return nil, nil, ctx.Err() + } + + select { + case resp := <-ret: + return resp.worker, resp.done, resp.err + case <-m.closing: + return nil, nil, xerrors.New("closing") + case <-ctx.Done(): + return nil, nil, ctx.Err() + } +} + func (m *Manager) NewSector(ctx context.Context, sector abi.SectorID) error { log.Warnf("stub NewSector") return nil @@ -201,9 +261,15 @@ func (m *Manager) AddPiece(ctx context.Context, sector abi.SectorID, existingPie return abi.PieceInfo{}, xerrors.New("no worker found") } + worker, done, err := m.getWorker(ctx, sealmgr.TTAddPiece, candidateWorkers) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("scheduling worker: %w", err) + } + defer done() + // TODO: select(candidateWorkers, ...) // TODO: remove the sectorbuilder abstraction, pass path directly - return candidateWorkers[0].AddPiece(ctx, sector, existingPieces, sz, r) + return worker.AddPiece(ctx, sector, existingPieces, sz, r) } func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) { @@ -216,12 +282,18 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticke candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit1, best) if len(candidateWorkers) == 0 { - return nil, xerrors.New("no suitable workers found") // TODO: wait? + return nil, xerrors.New("no suitable workers found") } + worker, done, err := m.getWorker(ctx, sealmgr.TTPreCommit1, candidateWorkers) + if err != nil { + return nil, xerrors.Errorf("scheduling worker: %w", err) + } + defer done() + // TODO: select(candidateWorkers, ...) // TODO: remove the sectorbuilder abstraction, pass path directly - return candidateWorkers[0].SealPreCommit1(ctx, sector, ticket, pieces) + return worker.SealPreCommit1(ctx, sector, ticket, pieces) } func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.PreCommit1Out) (cids storage2.SectorCids, err error) { @@ -234,12 +306,18 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit2, best) if len(candidateWorkers) == 0 { - return storage2.SectorCids{}, xerrors.New("no suitable workers found") // TODO: wait? + return storage2.SectorCids{}, xerrors.New("no suitable workers found") } + worker, done, err := m.getWorker(ctx, sealmgr.TTPreCommit2, candidateWorkers) + if err != nil { + return storage2.SectorCids{}, xerrors.Errorf("scheduling worker: %w", err) + } + defer done() + // TODO: select(candidateWorkers, ...) // TODO: remove the sectorbuilder abstraction, pass path directly - return candidateWorkers[0].SealPreCommit2(ctx, sector, phase1Out) + return worker.SealPreCommit2(ctx, sector, phase1Out) } func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage2.SectorCids) (output storage2.Commit1Out, err error) { @@ -253,14 +331,24 @@ func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket a return nil, xerrors.New("no suitable workers found") // TODO: wait? } + // TODO: Try very hard to execute on worker with access to the sectors + worker, done, err := m.getWorker(ctx, sealmgr.TTCommit1, candidateWorkers) + if err != nil { + return nil, xerrors.Errorf("scheduling worker: %w", err) + } + defer done() + // TODO: select(candidateWorkers, ...) // TODO: remove the sectorbuilder abstraction, pass path directly - return candidateWorkers[0].SealCommit1(ctx, sector, ticket, seed, pieces, cids) + return worker.SealCommit1(ctx, sector, ticket, seed, pieces, cids) } func (m *Manager) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) { - for _, worker := range m.workers { - tt, err := worker.TaskTypes(context.TODO()) + var candidateWorkers []workerID + + m.workersLk.Lock() + for id, worker := range m.workers { + tt, err := worker.w.TaskTypes(ctx) if err != nil { log.Errorf("error getting supported worker task types: %+v", err) continue @@ -268,11 +356,17 @@ func (m *Manager) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Ou if _, ok := tt[sealmgr.TTCommit2]; !ok { continue } - - return worker.SealCommit2(ctx, sector, phase1Out) + candidateWorkers = append(candidateWorkers, id) } + m.workersLk.Unlock() - return nil, xerrors.New("no worker found") + worker, done, err := m.getWorker(ctx, sealmgr.TTCommit2, candidateWorkers) + if err != nil { + return nil, xerrors.Errorf("scheduling worker: %w", err) + } + defer done() + + return worker.SealCommit2(ctx, sector, phase1Out) } func (m *Manager) FinalizeSector(ctx context.Context, sector abi.SectorID) error { @@ -281,11 +375,11 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector abi.SectorID) error return xerrors.Errorf("finding sealed sector: %w", err) } - candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTCommit1, best) + candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTFinalize, best) // TODO: Remove sector from sealing stores // TODO: Move the sector to long-term storage - return candidateWorkers[0].FinalizeSector(ctx, sector) + return m.workers[candidateWorkers[0]].w.FinalizeSector(ctx, sector) } func (m *Manager) StorageLocal(ctx context.Context) (map[stores.ID]string, error) { diff --git a/storage/sealmgr/advmgr/resources.go b/storage/sealmgr/advmgr/resources.go new file mode 100644 index 000000000..a80435ef9 --- /dev/null +++ b/storage/sealmgr/advmgr/resources.go @@ -0,0 +1,135 @@ +package advmgr + +import ( + "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/specs-actors/actors/abi" +) + +var FSOverheadSeal = map[sectorbuilder.SectorFileType]int{ // 10x overheads + sectorbuilder.FTUnsealed: 10, + sectorbuilder.FTSealed: 10, + sectorbuilder.FTCache: 70, // TODO: confirm for 32G +} + +var FsOverheadFinalized = map[sectorbuilder.SectorFileType]int{ + sectorbuilder.FTUnsealed: 10, + sectorbuilder.FTSealed: 10, + sectorbuilder.FTCache: 2, +} + +type Resources struct { + MinMemory uint64 // What Must be in RAM for decent perf + MaxMemory uint64 // Mamory required (swap + ram) + + MultiThread bool + CanGPU bool + + BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads) +} + +const MaxCachingOverhead = 32 << 30 + +var ResourceTable = map[sealmgr.TaskType]map[abi.RegisteredProof]Resources{ + sealmgr.TTAddPiece: { + abi.RegisteredProof_StackedDRG32GiBSeal: Resources{ // This is probably a bit conservative + MaxMemory: 32 << 30, + MinMemory: 32 << 30, + + MultiThread: false, + + BaseMinMemory: 1 << 30, + }, + abi.RegisteredProof_StackedDRG512MiBSeal: Resources{ + MaxMemory: 1 << 30, + MinMemory: 1 << 30, + + MultiThread: false, + + BaseMinMemory: 1 << 30, + }, + }, + sealmgr.TTPreCommit1: { + abi.RegisteredProof_StackedDRG32GiBSeal: Resources{ + MaxMemory: 64 << 30, + MinMemory: 32 << 30, + + MultiThread: false, + + BaseMinMemory: 30 << 30, + }, + abi.RegisteredProof_StackedDRG512MiBSeal: Resources{ + MaxMemory: 3 << 29, // 1.5G + MinMemory: 1 << 30, + + MultiThread: false, + + BaseMinMemory: 1 << 30, + }, + }, + sealmgr.TTPreCommit2: { + abi.RegisteredProof_StackedDRG32GiBSeal: Resources{ + MaxMemory: 96 << 30, + MinMemory: 64 << 30, + + MultiThread: true, + + BaseMinMemory: 30 << 30, + }, + abi.RegisteredProof_StackedDRG512MiBSeal: Resources{ + MaxMemory: 3 << 29, // 1.5G + MinMemory: 1 << 30, + + MultiThread: true, + + BaseMinMemory: 1 << 30, + }, + }, + sealmgr.TTCommit1: { // Very short (~100ms), so params are very light + abi.RegisteredProof_StackedDRG32GiBSeal: Resources{ + MaxMemory: 1 << 30, + MinMemory: 1 << 30, + + MultiThread: false, + + BaseMinMemory: 1 << 30, + }, + abi.RegisteredProof_StackedDRG512MiBSeal: Resources{ + MaxMemory: 1 << 30, + MinMemory: 1 << 30, + + MultiThread: false, + + BaseMinMemory: 1 << 30, + }, + }, + sealmgr.TTCommit2: { // TODO: Measure more accurately + abi.RegisteredProof_StackedDRG32GiBSeal: Resources{ + MaxMemory: 110 << 30, + MinMemory: 60 << 30, + + MultiThread: true, + CanGPU: true, + + BaseMinMemory: 64 << 30, // params + }, + abi.RegisteredProof_StackedDRG512MiBSeal: Resources{ + MaxMemory: 3 << 29, // 1.5G + MinMemory: 1 << 30, + + MultiThread: false, // This is fine + CanGPU: true, + + BaseMinMemory: 10 << 30, + }, + }, +} + +func init() { + // for now we just reuse params for 2kib and 8mib from 512mib + + for taskType := range ResourceTable { + ResourceTable[taskType][abi.RegisteredProof_StackedDRG8MiBSeal] = ResourceTable[taskType][abi.RegisteredProof_StackedDRG512MiBSeal] + ResourceTable[taskType][abi.RegisteredProof_StackedDRG2KiBSeal] = ResourceTable[taskType][abi.RegisteredProof_StackedDRG512MiBSeal] + } +} diff --git a/storage/sealmgr/advmgr/sched.go b/storage/sealmgr/advmgr/sched.go new file mode 100644 index 000000000..6badc7663 --- /dev/null +++ b/storage/sealmgr/advmgr/sched.go @@ -0,0 +1,239 @@ +package advmgr + +import ( + "github.com/filecoin-project/specs-actors/actors/abi" + "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/storage/sealmgr" +) + +const mib = 1 << 20 + +type workerRequest struct { + taskType sealmgr.TaskType + accept []workerID // ordered by preference + + ret chan<- workerResponse + cancel <-chan struct{} +} + +type workerResponse struct { + err error + + worker Worker + done func() +} + +func (r *workerRequest) respond(resp workerResponse) { + select { + case r.ret <- resp: + case <-r.cancel: + log.Warnf("request got cancelled before we could respond") + if resp.done != nil { + resp.done() + } + } +} + +type workerHandle struct { + w Worker + + info api.WorkerInfo + + memUsedMin uint64 + memUsedMax uint64 + gpuUsed bool + cpuUse int // -1 - multicore thing; 0 - free; 1+ - singlecore things +} + +func (m *Manager) runSched() { + for { + select { + case w := <-m.newWorkers: + m.schedNewWorker(w) + case req := <-m.schedule: + resp, err := m.maybeSchedRequest(req) + if err != nil { + req.respond(workerResponse{err: err}) + continue + } + + if resp != nil { + req.respond(*resp) + continue + } + + m.schedQueue.PushBack(req) + case wid := <-m.workerFree: + m.onWorkerFreed(wid) + } + } +} + +func (m *Manager) onWorkerFreed(wid workerID) { + for e := m.schedQueue.Front(); e != nil; e = e.Next() { + req := e.Value.(*workerRequest) + var ok bool + for _, id := range req.accept { + if id == wid { + ok = true + break + } + } + if !ok { + continue + } + + resp, err := m.maybeSchedRequest(req) + if err != nil { + req.respond(workerResponse{err: err}) + continue + } + + if resp != nil { + req.respond(*resp) + + pe := e.Prev() + m.schedQueue.Remove(e) + if pe == nil { + pe = m.schedQueue.Front() + } + e = pe + continue + } + } +} + +func (m *Manager) maybeSchedRequest(req *workerRequest) (*workerResponse, error) { + m.workersLk.Lock() + defer m.workersLk.Unlock() + + tried := 0 + + for _, id := range req.accept { + w, ok := m.workers[id] + if !ok { + log.Warnf("requested worker %d is not in scheduler", id) + } + tried++ + + canDo, err := m.canHandleRequest(id, w, req) + if err != nil { + return nil, err + } + + if !canDo { + continue + } + + return m.makeResponse(id, w, req), nil + } + + if tried == 0 { + return nil, xerrors.New("maybeSchedRequest didn't find any good workers") + } + + return nil, nil // put in waiting queue +} + +func (m *Manager) makeResponse(wid workerID, w *workerHandle, req *workerRequest) *workerResponse { + needRes := ResourceTable[req.taskType][m.scfg.SealProofType] + + w.gpuUsed = needRes.CanGPU + if needRes.MultiThread { + w.cpuUse = -1 + } else { + if w.cpuUse != -1 { + w.cpuUse++ + } else { + log.Warnf("sched: makeResponse for worker %d: worker cpu is in multicore use, but a single core task was scheduled", wid) + } + } + + w.memUsedMin += needRes.MinMemory + w.memUsedMax += needRes.MaxMemory + + return &workerResponse{ + err: nil, + worker: w.w, + done: func() { + m.workersLk.Lock() + + if needRes.CanGPU { + w.gpuUsed = false + } + + if needRes.MultiThread { + w.cpuUse = 0 + } else if w.cpuUse != -1 { + w.cpuUse-- + } + + w.memUsedMin -= needRes.MinMemory + w.memUsedMax -= needRes.MaxMemory + + m.workersLk.Unlock() + + select { + case m.workerFree <- wid: + case <-m.closing: + } + }, + } +} + +func (m *Manager) canHandleRequest(wid workerID, w *workerHandle, req *workerRequest) (bool, error) { + needRes, ok := ResourceTable[req.taskType][m.scfg.SealProofType] + if !ok { + return false, xerrors.Errorf("canHandleRequest: missing ResourceTable entry for %s/%d", req.taskType, m.scfg.SealProofType) + } + + res := w.info.Resources + + // TODO: dedupe needRes.BaseMinMemory per task type (don't add if that task is already running) + minNeedMem := res.MemReserved + w.memUsedMin + needRes.MinMemory + needRes.BaseMinMemory + if minNeedMem > res.MemPhysical { + log.Debugf("sched: not scheduling on worker %d; not enough physical memory - need: %dM, have %dM", wid, minNeedMem/mib, res.MemPhysical/mib) + return false, nil + } + + maxNeedMem := res.MemReserved + w.memUsedMax + needRes.MaxMemory + needRes.BaseMinMemory + if m.scfg.SealProofType == abi.RegisteredProof_StackedDRG32GiBSeal { + maxNeedMem += MaxCachingOverhead + } + if maxNeedMem > res.MemSwap+res.MemPhysical { + log.Debugf("sched: not scheduling on worker %d; not enough virtual memory - need: %dM, have %dM", wid, maxNeedMem/mib, (res.MemSwap+res.MemPhysical)/mib) + return false, nil + } + + if needRes.MultiThread { + if w.cpuUse != 0 { + log.Debugf("sched: not scheduling on worker %d; multicore process needs free CPU", wid) + return false, nil + } + } else { + if w.cpuUse == -1 { + log.Debugf("sched: not scheduling on worker %d; CPU in use by a multicore process", wid) + return false, nil + } + } + + if len(res.GPUs) > 0 && needRes.CanGPU { + if w.gpuUsed { + log.Debugf("sched: not scheduling on worker %d; GPU in use", wid) + return false, nil + } + } + + return true, nil +} + +func (m *Manager) schedNewWorker(w *workerHandle) { + m.workersLk.Lock() + defer m.workersLk.Unlock() + + id := m.nextWorker + m.workers[id] = w + m.nextWorker++ +} diff --git a/storage/sealmgr/advmgr/worker_local.go b/storage/sealmgr/advmgr/worker_local.go index 4c7ebbc26..f62ad58c9 100644 --- a/storage/sealmgr/advmgr/worker_local.go +++ b/storage/sealmgr/advmgr/worker_local.go @@ -3,11 +3,17 @@ package advmgr import ( "context" "io" + "os" + "github.com/elastic/go-sysinfo" + "golang.org/x/xerrors" + + ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" storage2 "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/storage/sealmgr" "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" "github.com/filecoin-project/lotus/storage/sealmgr/stores" @@ -157,4 +163,36 @@ func (l *LocalWorker) Paths(ctx context.Context) ([]stores.StoragePath, error) { return l.localStore.Local(ctx) } +func (l *LocalWorker) Info(context.Context) (api.WorkerInfo, error) { + hostname, err := os.Hostname() // TODO: allow overriding from config + if err != nil { + panic(err) + } + + gpus, err := ffi.GetGPUDevices() + if err != nil { + log.Errorf("getting gpu devices failed: %+v", err) + } + + h, err := sysinfo.Host() + if err != nil { + return api.WorkerInfo{}, xerrors.Errorf("getting host info: %w", err) + } + + mem, err := h.Memory() + if err != nil { + return api.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err) + } + + return api.WorkerInfo{ + Hostname: hostname, + Resources: api.WorkerResources{ + MemPhysical: mem.Total, + MemSwap: mem.VirtualTotal, + MemReserved: mem.VirtualUsed + mem.Total - mem.Available, // TODO: sub this process + GPUs: gpus, + }, + }, nil +} + var _ Worker = &LocalWorker{} From c6f67911de81d4eb0ce22df154ce974d0903c725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 20 Mar 2020 23:39:07 +0100 Subject: [PATCH 25/45] storageminer: Improve storage find cmd --- cmd/lotus-storage-miner/storage.go | 33 ++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 7c84eb10c..0293b4328 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" "strconv" "github.com/google/uuid" @@ -168,7 +169,9 @@ var storageListCmd = &cli.Command{ } type storedSector struct { - store stores.StorageInfo + id stores.ID + store stores.StorageInfo + unsealed, sealed, cache bool } @@ -228,6 +231,7 @@ var storageFindCmd = &cli.Command{ sts, ok := byId[info.ID] if !ok { sts = &storedSector{ + id: info.ID, store: info, } byId[info.ID] = sts @@ -238,6 +242,7 @@ var storageFindCmd = &cli.Command{ sts, ok := byId[info.ID] if !ok { sts = &storedSector{ + id: info.ID, store: info, } byId[info.ID] = sts @@ -248,6 +253,7 @@ var storageFindCmd = &cli.Command{ sts, ok := byId[info.ID] if !ok { sts = &storedSector{ + id: info.ID, store: info, } byId[info.ID] = sts @@ -260,10 +266,29 @@ var storageFindCmd = &cli.Command{ return err } - for id, info := range byId { - fmt.Printf("In %s (Unsealed: %t; Sealed: %t; Cache: %t)\n", id, info.unsealed, info.sealed, info.cache) + var out []*storedSector + for _, sector := range byId { + out = append(out, sector) + } + sort.Slice(out, func(i, j int) bool { + return out[i].id < out[j].id + }) + + for _, info := range out { + var types string + if info.unsealed { + types += "Unsealed, " + } + if info.sealed { + types += "Sealed, " + } + if info.cache { + types += "Cache, " + } + + fmt.Printf("In %s (%s)\n", info.id, types[:len(types)-2]) fmt.Printf("\tSealing: %t; Storage: %t\n", info.store.CanSeal, info.store.CanSeal) - if localPath, ok := local[id]; ok { + if localPath, ok := local[info.id]; ok { fmt.Printf("\tLocal (%s)\n", localPath) } else { fmt.Printf("\tRemote\n") From 709fd034b4494cc9a223a57d7747a5a405b572c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 20 Mar 2020 23:43:20 +0100 Subject: [PATCH 26/45] mod tidy --- go.mod | 1 - go.sum | 6 ------ 2 files changed, 7 deletions(-) diff --git a/go.mod b/go.mod index 349cf048f..3b16a5b64 100644 --- a/go.mod +++ b/go.mod @@ -103,7 +103,6 @@ require ( go.uber.org/multierr v1.4.0 go.uber.org/zap v1.13.0 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 - golang.org/x/tools v0.0.0-20191216173652-a0e659d51361 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 gotest.tools v2.2.0+incompatible diff --git a/go.sum b/go.sum index 16ee4eab3..39eb27928 100644 --- a/go.sum +++ b/go.sum @@ -127,10 +127,6 @@ github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.m github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 h1:eYxi6vI5CyeXD15X1bB3bledDXbqKxqf0wQzTLgwYwA= github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200311224656-7d83652bdbed/go.mod h1:xAd/X905Ncgj8kkHsP2pmQUf6MQT2qJTDcOEfkwCjYc= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200314022627-38af9db49ba2 h1:4RjDynwobd/UYlZUprRg/GMEsMP6fAfVRTXgFs4XNfo= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200314022627-38af9db49ba2/go.mod h1:NcE+iL0bbYnamGmYQgCPVGbSaf8VF2/CLra/61B3I3I= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317165603-bd9e7cb04d81 h1:W5yekTpVTUiB86rSDiZo6rTI3lrLKrsrdY0tx/IqgJA= -github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317165603-bd9e7cb04d81/go.mod h1:3c3MEU9GHLlau37+MmefFNunTo9sVEKfjaJuHBgksdY= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317221918-42574fc2aab9 h1:ROfxm5X9dMATYk6MvNe8WBcL2yaelpgGEvve5CnZ1+g= github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200317221918-42574fc2aab9/go.mod h1:3c3MEU9GHLlau37+MmefFNunTo9sVEKfjaJuHBgksdY= github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 h1:k9qVR9ItcziSB2rxtlkN/MDWNlbsI6yzec+zjUatLW0= @@ -143,8 +139,6 @@ github.com/filecoin-project/specs-actors v0.0.0-20200302223606-0eaf97b10aaf/go.m github.com/filecoin-project/specs-actors v0.0.0-20200306000749-99e98e61e2a0/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/filecoin-project/specs-actors v0.0.0-20200312030511-3f5510bf6130 h1:atiWEDtI/gzSm89fL+NyneLN3eHfBd1QPgOZyXPjA5M= github.com/filecoin-project/specs-actors v0.0.0-20200312030511-3f5510bf6130/go.mod h1:5WngRgTN5Eo4+0SjCBqLzEr2l6Mj45DrP2606gBhqI0= -github.com/filecoin-project/specs-storage v0.0.0-20200303233430-1a5a408f7513 h1:okBx3lPomwDxlPmRvyP078BwivDfdxNUlpCDhDD0ia8= -github.com/filecoin-project/specs-storage v0.0.0-20200303233430-1a5a408f7513/go.mod h1:sC2Ck2l1G8hXI5Do/3sp0yxbMRMnukbFwP9KF1CRFLw= github.com/filecoin-project/specs-storage v0.0.0-20200317133846-063ba163b217 h1:doPA79fSLg5TnY2rJhXs5dIZHP3IoCcIiCLKFGfgrY8= github.com/filecoin-project/specs-storage v0.0.0-20200317133846-063ba163b217/go.mod h1:dUmzHS7izOD6HW3/JpzFrjxnptxbsHXBlO8puK2UzBk= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= From 971fe6fdfded3faf2bde4c1e338374df8eaf8eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 22 Mar 2020 05:27:28 +0100 Subject: [PATCH 27/45] workers: Drop sectors from origin after transfers --- api/apistruct/struct.go | 5 +++ cmd/lotus-seal-worker/main.go | 2 +- storage/sealmgr/advmgr/manager.go | 2 +- storage/sealmgr/stores/http_handler.go | 42 ++++++++++++++++++++----- storage/sealmgr/stores/index.go | 37 +++++++++++++++++++++- storage/sealmgr/stores/local.go | 33 ++++++++++++++++++++ storage/sealmgr/stores/remote.go | 43 ++++++++++++++++++++++---- 7 files changed, 147 insertions(+), 17 deletions(-) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 059f2e79d..10ddfde10 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -186,6 +186,7 @@ type StorageMinerStruct struct { StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, sectorbuilder.SectorFileType) error `perm:"admin"` + StorageDropSector func(context.Context, stores.ID, abi.SectorID, sectorbuilder.SectorFileType) error `perm:"admin"` StorageFindSector func(context.Context, abi.SectorID, sectorbuilder.SectorFileType, bool) ([]stores.StorageInfo, error) `perm:"admin"` StorageInfo func(context.Context, stores.ID) (stores.StorageInfo, error) `perm:"admin"` StorageBestAlloc func(ctx context.Context, allocate sectorbuilder.SectorFileType, sealing bool) ([]stores.StorageInfo, error) `perm:"admin"` @@ -665,6 +666,10 @@ func (c *StorageMinerStruct) StorageDeclareSector(ctx context.Context, storageId return c.Internal.StorageDeclareSector(ctx, storageId, s, ft) } +func (c *StorageMinerStruct) StorageDropSector(ctx context.Context, storageId stores.ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error { + return c.Internal.StorageDropSector(ctx, storageId, s, ft) +} + func (c *StorageMinerStruct) StorageFindSector(ctx context.Context, si abi.SectorID, types sectorbuilder.SectorFileType, allowFetch bool) ([]stores.StorageInfo, error) { return c.Internal.StorageFindSector(ctx, si, types, allowFetch) } diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 1fc5cbbfa..699a2bb04 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -237,7 +237,7 @@ var runCmd = &cli.Command{ rpcServer.Register("Filecoin", apistruct.PermissionedWorkerAPI(workerApi)) mux.Handle("/rpc/v0", rpcServer) - mux.PathPrefix("/remote").HandlerFunc((&stores.FetchHandler{Store: localStore}).ServeHTTP) + mux.PathPrefix("/remote").HandlerFunc((&stores.FetchHandler{Local: localStore}).ServeHTTP) mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof ah := &auth.Handler{ diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 3846baa42..153ae661f 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -86,7 +86,7 @@ func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Confi ls: ls, storage: stor, localStore: lstor, - remoteHnd: &stores.FetchHandler{Store: lstor}, + remoteHnd: &stores.FetchHandler{Local: lstor}, index: si, nextWorker: 0, diff --git a/storage/sealmgr/stores/http_handler.go b/storage/sealmgr/stores/http_handler.go index e50c85e38..f4e519cc6 100644 --- a/storage/sealmgr/stores/http_handler.go +++ b/storage/sealmgr/stores/http_handler.go @@ -17,35 +17,37 @@ import ( var log = logging.Logger("stores") type FetchHandler struct { - Store + *Local } func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // /remote/ mux := mux.NewRouter() mux.HandleFunc("/remote/{type}/{id}", handler.remoteGetSector).Methods("GET") - - log.Infof("SERVEGETREMOTE %s", r.URL) + mux.HandleFunc("/remote/{type}/{id}", handler.remoteDeleteSector).Methods("DELETE") mux.ServeHTTP(w, r) } func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Request) { + log.Infof("SERVE GET %s", r.URL) vars := mux.Vars(r) id, err := sectorutil.ParseSectorID(vars["id"]) if err != nil { - log.Error(err) + log.Error("%+v", err) w.WriteHeader(500) return } ft, err := ftFromString(vars["type"]) if err != nil { + log.Error("%+v", err) return } - paths, _, done, err := handler.Store.AcquireSector(r.Context(), id, ft, 0, false) + paths, _, done, err := handler.Local.AcquireSector(r.Context(), id, ft, 0, false) if err != nil { + log.Error("%+v", err) return } defer done() @@ -59,7 +61,7 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ stat, err := os.Stat(path) if err != nil { - log.Error(err) + log.Error("%+v", err) w.WriteHeader(500) return } @@ -73,14 +75,38 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ w.Header().Set("Content-Type", "application/octet-stream") } if err != nil { - log.Error(err) + log.Error("%+v", err) w.WriteHeader(500) return } w.WriteHeader(200) if _, err := io.Copy(w, rd); err != nil { // TODO: default 32k buf may be too small - log.Error(err) + log.Error("%+v", err) + return + } +} + +func (handler *FetchHandler) remoteDeleteSector(w http.ResponseWriter, r *http.Request) { + log.Infof("SERVE DELETE %s", r.URL) + vars := mux.Vars(r) + + id, err := sectorutil.ParseSectorID(vars["id"]) + if err != nil { + log.Error("%+v", err) + w.WriteHeader(500) + return + } + + ft, err := ftFromString(vars["type"]) + if err != nil { + log.Error("%+v", err) + return + } + + if err := handler.delete(r.Context(), id, ft); err != nil { + log.Error("%+v", err) + w.WriteHeader(500) return } } diff --git a/storage/sealmgr/stores/index.go b/storage/sealmgr/stores/index.go index fbacf2987..d77c1b811 100644 --- a/storage/sealmgr/stores/index.go +++ b/storage/sealmgr/stores/index.go @@ -35,7 +35,8 @@ type SectorIndex interface { // part of storage-miner api // TODO: StorageUpdateStats(FsStat) StorageDeclareSector(ctx context.Context, storageId ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error - StorageFindSector(ctx context.Context, s abi.SectorID, ft sectorbuilder.SectorFileType, allowFetch bool) ([]StorageInfo, error) + StorageDropSector(ctx context.Context, storageId ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error + StorageFindSector(ctx context.Context, sector abi.SectorID, ft sectorbuilder.SectorFileType, allowFetch bool) ([]StorageInfo, error) StorageBestAlloc(ctx context.Context, allocate sectorbuilder.SectorFileType, sealing bool) ([]StorageInfo, error) } @@ -137,6 +138,40 @@ func (i *Index) StorageDeclareSector(ctx context.Context, storageId ID, s abi.Se return nil } +func (i *Index) StorageDropSector(ctx context.Context, storageId ID, s abi.SectorID, ft sectorbuilder.SectorFileType) error { + i.lk.Lock() + defer i.lk.Unlock() + + for _, fileType := range pathTypes { + if fileType&ft == 0 { + continue + } + + d := Decl{s, fileType} + + if len(i.sectors[d]) == 0 { + return nil + } + + rewritten := make([]ID, 0, len(i.sectors[d])-1) + for _, sid := range i.sectors[d] { + if sid == storageId { + continue + } + + rewritten = append(rewritten, sid) + } + if len(rewritten) == 0 { + delete(i.sectors, d) + return nil + } + + i.sectors[d] = rewritten + } + + return nil +} + func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft sectorbuilder.SectorFileType, allowFetch bool) ([]StorageInfo, error) { i.lk.RLock() defer i.lk.RUnlock() diff --git a/storage/sealmgr/stores/local.go b/storage/sealmgr/stores/local.go index 2395f840c..304c64ba4 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sealmgr/stores/local.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "io/ioutil" + "math/bits" "os" "path/filepath" "sync" @@ -187,6 +188,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing s sectorutil.SetPathByType(&storageIDs, fileType, string(info.ID)) existing ^= fileType + break } } @@ -268,6 +270,37 @@ func (st *Local) Local(ctx context.Context) ([]StoragePath, error) { return out, nil } +func (st *Local) delete(ctx context.Context, sid abi.SectorID, typ sectorbuilder.SectorFileType) error { + if bits.OnesCount(uint(typ)) != 1 { + return xerrors.New("delete expects one file type") + } + + si, err := st.index.StorageFindSector(ctx, sid, typ, false) + if err != nil { + return xerrors.Errorf("finding existing sector %d(t:%d) failed: %w", sid, typ, err) + } + + for _, info := range si { + p, ok := st.paths[info.ID] + if !ok { + continue + } + + if p.local == "" { // TODO: can that even be the case? + continue + } + + spath := filepath.Join(p.local, typ.String(), sectorutil.SectorName(sid)) + log.Infof("remove %s", spath) + + if err := os.RemoveAll(spath); err != nil { + log.Errorf("removing sector (%v) from %s: %+v", sid, spath, err) + } + } + + return nil +} + func (st *Local) FsStat(id ID) (FsStat, error) { st.localLk.RLock() defer st.localLk.RUnlock() diff --git a/storage/sealmgr/stores/remote.go b/storage/sealmgr/stores/remote.go index 5866614b0..fca680e96 100644 --- a/storage/sealmgr/stores/remote.go +++ b/storage/sealmgr/stores/remote.go @@ -59,7 +59,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec continue } - ap, storageID, rdone, err := r.acquireFromRemote(ctx, s, fileType, sealing) + ap, storageID, url, foundIn, rdone, err := r.acquireFromRemote(ctx, s, fileType, sealing) if err != nil { done() return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, err @@ -71,16 +71,26 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec if err := r.index.StorageDeclareSector(ctx, storageID, s, fileType); err != nil { log.Warnf("declaring sector %v in %s failed: %+v", s, storageID, err) + continue + } + + // TODO: some way to allow having duplicated sectors in the system for perf + if err := r.index.StorageDropSector(ctx, foundIn, s, fileType); err != nil { + log.Warnf("dropping sector %v from %s from sector index failed: %+v", s, storageID, err) + } + + if err := r.deleteFromRemote(url); err != nil { + log.Warnf("deleting sector %v from %s (delete %s): %+v", s, storageID, url, err) } } return paths, stores, done, nil } -func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, ID, func(), error) { +func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, ID, string, ID, func(), error) { si, err := r.index.StorageFindSector(ctx, s, fileType, false) if err != nil { - return "", "", nil, err + return "", "", "", "", nil, err } sort.Slice(si, func(i, j int) bool { @@ -89,7 +99,7 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType apaths, ids, done, err := r.local.AcquireSector(ctx, s, 0, fileType, sealing) if err != nil { - return "", "", nil, xerrors.Errorf("allocate local sector for fetching: %w", err) + return "", "", "", "", nil, xerrors.Errorf("allocate local sector for fetching: %w", err) } dest := sectorutil.PathByType(apaths, fileType) storageID := sectorutil.PathByType(ids, fileType) @@ -106,12 +116,12 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType if merr != nil { log.Warnw("acquireFromRemote encountered errors when fetching sector from remote", "errors", merr) } - return dest, ID(storageID), done, nil + return dest, ID(storageID), url, info.ID, done, nil } } done() - return "", "", nil, xerrors.Errorf("failed to acquire sector %v from remote (tried %v): %w", s, si, merr) + return "", "", "", "", nil, xerrors.Errorf("failed to acquire sector %v from remote (tried %v): %w", s, si, merr) } func (r *Remote) fetch(url, outname string) error { @@ -160,7 +170,28 @@ func (r *Remote) fetch(url, outname string) error { default: return xerrors.Errorf("unknown content type: '%s'", mediatype) } +} +func (r *Remote) deleteFromRemote(url string) error { + log.Infof("Delete %s", url) + + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return xerrors.Errorf("request: %w", err) + } + req.Header = r.auth + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return xerrors.Errorf("do request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return xerrors.Errorf("non-200 code: %d", resp.StatusCode) + } + + return nil } func mergeDone(a func(), b func()) func() { From 66d71d9974ebd20a4b868c7b10fb209ac2cb36c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 22 Mar 2020 21:44:27 +0100 Subject: [PATCH 28/45] sealing: Improve sector log --- storage/sealing/fsm.go | 12 +++++-- storage/sealing/fsm_events.go | 60 +++++++++++++++++++---------------- storage/sealing/fsm_test.go | 4 +-- storage/sealing/sealing.go | 6 ++-- storage/sealing/states.go | 16 +++++----- 5 files changed, 55 insertions(+), 43 deletions(-) diff --git a/storage/sealing/fsm.go b/storage/sealing/fsm.go index bd3031aca..89d27178c 100644 --- a/storage/sealing/fsm.go +++ b/storage/sealing/fsm.go @@ -2,6 +2,7 @@ package sealing import ( "context" + "encoding/json" "fmt" "reflect" "time" @@ -82,10 +83,17 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta ///// // First process all events + for _, event := range events { + e, err := json.Marshal(event) + if err != nil { + log.Errorf("marshaling event for logging: %+v", err) + continue + } + l := Log{ Timestamp: uint64(time.Now().Unix()), - Message: fmt.Sprintf("%s", event), + Message: string(e), Kind: fmt.Sprintf("event;%T", event.User), } @@ -201,7 +209,7 @@ func planCommitting(events []statemachine.Event, state *SectorInfo) error { e.apply(state) state.State = api.CommitWait case SectorSeedReady: // seed changed :/ - if e.seed.Equals(&state.Seed) { + if e.Seed.Equals(&state.Seed) { log.Warnf("planCommitting: got SectorSeedReady, but the seed didn't change") continue // or it didn't! } diff --git a/storage/sealing/fsm_events.go b/storage/sealing/fsm_events.go index e809685eb..d358ba360 100644 --- a/storage/sealing/fsm_events.go +++ b/storage/sealing/fsm_events.go @@ -3,6 +3,7 @@ package sealing import ( "github.com/filecoin-project/specs-actors/actors/abi" "github.com/ipfs/go-cid" + "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" ) @@ -25,6 +26,7 @@ type SectorRestart struct{} func (evt SectorRestart) applyGlobal(*SectorInfo) bool { return false } type SectorFatalError struct{ error } +func (evt SectorFatalError) FormatError(xerrors.Printer) (next error) {return evt.error} func (evt SectorFatalError) applyGlobal(state *SectorInfo) bool { log.Errorf("Fatal error on sector %d: %+v", state.SectorID, evt.error) @@ -35,32 +37,32 @@ func (evt SectorFatalError) applyGlobal(state *SectorInfo) bool { } type SectorForceState struct { - state api.SectorState + State api.SectorState } func (evt SectorForceState) applyGlobal(state *SectorInfo) bool { - state.State = evt.state + state.State = evt.State return true } // Normal path type SectorStart struct { - id abi.SectorNumber - sectorType abi.RegisteredProof - pieces []Piece + ID abi.SectorNumber + SectorType abi.RegisteredProof + Pieces []Piece } func (evt SectorStart) apply(state *SectorInfo) { - state.SectorID = evt.id - state.Pieces = evt.pieces - state.SectorType = evt.sectorType + state.SectorID = evt.ID + state.Pieces = evt.Pieces + state.SectorType = evt.SectorType } -type SectorPacked struct{ pieces []Piece } +type SectorPacked struct{ Pieces []Piece } func (evt SectorPacked) apply(state *SectorInfo) { - state.Pieces = append(state.Pieces, evt.pieces...) + state.Pieces = append(state.Pieces, evt.Pieces...) } type SectorPackingFailed struct{ error } @@ -68,57 +70,59 @@ type SectorPackingFailed struct{ error } func (evt SectorPackingFailed) apply(*SectorInfo) {} type SectorSealed struct { - commR cid.Cid - commD cid.Cid - ticket api.SealTicket + Sealed cid.Cid + Unsealed cid.Cid + Ticket api.SealTicket } func (evt SectorSealed) apply(state *SectorInfo) { - commd := evt.commD + commd := evt.Unsealed state.CommD = &commd - commr := evt.commR + commr := evt.Sealed state.CommR = &commr - state.Ticket = evt.ticket + state.Ticket = evt.Ticket } type SectorSealFailed struct{ error } - +func (evt SectorSealFailed) FormatError(xerrors.Printer) (next error) {return evt.error} func (evt SectorSealFailed) apply(*SectorInfo) {} type SectorPreCommitFailed struct{ error } - +func (evt SectorPreCommitFailed) FormatError(xerrors.Printer) (next error) {return evt.error} func (evt SectorPreCommitFailed) apply(*SectorInfo) {} type SectorPreCommitted struct { - message cid.Cid + Message cid.Cid } func (evt SectorPreCommitted) apply(state *SectorInfo) { - state.PreCommitMessage = &evt.message + state.PreCommitMessage = &evt.Message } type SectorSeedReady struct { - seed api.SealSeed + Seed api.SealSeed } func (evt SectorSeedReady) apply(state *SectorInfo) { - state.Seed = evt.seed + state.Seed = evt.Seed } type SectorComputeProofFailed struct{ error } +func (evt SectorComputeProofFailed) FormatError(xerrors.Printer) (next error) {return evt.error} +func (evt SectorComputeProofFailed) apply(*SectorInfo) {} type SectorCommitFailed struct{ error } - +func (evt SectorCommitFailed) FormatError(xerrors.Printer) (next error) {return evt.error} func (evt SectorCommitFailed) apply(*SectorInfo) {} type SectorCommitted struct { - message cid.Cid - proof []byte + Message cid.Cid + Proof []byte } func (evt SectorCommitted) apply(state *SectorInfo) { - state.Proof = evt.proof - state.CommitMessage = &evt.message + state.Proof = evt.Proof + state.CommitMessage = &evt.Message } type SectorProving struct{} @@ -130,7 +134,7 @@ type SectorFinalized struct{} func (evt SectorFinalized) apply(*SectorInfo) {} type SectorFinalizeFailed struct{ error } - +func (evt SectorFinalizeFailed) FormatError(xerrors.Printer) (next error) {return evt.error} func (evt SectorFinalizeFailed) apply(*SectorInfo) {} // Failed state recovery diff --git a/storage/sealing/fsm_test.go b/storage/sealing/fsm_test.go index 41becc4f3..9b1a5d3e8 100644 --- a/storage/sealing/fsm_test.go +++ b/storage/sealing/fsm_test.go @@ -76,12 +76,12 @@ func TestSeedRevert(t *testing.T) { m.planSingle(SectorSeedReady{}) require.Equal(m.t, m.state.State, api.Committing) - _, err := m.s.plan([]statemachine.Event{{SectorSeedReady{seed: api.SealSeed{Epoch: 5}}}, {SectorCommitted{}}}, m.state) + _, err := m.s.plan([]statemachine.Event{{SectorSeedReady{Seed: api.SealSeed{Epoch: 5}}}, {SectorCommitted{}}}, m.state) require.NoError(t, err) require.Equal(m.t, m.state.State, api.Committing) // not changing the seed this time - _, err = m.s.plan([]statemachine.Event{{SectorSeedReady{seed: api.SealSeed{Epoch: 5}}}, {SectorCommitted{}}}, m.state) + _, err = m.s.plan([]statemachine.Event{{SectorSeedReady{Seed: api.SealSeed{Epoch: 5}}}, {SectorCommitted{}}}, m.state) require.Equal(m.t, m.state.State, api.CommitWait) m.planSingle(SectorProving{}) diff --git a/storage/sealing/sealing.go b/storage/sealing/sealing.go index 3c50a1507..bcfdab3b2 100644 --- a/storage/sealing/sealing.go +++ b/storage/sealing/sealing.go @@ -150,9 +150,9 @@ func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r i func (m *Sealing) newSector(sid abi.SectorNumber, rt abi.RegisteredProof, pieces []Piece) error { log.Infof("Start sealing %d", sid) return m.sectors.Send(uint64(sid), SectorStart{ - id: sid, - pieces: pieces, - sectorType: rt, + ID: sid, + Pieces: pieces, + SectorType: rt, }) } diff --git a/storage/sealing/states.go b/storage/sealing/states.go index 24f44fdf0..3e6037b6d 100644 --- a/storage/sealing/states.go +++ b/storage/sealing/states.go @@ -46,7 +46,7 @@ func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) err return xerrors.Errorf("filling up the sector (%v): %w", fillerSizes, err) } - return ctx.Send(SectorPacked{pieces: pieces}) + return ctx.Send(SectorPacked{Pieces: pieces}) } func (m *Sealing) handleUnsealed(ctx statemachine.Context, sector SectorInfo) error { @@ -81,9 +81,9 @@ func (m *Sealing) handleUnsealed(ctx statemachine.Context, sector SectorInfo) er } return ctx.Send(SectorSealed{ - commD: cids.Unsealed, - commR: cids.Sealed, - ticket: *ticket, + Unsealed: cids.Unsealed, + Sealed: cids.Sealed, + Ticket: *ticket, }) } @@ -132,7 +132,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf return ctx.Send(SectorPreCommitFailed{xerrors.Errorf("pushing message to mpool: %w", err)}) } - return ctx.Send(SectorPreCommitted{message: smsg.Cid()}) + return ctx.Send(SectorPreCommitted{Message: smsg.Cid()}) } func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) error { @@ -162,7 +162,7 @@ func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) er return err } - ctx.Send(SectorSeedReady{seed: api.SealSeed{ + ctx.Send(SectorSeedReady{Seed: api.SealSeed{ Epoch: randHeight, Value: abi.InteractiveSealRandomness(rand), }}) @@ -229,8 +229,8 @@ func (m *Sealing) handleCommitting(ctx statemachine.Context, sector SectorInfo) } return ctx.Send(SectorCommitted{ - proof: proof, - message: smsg.Cid(), + Proof: proof, + Message: smsg.Cid(), }) } From 09c9fcce00820dd62b755324490dfb1880134b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 22 Mar 2020 21:45:12 +0100 Subject: [PATCH 29/45] workers: fix panics with schedQueue --- storage/sealmgr/advmgr/manager.go | 4 +++- storage/sealmgr/advmgr/sched.go | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sealmgr/advmgr/manager.go index 153ae661f..50ba24452 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sealmgr/advmgr/manager.go @@ -59,7 +59,7 @@ type Manager struct { workerFree chan workerID closing chan struct{} - schedQueue list.List // List[*workerRequest] + schedQueue *list.List // List[*workerRequest] } func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls URLs, ca api.Common) (*Manager, error) { @@ -97,6 +97,8 @@ func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Confi workerFree: make(chan workerID), closing: make(chan struct{}), + schedQueue: list.New(), + Prover: prover, } diff --git a/storage/sealmgr/advmgr/sched.go b/storage/sealmgr/advmgr/sched.go index 6badc7663..328656a0d 100644 --- a/storage/sealmgr/advmgr/sched.go +++ b/storage/sealmgr/advmgr/sched.go @@ -99,6 +99,9 @@ func (m *Manager) onWorkerFreed(wid workerID) { if pe == nil { pe = m.schedQueue.Front() } + if pe == nil { + break + } e = pe continue } From df9aa0e3576fe95c089d7c07a02fc66cae461a7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 22 Mar 2020 22:08:22 +0100 Subject: [PATCH 30/45] storagemitner: manage ulimit on start --- cmd/lotus-storage-miner/run.go | 5 ++ go.mod | 1 + lib/ulimit/ulimit.go | 116 +++++++++++++++++++++++++++++++++ lib/ulimit/ulimit_freebsd.go | 36 ++++++++++ lib/ulimit/ulimit_test.go | 84 ++++++++++++++++++++++++ lib/ulimit/ulimit_unix.go | 27 ++++++++ 6 files changed, 269 insertions(+) create mode 100644 lib/ulimit/ulimit.go create mode 100644 lib/ulimit/ulimit_freebsd.go create mode 100644 lib/ulimit/ulimit_test.go create mode 100644 lib/ulimit/ulimit_unix.go diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 1454d8b42..897e60f27 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -20,6 +20,7 @@ import ( lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/auth" "github.com/filecoin-project/lotus/lib/jsonrpc" + "github.com/filecoin-project/lotus/lib/ulimit" "github.com/filecoin-project/lotus/node" "github.com/filecoin-project/lotus/node/impl" "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -57,6 +58,10 @@ var runCmd = &cli.Command{ return err } + if _, _, err := ulimit.ManageFdLimit(); err != nil { + log.Errorf("setting file descriptor limit: %s", err) + } + if v.APIVersion != build.APIVersion { return xerrors.Errorf("lotus-daemon API version doesn't match: local: %s", api.Version{APIVersion: build.APIVersion}) } diff --git a/go.mod b/go.mod index 3b16a5b64..2c99a3f5b 100644 --- a/go.mod +++ b/go.mod @@ -102,6 +102,7 @@ require ( go.uber.org/goleak v1.0.0 // indirect go.uber.org/multierr v1.4.0 go.uber.org/zap v1.13.0 + golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 diff --git a/lib/ulimit/ulimit.go b/lib/ulimit/ulimit.go new file mode 100644 index 000000000..5686a0f9a --- /dev/null +++ b/lib/ulimit/ulimit.go @@ -0,0 +1,116 @@ +package ulimit + +// from go-ipfs + +import ( + "fmt" + "os" + "strconv" + "syscall" + + logging "github.com/ipfs/go-log" +) + +var log = logging.Logger("ulimit") + +var ( + supportsFDManagement = false + + // getlimit returns the soft and hard limits of file descriptors counts + getLimit func() (uint64, uint64, error) + // set limit sets the soft and hard limits of file descriptors counts + setLimit func(uint64, uint64) error +) + +// minimum file descriptor limit before we complain +const minFds = 2048 + +// default max file descriptor limit. +const maxFds = 16 << 10 + +// userMaxFDs returns the value of IPFS_FD_MAX +func userMaxFDs() uint64 { + // check if the IPFS_FD_MAX is set up and if it does + // not have a valid fds number notify the user + if val := os.Getenv("IPFS_FD_MAX"); val != "" { + fds, err := strconv.ParseUint(val, 10, 64) + if err != nil { + log.Errorf("bad value for IPFS_FD_MAX: %s", err) + return 0 + } + return fds + } + return 0 +} + +// ManageFdLimit raise the current max file descriptor count +// of the process based on the IPFS_FD_MAX value +func ManageFdLimit() (changed bool, newLimit uint64, err error) { + if !supportsFDManagement { + return false, 0, nil + } + + targetLimit := uint64(maxFds) + userLimit := userMaxFDs() + if userLimit > 0 { + targetLimit = userLimit + } + + soft, hard, err := getLimit() + if err != nil { + return false, 0, err + } + + if targetLimit <= soft { + return false, 0, nil + } + + // the soft limit is the value that the kernel enforces for the + // corresponding resource + // the hard limit acts as a ceiling for the soft limit + // an unprivileged process may only set it's soft limit to a + // alue in the range from 0 up to the hard limit + err = setLimit(targetLimit, targetLimit) + switch err { + case nil: + newLimit = targetLimit + case syscall.EPERM: + // lower limit if necessary. + if targetLimit > hard { + targetLimit = hard + } + + // the process does not have permission so we should only + // set the soft value + err = setLimit(targetLimit, hard) + if err != nil { + err = fmt.Errorf("error setting ulimit wihout hard limit: %s", err) + break + } + newLimit = targetLimit + + // Warn on lowered limit. + + if newLimit < userLimit { + err = fmt.Errorf( + "failed to raise ulimit to IPFS_FD_MAX (%d): set to %d", + userLimit, + newLimit, + ) + break + } + + if userLimit == 0 && newLimit < minFds { + err = fmt.Errorf( + "failed to raise ulimit to minimum %d: set to %d", + minFds, + newLimit, + ) + break + } + default: + err = fmt.Errorf("error setting: ulimit: %s", err) + } + + return newLimit > 0, newLimit, err +} diff --git a/lib/ulimit/ulimit_freebsd.go b/lib/ulimit/ulimit_freebsd.go new file mode 100644 index 000000000..7e50436f3 --- /dev/null +++ b/lib/ulimit/ulimit_freebsd.go @@ -0,0 +1,36 @@ +// +build freebsd + +package ulimit + +import ( + "errors" + "math" + + unix "golang.org/x/sys/unix" +) + +func init() { + supportsFDManagement = true + getLimit = freebsdGetLimit + setLimit = freebsdSetLimit +} + +func freebsdGetLimit() (uint64, uint64, error) { + rlimit := unix.Rlimit{} + err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit) + if (rlimit.Cur < 0) || (rlimit.Max < 0) { + return 0, 0, errors.New("invalid rlimits") + } + return uint64(rlimit.Cur), uint64(rlimit.Max), err +} + +func freebsdSetLimit(soft uint64, max uint64) error { + if (soft > math.MaxInt64) || (max > math.MaxInt64) { + return errors.New("invalid rlimits") + } + rlimit := unix.Rlimit{ + Cur: int64(soft), + Max: int64(max), + } + return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit) +} diff --git a/lib/ulimit/ulimit_test.go b/lib/ulimit/ulimit_test.go new file mode 100644 index 000000000..b09f957a4 --- /dev/null +++ b/lib/ulimit/ulimit_test.go @@ -0,0 +1,84 @@ +// +build !windows + +package ulimit + +import ( + "fmt" + "os" + "strings" + "syscall" + "testing" +) + +func TestManageFdLimit(t *testing.T) { + t.Log("Testing file descriptor count") + if _, _, err := ManageFdLimit(); err != nil { + t.Errorf("Cannot manage file descriptors") + } + + if maxFds != uint64(16 << 10) { + t.Errorf("Maximum file descriptors default value changed") + } +} + +func TestManageInvalidNFds(t *testing.T) { + t.Logf("Testing file descriptor invalidity") + var err error + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } + + rlimit := syscall.Rlimit{} + if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil { + t.Fatal("Cannot get the file descriptor count") + } + + value := rlimit.Max + rlimit.Cur + if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil { + t.Fatal("Cannot set the IPFS_FD_MAX env variable") + } + + t.Logf("setting ulimit to %d, max %d, cur %d", value, rlimit.Max, rlimit.Cur) + + if changed, new, err := ManageFdLimit(); err == nil { + t.Errorf("ManageFdLimit should return an error: changed %t, new: %d", changed, new) + } else if err != nil { + flag := strings.Contains(err.Error(), + "failed to raise ulimit to IPFS_FD_MAX") + if !flag { + t.Error("ManageFdLimit returned unexpected error", err) + } + } + + // unset all previous operations + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } +} + +func TestManageFdLimitWithEnvSet(t *testing.T) { + t.Logf("Testing file descriptor manager with IPFS_FD_MAX set") + var err error + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } + + rlimit := syscall.Rlimit{} + if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil { + t.Fatal("Cannot get the file descriptor count") + } + + value := rlimit.Max - rlimit.Cur + 1 + if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil { + t.Fatal("Cannot set the IPFS_FD_MAX env variable") + } + + if _, _, err = ManageFdLimit(); err != nil { + t.Errorf("Cannot manage file descriptor count") + } + + // unset all previous operations + if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { + t.Fatal("Cannot unset the IPFS_FD_MAX env variable") + } +} \ No newline at end of file diff --git a/lib/ulimit/ulimit_unix.go b/lib/ulimit/ulimit_unix.go new file mode 100644 index 000000000..e1e90c7eb --- /dev/null +++ b/lib/ulimit/ulimit_unix.go @@ -0,0 +1,27 @@ +// +build darwin linux netbsd openbsd + +package ulimit + +import ( +unix "golang.org/x/sys/unix" +) + +func init() { + supportsFDManagement = true + getLimit = unixGetLimit + setLimit = unixSetLimit +} + +func unixGetLimit() (uint64, uint64, error) { + rlimit := unix.Rlimit{} + err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit) + return rlimit.Cur, rlimit.Max, err +} + +func unixSetLimit(soft uint64, max uint64) error { + rlimit := unix.Rlimit{ + Cur: soft, + Max: max, + } + return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit) +} From d43f2a2e3b10d5950e077e6079acf383369f79a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 22 Mar 2020 22:39:06 +0100 Subject: [PATCH 31/45] workers: Address review --- cmd/lotus-bench/main.go | 4 ++-- cmd/lotus-seal-worker/main.go | 4 ++-- cmd/lotus-storage-miner/storage.go | 32 ++++++++++++++++++---------- markets/retrievaladapter/provider.go | 2 +- node/builder.go | 2 +- 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index a4e52a2df..54a8a7c36 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -137,11 +137,11 @@ func main() { if err != nil { return err } - /*defer func() { + defer func() { if err := os.RemoveAll(tsdir); err != nil { log.Warn("remove all: ", err) } - }()*/ + }() sbdir = tsdir } else { exp, err := homedir.Expand(robench) diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 699a2bb04..0f7d50f5e 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -49,7 +49,7 @@ func main() { Version: build.UserVersion, Flags: []cli.Flag{ &cli.StringFlag{ - Name: "workerrepo", + Name: FlagStorageRepo, EnvVars: []string{"WORKER_PATH"}, Value: "~/.lotusworker", // TODO: Consider XDG_DATA_HOME }, @@ -261,7 +261,7 @@ var runCmd = &cli.Command{ log.Warn("Graceful shutdown successful") }() - nl, err := net.Listen("tcp4", cctx.String("address")) + nl, err := net.Listen("tcp", cctx.String("address")) if err != nil { return err } diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 0293b4328..144c8e5eb 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -135,27 +135,37 @@ var storageListCmd = &cli.Command{ } for id, sectors := range st { - var u, s, c int + var cnt [3]int for _, decl := range sectors { - if decl.SectorFileType§orbuilder.FTUnsealed > 0 { - u++ - } - if decl.SectorFileType§orbuilder.FTSealed > 0 { - s++ - } - if decl.SectorFileType§orbuilder.FTCache > 0 { - c++ + for i := range cnt { + if decl.SectorFileType&(1< Date: Sun, 22 Mar 2020 22:39:27 +0100 Subject: [PATCH 32/45] gofmt --- lib/ulimit/ulimit_test.go | 4 ++-- lib/ulimit/ulimit_unix.go | 2 +- storage/sealing/fsm.go | 1 - storage/sealing/fsm_events.go | 28 +++++++++++++++++----------- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/ulimit/ulimit_test.go b/lib/ulimit/ulimit_test.go index b09f957a4..1e7fc7d16 100644 --- a/lib/ulimit/ulimit_test.go +++ b/lib/ulimit/ulimit_test.go @@ -16,7 +16,7 @@ func TestManageFdLimit(t *testing.T) { t.Errorf("Cannot manage file descriptors") } - if maxFds != uint64(16 << 10) { + if maxFds != uint64(16<<10) { t.Errorf("Maximum file descriptors default value changed") } } @@ -81,4 +81,4 @@ func TestManageFdLimitWithEnvSet(t *testing.T) { if err = os.Unsetenv("IPFS_FD_MAX"); err != nil { t.Fatal("Cannot unset the IPFS_FD_MAX env variable") } -} \ No newline at end of file +} diff --git a/lib/ulimit/ulimit_unix.go b/lib/ulimit/ulimit_unix.go index e1e90c7eb..a351236dc 100644 --- a/lib/ulimit/ulimit_unix.go +++ b/lib/ulimit/ulimit_unix.go @@ -3,7 +3,7 @@ package ulimit import ( -unix "golang.org/x/sys/unix" + unix "golang.org/x/sys/unix" ) func init() { diff --git a/storage/sealing/fsm.go b/storage/sealing/fsm.go index 89d27178c..0911bfba5 100644 --- a/storage/sealing/fsm.go +++ b/storage/sealing/fsm.go @@ -83,7 +83,6 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta ///// // First process all events - for _, event := range events { e, err := json.Marshal(event) if err != nil { diff --git a/storage/sealing/fsm_events.go b/storage/sealing/fsm_events.go index d358ba360..d433765ee 100644 --- a/storage/sealing/fsm_events.go +++ b/storage/sealing/fsm_events.go @@ -26,7 +26,8 @@ type SectorRestart struct{} func (evt SectorRestart) applyGlobal(*SectorInfo) bool { return false } type SectorFatalError struct{ error } -func (evt SectorFatalError) FormatError(xerrors.Printer) (next error) {return evt.error} + +func (evt SectorFatalError) FormatError(xerrors.Printer) (next error) { return evt.error } func (evt SectorFatalError) applyGlobal(state *SectorInfo) bool { log.Errorf("Fatal error on sector %d: %+v", state.SectorID, evt.error) @@ -84,12 +85,14 @@ func (evt SectorSealed) apply(state *SectorInfo) { } type SectorSealFailed struct{ error } -func (evt SectorSealFailed) FormatError(xerrors.Printer) (next error) {return evt.error} -func (evt SectorSealFailed) apply(*SectorInfo) {} + +func (evt SectorSealFailed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorSealFailed) apply(*SectorInfo) {} type SectorPreCommitFailed struct{ error } -func (evt SectorPreCommitFailed) FormatError(xerrors.Printer) (next error) {return evt.error} -func (evt SectorPreCommitFailed) apply(*SectorInfo) {} + +func (evt SectorPreCommitFailed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorPreCommitFailed) apply(*SectorInfo) {} type SectorPreCommitted struct { Message cid.Cid @@ -108,12 +111,14 @@ func (evt SectorSeedReady) apply(state *SectorInfo) { } type SectorComputeProofFailed struct{ error } -func (evt SectorComputeProofFailed) FormatError(xerrors.Printer) (next error) {return evt.error} -func (evt SectorComputeProofFailed) apply(*SectorInfo) {} + +func (evt SectorComputeProofFailed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorComputeProofFailed) apply(*SectorInfo) {} type SectorCommitFailed struct{ error } -func (evt SectorCommitFailed) FormatError(xerrors.Printer) (next error) {return evt.error} -func (evt SectorCommitFailed) apply(*SectorInfo) {} + +func (evt SectorCommitFailed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorCommitFailed) apply(*SectorInfo) {} type SectorCommitted struct { Message cid.Cid @@ -134,8 +139,9 @@ type SectorFinalized struct{} func (evt SectorFinalized) apply(*SectorInfo) {} type SectorFinalizeFailed struct{ error } -func (evt SectorFinalizeFailed) FormatError(xerrors.Printer) (next error) {return evt.error} -func (evt SectorFinalizeFailed) apply(*SectorInfo) {} + +func (evt SectorFinalizeFailed) FormatError(xerrors.Printer) (next error) { return evt.error } +func (evt SectorFinalizeFailed) apply(*SectorInfo) {} // Failed state recovery From 5e4a7e54dfa86148d8d46455a2ead3eedffc00ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 23 Mar 2020 12:40:02 +0100 Subject: [PATCH 33/45] Rename agvmgr+sealmgr to sectorstorage --- api/api_storage.go | 2 +- api/api_worker.go | 6 +- api/apistruct/struct.go | 16 +-- chain/gen/gen.go | 4 +- cmd/lotus-seal-worker/main.go | 11 +- cmd/lotus-seal-worker/rpc.go | 7 +- cmd/lotus-seed/seed/seed.go | 2 +- cmd/lotus-storage-miner/init.go | 6 +- cmd/lotus-storage-miner/storage.go | 2 +- markets/retrievaladapter/provider.go | 6 +- node/builder.go | 15 ++- node/impl/storminer.go | 8 +- node/modules/storageminer.go | 6 +- node/node_test.go | 15 ++- node/repo/memrepo.go | 2 +- storage/miner.go | 6 +- storage/sealing/sealing.go | 6 +- storage/sealmgr/simple.go | 115 ------------------ storage/sealmgr/types.go | 30 ----- .../advmgr => sectorstorage}/manager.go | 65 +++++----- .../sbmock.go => sectorstorage/mock/mock.go} | 43 +++---- .../mock/mock_test.go} | 4 +- .../{sbmock => sectorstorage/mock}/preseal.go | 2 +- .../{sbmock => sectorstorage/mock}/util.go | 2 +- .../advmgr => sectorstorage}/resources.go | 16 +-- .../advmgr => sectorstorage}/roprov.go | 4 +- .../advmgr => sectorstorage}/sched.go | 6 +- .../sealtasks}/task.go | 2 +- .../sectorutil/utils.go | 0 .../stores/http_handler.go | 2 +- .../stores/index.go | 2 +- .../stores/interface.go | 0 .../stores/local.go | 2 +- .../stores/remote.go | 2 +- .../advmgr => sectorstorage}/worker_local.go | 16 +-- .../advmgr => sectorstorage}/worker_remote.go | 2 +- 36 files changed, 150 insertions(+), 285 deletions(-) delete mode 100644 storage/sealmgr/simple.go delete mode 100644 storage/sealmgr/types.go rename storage/{sealmgr/advmgr => sectorstorage}/manager.go (80%) rename storage/{sbmock/sbmock.go => sectorstorage/mock/mock.go} (74%) rename storage/{sbmock/sbmock_test.go => sectorstorage/mock/mock_test.go} (93%) rename storage/{sbmock => sectorstorage/mock}/preseal.go (99%) rename storage/{sbmock => sectorstorage/mock}/util.go (94%) rename storage/{sealmgr/advmgr => sectorstorage}/resources.go (88%) rename storage/{sealmgr/advmgr => sectorstorage}/roprov.go (87%) rename storage/{sealmgr/advmgr => sectorstorage}/sched.go (97%) rename storage/{sealmgr => sectorstorage/sealtasks}/task.go (95%) rename storage/{sealmgr => sectorstorage}/sectorutil/utils.go (100%) rename storage/{sealmgr => sectorstorage}/stores/http_handler.go (97%) rename storage/{sealmgr => sectorstorage}/stores/index.go (99%) rename storage/{sealmgr => sectorstorage}/stores/interface.go (100%) rename storage/{sealmgr => sectorstorage}/stores/local.go (99%) rename storage/{sealmgr => sectorstorage}/stores/remote.go (98%) rename storage/{sealmgr/advmgr => sectorstorage}/worker_local.go (91%) rename storage/{sealmgr/advmgr => sectorstorage}/worker_remote.go (98%) diff --git a/api/api_storage.go b/api/api_storage.go index 863feed1b..9f032ecb2 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) // alias because cbor-gen doesn't like non-alias types diff --git a/api/api_worker.go b/api/api_worker.go index 7a0004656..999120643 100644 --- a/api/api_worker.go +++ b/api/api_worker.go @@ -6,15 +6,15 @@ import ( "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/storage/sealmgr" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "github.com/filecoin-project/lotus/storage/sectorstorage/sealtasks" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) type WorkerApi interface { Version(context.Context) (build.Version, error) // TODO: Info() (name, ...) ? - TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) // TaskType -> Weight + TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) // TaskType -> Weight Paths(context.Context) ([]stores.StoragePath, error) Info(context.Context) (WorkerInfo, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 10ddfde10..b1a6b265e 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -2,28 +2,28 @@ package apistruct import ( "context" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" - "github.com/filecoin-project/specs-storage/storage" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi/big" "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/specs-actors/actors/builtin/paych" "github.com/filecoin-project/specs-actors/actors/builtin/reward" "github.com/filecoin-project/specs-actors/actors/crypto" + "github.com/filecoin-project/specs-storage/storage" - "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sectorstorage/sealtasks" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) // All permissions are listed in permissioned.go @@ -204,9 +204,9 @@ type WorkerStruct struct { Version func(context.Context) (build.Version, error) `perm:"admin"` - TaskTypes func(context.Context) (map[sealmgr.TaskType]struct{}, error) `perm:"admin"` - Paths func(context.Context) ([]stores.StoragePath, error) `perm:"admin"` - Info func(context.Context) (api.WorkerInfo, error) `perm:"admin"` + TaskTypes func(context.Context) (map[sealtasks.TaskType]struct{}, error) `perm:"admin"` + Paths func(context.Context) ([]stores.StoragePath, error) `perm:"admin"` + Info func(context.Context) (api.WorkerInfo, error) `perm:"admin"` SealPreCommit1 func(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) `perm:"admin"` SealPreCommit2 func(context.Context, abi.SectorID, storage.PreCommit1Out) (cids storage.SectorCids, err error) `perm:"admin"` @@ -722,7 +722,7 @@ func (w *WorkerStruct) Version(ctx context.Context) (build.Version, error) { return w.Internal.Version(ctx) } -func (w *WorkerStruct) TaskTypes(ctx context.Context) (map[sealmgr.TaskType]struct{}, error) { +func (w *WorkerStruct) TaskTypes(ctx context.Context) (map[sealtasks.TaskType]struct{}, error) { return w.Internal.TaskTypes(ctx) } diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 9fc9a79c6..c14c62b7d 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -36,7 +36,7 @@ import ( "github.com/filecoin-project/lotus/genesis" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage/sbmock" + "github.com/filecoin-project/lotus/storage/sectorstorage/mock" "go.opencensus.io/trace" "golang.org/x/xerrors" @@ -690,7 +690,7 @@ func (m genFakeVerifier) VerifyElectionPost(ctx context.Context, pvi abi.PoStVer panic("nyi") } func (m genFakeVerifier) GenerateDataCommitment(ssize abi.PaddedPieceSize, pieces []abi.PieceInfo) (cid.Cid, error) { - return sbmock.MockVerifier.GenerateDataCommitment(ssize, pieces) + return mock.MockVerifier.GenerateDataCommitment(ssize, pieces) } func (m genFakeVerifier) VerifySeal(svi abi.SealVerifyInfo) (bool, error) { diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index 0f7d50f5e..c3717735d 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -16,6 +16,7 @@ import ( "gopkg.in/urfave/cli.v2" paramfetch "github.com/filecoin-project/go-paramfetch" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/build" @@ -25,9 +26,9 @@ import ( "github.com/filecoin-project/lotus/lib/lotuslog" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage/sealmgr" - "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "github.com/filecoin-project/lotus/storage/sectorstorage" + "github.com/filecoin-project/lotus/storage/sectorstorage/sealtasks" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) var log = logging.Logger("main") @@ -223,9 +224,9 @@ var runCmd = &cli.Command{ // Create / expose the worker workerApi := &worker{ - LocalWorker: advmgr.NewLocalWorker(advmgr.WorkerConfig{ + LocalWorker: sectorstorage.NewLocalWorker(sectorstorage.WorkerConfig{ SealProof: spt, - TaskTypes: []sealmgr.TaskType{sealmgr.TTPreCommit1, sealmgr.TTPreCommit2, sealmgr.TTCommit2}, + TaskTypes: []sealtasks.TaskType{sealtasks.TTPreCommit1, sealtasks.TTPreCommit2, sealtasks.TTCommit2}, }, remote, localStore, nodeApi), } diff --git a/cmd/lotus-seal-worker/rpc.go b/cmd/lotus-seal-worker/rpc.go index 9de4d1079..5d8ce3cb7 100644 --- a/cmd/lotus-seal-worker/rpc.go +++ b/cmd/lotus-seal-worker/rpc.go @@ -3,13 +3,14 @@ package main import ( "context" - "github.com/filecoin-project/lotus/build" - "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" "github.com/filecoin-project/specs-storage/storage" + + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/storage/sectorstorage" ) type worker struct { // TODO: use advmgr.LocalWorker here - *advmgr.LocalWorker + *sectorstorage.LocalWorker } func (w *worker) Version(context.Context) (build.Version, error) { diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 8ad1896e1..19347923e 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -7,7 +7,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" "io/ioutil" "os" "path/filepath" @@ -27,6 +26,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" "github.com/filecoin-project/lotus/genesis" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) var log = logging.Logger("preseal") diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 94c8ee0c5..5a62c4e19 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -43,8 +43,8 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sealing" - "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "github.com/filecoin-project/lotus/storage/sectorstorage" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) var initCmd = &cli.Command{ @@ -404,7 +404,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return xerrors.Errorf("getting id address: %w", err) } - smgr, err := advmgr.New(lr, stores.NewIndex(), §orbuilder.Config{ + smgr, err := sectorstorage.New(lr, stores.NewIndex(), §orbuilder.Config{ SealProofType: spt, PoStProofType: ppt, }, nil, api) diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 144c8e5eb..417c5b83a 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -19,7 +19,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) const metaFile = "sectorstore.json" diff --git a/markets/retrievaladapter/provider.go b/markets/retrievaladapter/provider.go index b7476116d..685e9b2dd 100644 --- a/markets/retrievaladapter/provider.go +++ b/markets/retrievaladapter/provider.go @@ -14,18 +14,18 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sectorstorage" ) type retrievalProviderNode struct { miner *storage.Miner - sealer sealmgr.Manager + sealer sectorstorage.SectorManager full api.FullNode } // NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the // Lotus Node -func NewRetrievalProviderNode(miner *storage.Miner, sealer sealmgr.Manager, full api.FullNode) retrievalmarket.RetrievalProviderNode { +func NewRetrievalProviderNode(miner *storage.Miner, sealer sectorstorage.SectorManager, full api.FullNode) retrievalmarket.RetrievalProviderNode { return &retrievalProviderNode{miner, sealer, full} } diff --git a/node/builder.go b/node/builder.go index 793394ae9..ec8c11f63 100644 --- a/node/builder.go +++ b/node/builder.go @@ -26,7 +26,6 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket/impl/requestvalidation" sectorbuilder "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" "github.com/filecoin-project/specs-actors/actors/runtime" storage2 "github.com/filecoin-project/specs-storage/storage" @@ -61,9 +60,9 @@ import ( "github.com/filecoin-project/lotus/paychmgr" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sealing" - "github.com/filecoin-project/lotus/storage/sealmgr" - "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/filecoin-project/lotus/storage/sectorstorage" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) var log = logging.Logger("builder") @@ -273,10 +272,10 @@ func Online() Option { Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig), Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), Override(new(sealing.SectorIDCounter), modules.SectorIDCounter), - Override(new(*advmgr.Manager), advmgr.New), + Override(new(*sectorstorage.Manager), sectorstorage.New), - Override(new(sealmgr.Manager), From(new(*advmgr.Manager))), - Override(new(storage2.Prover), From(new(sealmgr.Manager))), + Override(new(sectorstorage.SectorManager), From(new(*sectorstorage.Manager))), + Override(new(storage2.Prover), From(new(sectorstorage.SectorManager))), Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks), Override(new(sealing.TicketFn), modules.SealTicketGen), @@ -335,13 +334,13 @@ func ConfigCommon(cfg *config.Common) Option { Override(SetApiEndpointKey, func(lr repo.LockedRepo, e dtypes.APIEndpoint) error { return lr.SetAPIEndpoint(e) }), - Override(new(advmgr.URLs), func(e dtypes.APIEndpoint) (advmgr.URLs, error) { + Override(new(sectorstorage.URLs), func(e dtypes.APIEndpoint) (sectorstorage.URLs, error) { _, ip, err := manet.DialArgs(e) if err != nil { return nil, xerrors.Errorf("getting api endpoint dial args: %w", err) } - var urls advmgr.URLs + var urls sectorstorage.URLs urls = append(urls, "http://"+ip+"/remote") // TODO: This makes assumptions, and probably bad ones too return urls, nil }), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 45fb726e2..41c0afa20 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -21,9 +21,9 @@ import ( "github.com/filecoin-project/lotus/miner" "github.com/filecoin-project/lotus/node/impl/common" "github.com/filecoin-project/lotus/storage" - "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/filecoin-project/lotus/storage/sectorstorage" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) type StorageMinerAPI struct { @@ -37,7 +37,7 @@ type StorageMinerAPI struct { Miner *storage.Miner BlockMiner *miner.Miner Full api.FullNode - StorageMgr *advmgr.Manager `optional:"true"` + StorageMgr *sectorstorage.Manager `optional:"true"` *stores.Index } @@ -148,7 +148,7 @@ func (sm *StorageMinerAPI) SectorsUpdate(ctx context.Context, id abi.SectorNumbe } func (sm *StorageMinerAPI) WorkerConnect(ctx context.Context, url string) error { - w, err := advmgr.ConnectRemote(ctx, sm, url) + w, err := sectorstorage.ConnectRemote(ctx, sm, url) if err != nil { return xerrors.Errorf("connecting remote storage failed: %w", err) } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 07dc90344..2c6a4c805 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -49,7 +49,7 @@ import ( "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sealing" - "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sectorstorage" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { @@ -117,7 +117,7 @@ func SectorIDCounter(ds dtypes.MetadataDS) sealing.SectorIDCounter { return &sidsc{sc} } -func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sealmgr.Manager, sc sealing.SectorIDCounter, tktFn sealing.TicketFn) (*storage.Miner, error) { +func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api lapi.FullNode, h host.Host, ds dtypes.MetadataDS, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, tktFn sealing.TicketFn) (*storage.Miner, error) { maddr, err := minerAddrFromDS(ds) if err != nil { return nil, err @@ -327,7 +327,7 @@ func StorageProvider(ctx helpers.MetricsCtx, fapi lapi.FullNode, h host.Host, ds } // RetrievalProvider creates a new retrieval provider attached to the provider blockstore -func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sealmgr.Manager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, ibs dtypes.StagingBlockstore) (retrievalmarket.RetrievalProvider, error) { +func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.SectorManager, full lapi.FullNode, ds dtypes.MetadataDS, pieceStore dtypes.ProviderPieceStore, ibs dtypes.StagingBlockstore) (retrievalmarket.RetrievalProvider, error) { adapter := retrievaladapter.NewRetrievalProviderNode(miner, sealer, full) address, err := minerAddrFromDS(ds) if err != nil { diff --git a/node/node_test.go b/node/node_test.go index 315c4dc26..e7884c377 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -41,9 +41,8 @@ import ( "github.com/filecoin-project/lotus/node/modules" modtest "github.com/filecoin-project/lotus/node/modules/testing" "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage/sbmock" - "github.com/filecoin-project/lotus/storage/sealmgr" - "github.com/filecoin-project/lotus/storage/sealmgr/advmgr" + "github.com/filecoin-project/lotus/storage/sectorstorage" + "github.com/filecoin-project/lotus/storage/sectorstorage/mock" ) func init() { @@ -310,7 +309,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t if err != nil { t.Fatal(err) } - genm, k, err := sbmock.PreSeal(2048, maddr, nPreseal) + genm, k, err := mock.PreSeal(2048, maddr, nPreseal) if err != nil { t.Fatal(err) } @@ -356,7 +355,7 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t node.MockHost(mn), node.Test(), - node.Override(new(sectorbuilder.Verifier), sbmock.MockVerifier), + node.Override(new(sectorbuilder.Verifier), mock.MockVerifier), genesis, ) @@ -386,10 +385,10 @@ func mockSbBuilder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []t wa := genms[i].Worker storers[i] = testStorageNode(ctx, t, wa, genMiner, pk, f, mn, node.Options( - node.Override(new(sealmgr.Manager), func() (sealmgr.Manager, error) { - return sealmgr.NewSimpleManager(genMiner, sbmock.NewMockSectorBuilder(5, build.SectorSizes[0])) + node.Override(new(sectorstorage.SectorManager), func() (sectorstorage.SectorManager, error) { + return mock.NewMockSectorMgr(5, build.SectorSizes[0]), nil }), - node.Unset(new(*advmgr.Manager)), + node.Unset(new(*sectorstorage.Manager)), )) } diff --git a/node/repo/memrepo.go b/node/repo/memrepo.go index 61141f115..805eab157 100644 --- a/node/repo/memrepo.go +++ b/node/repo/memrepo.go @@ -2,7 +2,6 @@ package repo import ( "encoding/json" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" "io/ioutil" "os" "path/filepath" @@ -17,6 +16,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/config" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) type MemRepo struct { diff --git a/storage/miner.go b/storage/miner.go index 9fea5f042..9b6ed7b5a 100644 --- a/storage/miner.go +++ b/storage/miner.go @@ -14,7 +14,7 @@ import ( "github.com/libp2p/go-libp2p-core/host" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sectorstorage" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/builtin/miner" @@ -34,7 +34,7 @@ var log = logging.Logger("storageminer") type Miner struct { api storageMinerApi h host.Host - sealer sealmgr.Manager + sealer sectorstorage.SectorManager ds datastore.Batching tktFn sealing.TicketFn sc sealing.SectorIDCounter @@ -74,7 +74,7 @@ type storageMinerApi interface { WalletHas(context.Context, address.Address) (bool, error) } -func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sealmgr.Manager, sc sealing.SectorIDCounter, tktFn sealing.TicketFn) (*Miner, error) { +func NewMiner(api storageMinerApi, maddr, worker address.Address, h host.Host, ds datastore.Batching, sealer sectorstorage.SectorManager, sc sealing.SectorIDCounter, tktFn sealing.TicketFn) (*Miner, error) { m := &Miner{ api: api, h: h, diff --git a/storage/sealing/sealing.go b/storage/sealing/sealing.go index bcfdab3b2..97d2cee51 100644 --- a/storage/sealing/sealing.go +++ b/storage/sealing/sealing.go @@ -21,7 +21,7 @@ import ( "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/store" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sectorstorage" ) const SectorStorePrefix = "/sectors" @@ -69,13 +69,13 @@ type Sealing struct { maddr address.Address worker address.Address - sealer sealmgr.Manager + sealer sectorstorage.SectorManager sectors *statemachine.StateGroup tktFn TicketFn sc SectorIDCounter } -func New(api sealingApi, events *events.Events, maddr address.Address, worker address.Address, ds datastore.Batching, sealer sealmgr.Manager, sc SectorIDCounter, tktFn TicketFn) *Sealing { +func New(api sealingApi, events *events.Events, maddr address.Address, worker address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, tktFn TicketFn) *Sealing { s := &Sealing{ api: api, events: events, diff --git a/storage/sealmgr/simple.go b/storage/sealmgr/simple.go deleted file mode 100644 index 1dc47b64e..000000000 --- a/storage/sealmgr/simple.go +++ /dev/null @@ -1,115 +0,0 @@ -package sealmgr - -import ( - "context" - "io" - "sync" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/specs-storage/storage" - "github.com/ipfs/go-cid" -) - -type LocalWorker struct { - sectorbuilder.Basic -} - -var _ Worker = &LocalWorker{} - -// Simple implements a very basic storage manager which has one local worker, -// running one thing locally -type Simple struct { - maddr address.Address - - rateLimiter sync.Mutex - worker Worker -} - -type sszgetter interface { - SectorSize() abi.SectorSize -} - -func (s *LocalWorker) SectorSize() abi.SectorSize { - return s.Basic.(sszgetter).SectorSize() -} - -func (s *Simple) SectorSize() abi.SectorSize { - return s.worker.(sszgetter).SectorSize() -} - -func NewSimpleManager(maddr address.Address, sb sectorbuilder.Basic) (*Simple, error) { - w := &LocalWorker{ - sb, - } - - return &Simple{ - maddr: maddr, - worker: w, - }, nil -} - -func (s *Simple) NewSector(ctx context.Context, id abi.SectorID) error { - return s.worker.NewSector(ctx, id) -} - -func (s *Simple) AddPiece(ctx context.Context, id abi.SectorID, existingPieces []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r storage.Data) (abi.PieceInfo, error) { - s.rateLimiter.Lock() - defer s.rateLimiter.Unlock() - - return s.worker.AddPiece(ctx, id, existingPieces, sz, r) -} - -func (s *Simple) SealPreCommit1(ctx context.Context, id abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { - s.rateLimiter.Lock() - defer s.rateLimiter.Unlock() - - return s.worker.SealPreCommit1(ctx, id, ticket, pieces) -} - -func (s *Simple) SealPreCommit2(ctx context.Context, id abi.SectorID, phase1Out storage.PreCommit1Out) (cids storage.SectorCids, err error) { - s.rateLimiter.Lock() - defer s.rateLimiter.Unlock() - - return s.worker.SealPreCommit2(ctx, id, phase1Out) -} - -func (s *Simple) SealCommit1(ctx context.Context, id abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (output storage.Commit1Out, err error) { - s.rateLimiter.Lock() - defer s.rateLimiter.Unlock() - - return s.worker.SealCommit1(ctx, id, ticket, seed, pieces, cids) -} - -func (s *Simple) SealCommit2(ctx context.Context, id abi.SectorID, phase1Out storage.Commit1Out) (proof storage.Proof, err error) { - s.rateLimiter.Lock() - defer s.rateLimiter.Unlock() - - return s.worker.SealCommit2(ctx, id, phase1Out) -} - -func (s *Simple) FinalizeSector(ctx context.Context, id abi.SectorID) error { - s.rateLimiter.Lock() - defer s.rateLimiter.Unlock() - - return s.worker.FinalizeSector(ctx, id) -} - -func (s *Simple) GenerateEPostCandidates(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) { - return s.worker.GenerateEPostCandidates(ctx, miner, sectorInfo, challengeSeed, faults) -} - -func (s *Simple) GenerateFallbackPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) (storage.FallbackPostOut, error) { - return s.worker.GenerateFallbackPoSt(ctx, miner, sectorInfo, challengeSeed, faults) -} - -func (s *Simple) ComputeElectionPoSt(ctx context.Context, miner abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) { - return s.worker.ComputeElectionPoSt(ctx, miner, sectorInfo, challengeSeed, winners) -} - -func (s *Simple) ReadPieceFromSealedSector(context.Context, abi.SectorID, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) { - panic("todo") -} - -var _ Manager = &Simple{} diff --git a/storage/sealmgr/types.go b/storage/sealmgr/types.go deleted file mode 100644 index c9f11d38b..000000000 --- a/storage/sealmgr/types.go +++ /dev/null @@ -1,30 +0,0 @@ -package sealmgr - -import ( - "context" - "io" - - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/specs-storage/storage" -) - -type Worker interface { - sectorbuilder.Sealer - storage.Prover -} - -type Manager interface { - SectorSize() abi.SectorSize - - // TODO: Can[Pre]Commit[1,2] - // TODO: Scrub() []Faults - - // TODO: Separate iface - ReadPieceFromSealedSector(context.Context, abi.SectorID, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) - - sectorbuilder.Sealer - storage.Prover -} diff --git a/storage/sealmgr/advmgr/manager.go b/storage/sectorstorage/manager.go similarity index 80% rename from storage/sealmgr/advmgr/manager.go rename to storage/sectorstorage/manager.go index 50ba24452..08597bd31 100644 --- a/storage/sealmgr/advmgr/manager.go +++ b/storage/sectorstorage/manager.go @@ -1,4 +1,4 @@ -package advmgr +package sectorstorage import ( "container/list" @@ -14,12 +14,12 @@ import ( "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" - storage2 "github.com/filecoin-project/specs-storage/storage" + "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/node/config" - "github.com/filecoin-project/lotus/storage/sealmgr" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "github.com/filecoin-project/lotus/storage/sectorstorage/sealtasks" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) var log = logging.Logger("advmgr") @@ -29,7 +29,7 @@ type URLs []string type Worker interface { sectorbuilder.Sealer - TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) + TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) // Returns paths accessible to the worker Paths(context.Context) ([]stores.StoragePath, error) @@ -37,6 +37,15 @@ type Worker interface { Info(context.Context) (api.WorkerInfo, error) } +type SectorManager interface { + SectorSize() abi.SectorSize + + ReadPieceFromSealedSector(context.Context, abi.SectorID, sectorbuilder.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (io.ReadCloser, error) + + sectorbuilder.Sealer + storage.Prover +} + type workerID uint64 type Manager struct { @@ -48,7 +57,7 @@ type Manager struct { remoteHnd *stores.FetchHandler index stores.SectorIndex - storage2.Prover + storage.Prover workersLk sync.Mutex nextWorker workerID @@ -106,7 +115,7 @@ func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Confi err = m.AddWorker(ctx, NewLocalWorker(WorkerConfig{ SealProof: cfg.SealProofType, - TaskTypes: []sealmgr.TaskType{sealmgr.TTAddPiece, sealmgr.TTCommit1, sealmgr.TTFinalize}, + TaskTypes: []sealtasks.TaskType{sealtasks.TTAddPiece, sealtasks.TTCommit1, sealtasks.TTFinalize}, }, stor, lstor, si)) if err != nil { return nil, xerrors.Errorf("adding local worker: %w", err) @@ -159,7 +168,7 @@ func (m *Manager) ReadPieceFromSealedSector(context.Context, abi.SectorID, secto panic("implement me") } -func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.StorageInfo) ([]workerID, map[workerID]stores.StorageInfo) { +func (m *Manager) getWorkersByPaths(task sealtasks.TaskType, inPaths []stores.StorageInfo) ([]workerID, map[workerID]stores.StorageInfo) { m.workersLk.Lock() defer m.workersLk.Unlock() @@ -210,7 +219,7 @@ func (m *Manager) getWorkersByPaths(task sealmgr.TaskType, inPaths []stores.Stor return workers, paths } -func (m *Manager) getWorker(ctx context.Context, taskType sealmgr.TaskType, accept []workerID) (Worker, func(), error) { +func (m *Manager) getWorker(ctx context.Context, taskType sealtasks.TaskType, accept []workerID) (Worker, func(), error) { ret := make(chan workerResponse) select { @@ -257,13 +266,13 @@ func (m *Manager) AddPiece(ctx context.Context, sector abi.SectorID, existingPie } log.Debugf("find workers for %v", best) - candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTAddPiece, best) + candidateWorkers, _ := m.getWorkersByPaths(sealtasks.TTAddPiece, best) if len(candidateWorkers) == 0 { return abi.PieceInfo{}, xerrors.New("no worker found") } - worker, done, err := m.getWorker(ctx, sealmgr.TTAddPiece, candidateWorkers) + worker, done, err := m.getWorker(ctx, sealtasks.TTAddPiece, candidateWorkers) if err != nil { return abi.PieceInfo{}, xerrors.Errorf("scheduling worker: %w", err) } @@ -274,7 +283,7 @@ func (m *Manager) AddPiece(ctx context.Context, sector abi.SectorID, existingPie return worker.AddPiece(ctx, sector, existingPieces, sz, r) } -func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage2.PreCommit1Out, err error) { +func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { // TODO: also consider where the unsealed data sits best, err := m.index.StorageBestAlloc(ctx, sectorbuilder.FTCache|sectorbuilder.FTSealed, true) @@ -282,12 +291,12 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticke return nil, xerrors.Errorf("finding path for sector sealing: %w", err) } - candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit1, best) + candidateWorkers, _ := m.getWorkersByPaths(sealtasks.TTPreCommit1, best) if len(candidateWorkers) == 0 { return nil, xerrors.New("no suitable workers found") } - worker, done, err := m.getWorker(ctx, sealmgr.TTPreCommit1, candidateWorkers) + worker, done, err := m.getWorker(ctx, sealtasks.TTPreCommit1, candidateWorkers) if err != nil { return nil, xerrors.Errorf("scheduling worker: %w", err) } @@ -298,22 +307,22 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticke return worker.SealPreCommit1(ctx, sector, ticket, pieces) } -func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.PreCommit1Out) (cids storage2.SectorCids, err error) { +func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.PreCommit1Out) (cids storage.SectorCids, err error) { // TODO: allow workers to fetch the sectors best, err := m.index.StorageFindSector(ctx, sector, sectorbuilder.FTCache|sectorbuilder.FTSealed, true) if err != nil { - return storage2.SectorCids{}, xerrors.Errorf("finding path for sector sealing: %w", err) + return storage.SectorCids{}, xerrors.Errorf("finding path for sector sealing: %w", err) } - candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTPreCommit2, best) + candidateWorkers, _ := m.getWorkersByPaths(sealtasks.TTPreCommit2, best) if len(candidateWorkers) == 0 { - return storage2.SectorCids{}, xerrors.New("no suitable workers found") + return storage.SectorCids{}, xerrors.New("no suitable workers found") } - worker, done, err := m.getWorker(ctx, sealmgr.TTPreCommit2, candidateWorkers) + worker, done, err := m.getWorker(ctx, sealtasks.TTPreCommit2, candidateWorkers) if err != nil { - return storage2.SectorCids{}, xerrors.Errorf("scheduling worker: %w", err) + return storage.SectorCids{}, xerrors.Errorf("scheduling worker: %w", err) } defer done() @@ -322,19 +331,19 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase return worker.SealPreCommit2(ctx, sector, phase1Out) } -func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage2.SectorCids) (output storage2.Commit1Out, err error) { +func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (output storage.Commit1Out, err error) { best, err := m.index.StorageFindSector(ctx, sector, sectorbuilder.FTCache|sectorbuilder.FTSealed, true) if err != nil { return nil, xerrors.Errorf("finding path for sector sealing: %w", err) } - candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTCommit1, best) + candidateWorkers, _ := m.getWorkersByPaths(sealtasks.TTCommit1, best) if len(candidateWorkers) == 0 { return nil, xerrors.New("no suitable workers found") // TODO: wait? } // TODO: Try very hard to execute on worker with access to the sectors - worker, done, err := m.getWorker(ctx, sealmgr.TTCommit1, candidateWorkers) + worker, done, err := m.getWorker(ctx, sealtasks.TTCommit1, candidateWorkers) if err != nil { return nil, xerrors.Errorf("scheduling worker: %w", err) } @@ -345,7 +354,7 @@ func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket a return worker.SealCommit1(ctx, sector, ticket, seed, pieces, cids) } -func (m *Manager) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage2.Commit1Out) (proof storage2.Proof, err error) { +func (m *Manager) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.Commit1Out) (proof storage.Proof, err error) { var candidateWorkers []workerID m.workersLk.Lock() @@ -355,14 +364,14 @@ func (m *Manager) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Ou log.Errorf("error getting supported worker task types: %+v", err) continue } - if _, ok := tt[sealmgr.TTCommit2]; !ok { + if _, ok := tt[sealtasks.TTCommit2]; !ok { continue } candidateWorkers = append(candidateWorkers, id) } m.workersLk.Unlock() - worker, done, err := m.getWorker(ctx, sealmgr.TTCommit2, candidateWorkers) + worker, done, err := m.getWorker(ctx, sealtasks.TTCommit2, candidateWorkers) if err != nil { return nil, xerrors.Errorf("scheduling worker: %w", err) } @@ -377,7 +386,7 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector abi.SectorID) error return xerrors.Errorf("finding sealed sector: %w", err) } - candidateWorkers, _ := m.getWorkersByPaths(sealmgr.TTFinalize, best) + candidateWorkers, _ := m.getWorkersByPaths(sealtasks.TTFinalize, best) // TODO: Remove sector from sealing stores // TODO: Move the sector to long-term storage @@ -398,4 +407,4 @@ func (m *Manager) StorageLocal(ctx context.Context) (map[stores.ID]string, error return out, nil } -var _ sealmgr.Manager = &Manager{} +var _ SectorManager = &Manager{} diff --git a/storage/sbmock/sbmock.go b/storage/sectorstorage/mock/mock.go similarity index 74% rename from storage/sbmock/sbmock.go rename to storage/sectorstorage/mock/mock.go index 7708578bb..fdaae7f80 100644 --- a/storage/sbmock/sbmock.go +++ b/storage/sectorstorage/mock/mock.go @@ -1,9 +1,10 @@ -package sbmock +package mock import ( "bytes" "context" "fmt" + "github.com/filecoin-project/lotus/storage/sectorstorage" "io" "io/ioutil" "math/big" @@ -23,7 +24,7 @@ import ( var log = logging.Logger("sbmock") -type SBMock struct { +type SectorMgr struct { sectors map[abi.SectorID]*sectorState sectorSize abi.SectorSize nextSectorID abi.SectorNumber @@ -35,13 +36,13 @@ type SBMock struct { type mockVerif struct{} -func NewMockSectorBuilder(threads int, ssize abi.SectorSize) *SBMock { +func NewMockSectorMgr(threads int, ssize abi.SectorSize) *SectorMgr { rt, _, err := api.ProofTypeFromSectorSize(ssize) if err != nil { panic(err) } - return &SBMock{ + return &SectorMgr{ sectors: make(map[abi.SectorID]*sectorState), sectorSize: ssize, nextSectorID: 5, @@ -65,7 +66,7 @@ type sectorState struct { lk sync.Mutex } -func (sb *SBMock) RateLimit() func() { +func (sb *SectorMgr) RateLimit() func() { sb.rateLimit <- struct{}{} // TODO: probably want to copy over rate limit code @@ -74,11 +75,11 @@ func (sb *SBMock) RateLimit() func() { } } -func (sb *SBMock) NewSector(ctx context.Context, sector abi.SectorID) error { +func (sb *SectorMgr) NewSector(ctx context.Context, sector abi.SectorID) error { return nil } -func (sb *SBMock) AddPiece(ctx context.Context, sectorId abi.SectorID, existingPieces []abi.UnpaddedPieceSize, size abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { +func (sb *SectorMgr) AddPiece(ctx context.Context, sectorId abi.SectorID, existingPieces []abi.UnpaddedPieceSize, size abi.UnpaddedPieceSize, r io.Reader) (abi.PieceInfo, error) { log.Warn("Add piece: ", sectorId, size, sb.proofType) sb.lk.Lock() ss, ok := sb.sectors[sectorId] @@ -106,11 +107,11 @@ func (sb *SBMock) AddPiece(ctx context.Context, sectorId abi.SectorID, existingP }, nil } -func (sb *SBMock) SectorSize() abi.SectorSize { +func (sb *SectorMgr) SectorSize() abi.SectorSize { return sb.sectorSize } -func (sb *SBMock) AcquireSectorNumber() (abi.SectorNumber, error) { +func (sb *SectorMgr) AcquireSectorNumber() (abi.SectorNumber, error) { sb.lk.Lock() defer sb.lk.Unlock() id := sb.nextSectorID @@ -118,7 +119,7 @@ func (sb *SBMock) AcquireSectorNumber() (abi.SectorNumber, error) { return id, nil } -func (sb *SBMock) SealPreCommit1(ctx context.Context, sid abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { +func (sb *SectorMgr) SealPreCommit1(ctx context.Context, sid abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { sb.lk.Lock() ss, ok := sb.sectors[sid] sb.lk.Unlock() @@ -173,7 +174,7 @@ func (sb *SBMock) SealPreCommit1(ctx context.Context, sid abi.SectorID, ticket a return cc, nil } -func (sb *SBMock) SealPreCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.PreCommit1Out) (cids storage.SectorCids, err error) { +func (sb *SectorMgr) SealPreCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.PreCommit1Out) (cids storage.SectorCids, err error) { db := []byte(string(phase1Out)) db[0] ^= 'd' @@ -192,7 +193,7 @@ func (sb *SBMock) SealPreCommit2(ctx context.Context, sid abi.SectorID, phase1Ou }, nil } -func (sb *SBMock) SealCommit1(ctx context.Context, sid abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (output storage.Commit1Out, err error) { +func (sb *SectorMgr) SealCommit1(ctx context.Context, sid abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (output storage.Commit1Out, err error) { sb.lk.Lock() ss, ok := sb.sectors[sid] sb.lk.Unlock() @@ -220,7 +221,7 @@ func (sb *SBMock) SealCommit1(ctx context.Context, sid abi.SectorID, ticket abi. return out[:], nil } -func (sb *SBMock) SealCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.Commit1Out) (proof storage.Proof, err error) { +func (sb *SectorMgr) SealCommit2(ctx context.Context, sid abi.SectorID, phase1Out storage.Commit1Out) (proof storage.Proof, err error) { var out [32]byte for i := range out { out[i] = phase1Out[i] ^ byte(sid.Number&0xff) @@ -231,7 +232,7 @@ func (sb *SBMock) SealCommit2(ctx context.Context, sid abi.SectorID, phase1Out s // Test Instrumentation Methods -func (sb *SBMock) FailSector(sid abi.SectorID) error { +func (sb *SectorMgr) FailSector(sid abi.SectorID) error { sb.lk.Lock() defer sb.lk.Unlock() ss, ok := sb.sectors[sid] @@ -259,15 +260,15 @@ func AddOpFinish(ctx context.Context) (context.Context, func()) { } } -func (sb *SBMock) GenerateFallbackPoSt(context.Context, abi.ActorID, []abi.SectorInfo, abi.PoStRandomness, []abi.SectorNumber) (storage.FallbackPostOut, error) { +func (sb *SectorMgr) GenerateFallbackPoSt(context.Context, abi.ActorID, []abi.SectorInfo, abi.PoStRandomness, []abi.SectorNumber) (storage.FallbackPostOut, error) { panic("implement me") } -func (sb *SBMock) ComputeElectionPoSt(ctx context.Context, mid abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) { +func (sb *SectorMgr) ComputeElectionPoSt(ctx context.Context, mid abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, winners []abi.PoStCandidate) ([]abi.PoStProof, error) { panic("implement me") } -func (sb *SBMock) GenerateEPostCandidates(ctx context.Context, mid abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) { +func (sb *SectorMgr) GenerateEPostCandidates(ctx context.Context, mid abi.ActorID, sectorInfo []abi.SectorInfo, challengeSeed abi.PoStRandomness, faults []abi.SectorNumber) ([]storage.PoStCandidateWithTicket, error) { if len(faults) > 0 { panic("todo") } @@ -297,14 +298,14 @@ func (sb *SBMock) GenerateEPostCandidates(ctx context.Context, mid abi.ActorID, return out, nil } -func (sb *SBMock) ReadPieceFromSealedSector(ctx context.Context, sectorID abi.SectorID, offset sectorbuilder.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, commD cid.Cid) (io.ReadCloser, error) { +func (sb *SectorMgr) ReadPieceFromSealedSector(ctx context.Context, sectorID abi.SectorID, offset sectorbuilder.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, commD cid.Cid) (io.ReadCloser, error) { if len(sb.sectors[sectorID].pieces) > 1 { panic("implme") } return ioutil.NopCloser(io.LimitReader(bytes.NewReader(sb.sectors[sectorID].pieces[0].Bytes()[offset:]), int64(size))), nil } -func (sb *SBMock) StageFakeData(mid abi.ActorID) (abi.SectorID, []abi.PieceInfo, error) { +func (sb *SectorMgr) StageFakeData(mid abi.ActorID) (abi.SectorID, []abi.PieceInfo, error) { usize := abi.PaddedPieceSize(sb.sectorSize).Unpadded() sid, err := sb.AcquireSectorNumber() if err != nil { @@ -327,7 +328,7 @@ func (sb *SBMock) StageFakeData(mid abi.ActorID) (abi.SectorID, []abi.PieceInfo, return id, []abi.PieceInfo{pi}, nil } -func (sb *SBMock) FinalizeSector(context.Context, abi.SectorID) error { +func (sb *SectorMgr) FinalizeSector(context.Context, abi.SectorID) error { return nil } @@ -367,4 +368,4 @@ func (m mockVerif) GenerateDataCommitment(ssize abi.PaddedPieceSize, pieces []ab var MockVerifier = mockVerif{} var _ sectorbuilder.Verifier = MockVerifier -var _ sectorbuilder.Basic = &SBMock{} +var _ sectorstorage.SectorManager = &SectorMgr{} diff --git a/storage/sbmock/sbmock_test.go b/storage/sectorstorage/mock/mock_test.go similarity index 93% rename from storage/sbmock/sbmock_test.go rename to storage/sectorstorage/mock/mock_test.go index 1d6b1a4f6..524e8d615 100644 --- a/storage/sbmock/sbmock_test.go +++ b/storage/sectorstorage/mock/mock_test.go @@ -1,4 +1,4 @@ -package sbmock +package mock import ( "context" @@ -9,7 +9,7 @@ import ( ) func TestOpFinish(t *testing.T) { - sb := NewMockSectorBuilder(1, 2048) + sb := NewMockSectorMgr(1, 2048) sid, pieces, err := sb.StageFakeData(123) if err != nil { diff --git a/storage/sbmock/preseal.go b/storage/sectorstorage/mock/preseal.go similarity index 99% rename from storage/sbmock/preseal.go rename to storage/sectorstorage/mock/preseal.go index 4cc790c66..6bac0aaea 100644 --- a/storage/sbmock/preseal.go +++ b/storage/sectorstorage/mock/preseal.go @@ -1,4 +1,4 @@ -package sbmock +package mock import ( "github.com/filecoin-project/go-address" diff --git a/storage/sbmock/util.go b/storage/sectorstorage/mock/util.go similarity index 94% rename from storage/sbmock/util.go rename to storage/sectorstorage/mock/util.go index 4ce76da30..e37cf3552 100644 --- a/storage/sbmock/util.go +++ b/storage/sectorstorage/mock/util.go @@ -1,4 +1,4 @@ -package sbmock +package mock import ( "crypto/rand" diff --git a/storage/sealmgr/advmgr/resources.go b/storage/sectorstorage/resources.go similarity index 88% rename from storage/sealmgr/advmgr/resources.go rename to storage/sectorstorage/resources.go index a80435ef9..3587b41ea 100644 --- a/storage/sealmgr/advmgr/resources.go +++ b/storage/sectorstorage/resources.go @@ -1,8 +1,8 @@ -package advmgr +package sectorstorage import ( "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/lotus/storage/sealmgr" + "github.com/filecoin-project/lotus/storage/sectorstorage/sealtasks" "github.com/filecoin-project/specs-actors/actors/abi" ) @@ -30,8 +30,8 @@ type Resources struct { const MaxCachingOverhead = 32 << 30 -var ResourceTable = map[sealmgr.TaskType]map[abi.RegisteredProof]Resources{ - sealmgr.TTAddPiece: { +var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredProof]Resources{ + sealtasks.TTAddPiece: { abi.RegisteredProof_StackedDRG32GiBSeal: Resources{ // This is probably a bit conservative MaxMemory: 32 << 30, MinMemory: 32 << 30, @@ -49,7 +49,7 @@ var ResourceTable = map[sealmgr.TaskType]map[abi.RegisteredProof]Resources{ BaseMinMemory: 1 << 30, }, }, - sealmgr.TTPreCommit1: { + sealtasks.TTPreCommit1: { abi.RegisteredProof_StackedDRG32GiBSeal: Resources{ MaxMemory: 64 << 30, MinMemory: 32 << 30, @@ -67,7 +67,7 @@ var ResourceTable = map[sealmgr.TaskType]map[abi.RegisteredProof]Resources{ BaseMinMemory: 1 << 30, }, }, - sealmgr.TTPreCommit2: { + sealtasks.TTPreCommit2: { abi.RegisteredProof_StackedDRG32GiBSeal: Resources{ MaxMemory: 96 << 30, MinMemory: 64 << 30, @@ -85,7 +85,7 @@ var ResourceTable = map[sealmgr.TaskType]map[abi.RegisteredProof]Resources{ BaseMinMemory: 1 << 30, }, }, - sealmgr.TTCommit1: { // Very short (~100ms), so params are very light + sealtasks.TTCommit1: { // Very short (~100ms), so params are very light abi.RegisteredProof_StackedDRG32GiBSeal: Resources{ MaxMemory: 1 << 30, MinMemory: 1 << 30, @@ -103,7 +103,7 @@ var ResourceTable = map[sealmgr.TaskType]map[abi.RegisteredProof]Resources{ BaseMinMemory: 1 << 30, }, }, - sealmgr.TTCommit2: { // TODO: Measure more accurately + sealtasks.TTCommit2: { // TODO: Measure more accurately abi.RegisteredProof_StackedDRG32GiBSeal: Resources{ MaxMemory: 110 << 30, MinMemory: 60 << 30, diff --git a/storage/sealmgr/advmgr/roprov.go b/storage/sectorstorage/roprov.go similarity index 87% rename from storage/sealmgr/advmgr/roprov.go rename to storage/sectorstorage/roprov.go index 32e0c9ba4..99723e181 100644 --- a/storage/sealmgr/advmgr/roprov.go +++ b/storage/sectorstorage/roprov.go @@ -1,10 +1,10 @@ -package advmgr +package sectorstorage import ( "context" "github.com/filecoin-project/go-sectorbuilder" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" "github.com/filecoin-project/specs-actors/actors/abi" "golang.org/x/xerrors" diff --git a/storage/sealmgr/advmgr/sched.go b/storage/sectorstorage/sched.go similarity index 97% rename from storage/sealmgr/advmgr/sched.go rename to storage/sectorstorage/sched.go index 328656a0d..adf2e6cd3 100644 --- a/storage/sealmgr/advmgr/sched.go +++ b/storage/sectorstorage/sched.go @@ -1,17 +1,17 @@ -package advmgr +package sectorstorage import ( + "github.com/filecoin-project/lotus/storage/sectorstorage/sealtasks" "github.com/filecoin-project/specs-actors/actors/abi" "golang.org/x/xerrors" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/storage/sealmgr" ) const mib = 1 << 20 type workerRequest struct { - taskType sealmgr.TaskType + taskType sealtasks.TaskType accept []workerID // ordered by preference ret chan<- workerResponse diff --git a/storage/sealmgr/task.go b/storage/sectorstorage/sealtasks/task.go similarity index 95% rename from storage/sealmgr/task.go rename to storage/sectorstorage/sealtasks/task.go index fd35a1204..8eefa14fa 100644 --- a/storage/sealmgr/task.go +++ b/storage/sectorstorage/sealtasks/task.go @@ -1,4 +1,4 @@ -package sealmgr +package sealtasks type TaskType string diff --git a/storage/sealmgr/sectorutil/utils.go b/storage/sectorstorage/sectorutil/utils.go similarity index 100% rename from storage/sealmgr/sectorutil/utils.go rename to storage/sectorstorage/sectorutil/utils.go diff --git a/storage/sealmgr/stores/http_handler.go b/storage/sectorstorage/stores/http_handler.go similarity index 97% rename from storage/sealmgr/stores/http_handler.go rename to storage/sectorstorage/stores/http_handler.go index f4e519cc6..daa81061e 100644 --- a/storage/sealmgr/stores/http_handler.go +++ b/storage/sectorstorage/stores/http_handler.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/lib/tarutil" - "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" + "github.com/filecoin-project/lotus/storage/sectorstorage/sectorutil" ) var log = logging.Logger("stores") diff --git a/storage/sealmgr/stores/index.go b/storage/sectorstorage/stores/index.go similarity index 99% rename from storage/sealmgr/stores/index.go rename to storage/sectorstorage/stores/index.go index d77c1b811..e508171b7 100644 --- a/storage/sealmgr/stores/index.go +++ b/storage/sectorstorage/stores/index.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/abi/big" - "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" + "github.com/filecoin-project/lotus/storage/sectorstorage/sectorutil" ) // ID identifies sector storage by UUID. One sector storage should map to one diff --git a/storage/sealmgr/stores/interface.go b/storage/sectorstorage/stores/interface.go similarity index 100% rename from storage/sealmgr/stores/interface.go rename to storage/sectorstorage/stores/interface.go diff --git a/storage/sealmgr/stores/local.go b/storage/sectorstorage/stores/local.go similarity index 99% rename from storage/sealmgr/stores/local.go rename to storage/sectorstorage/stores/local.go index 304c64ba4..a8eb53ee8 100644 --- a/storage/sealmgr/stores/local.go +++ b/storage/sectorstorage/stores/local.go @@ -14,7 +14,7 @@ import ( "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/lotus/node/config" - "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" + "github.com/filecoin-project/lotus/storage/sectorstorage/sectorutil" ) type StoragePath struct { diff --git a/storage/sealmgr/stores/remote.go b/storage/sectorstorage/stores/remote.go similarity index 98% rename from storage/sealmgr/stores/remote.go rename to storage/sectorstorage/stores/remote.go index fca680e96..7bbd6d225 100644 --- a/storage/sealmgr/stores/remote.go +++ b/storage/sectorstorage/stores/remote.go @@ -16,7 +16,7 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/lotus/lib/tarutil" - "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" + "github.com/filecoin-project/lotus/storage/sectorstorage/sectorutil" ) type Remote struct { diff --git a/storage/sealmgr/advmgr/worker_local.go b/storage/sectorstorage/worker_local.go similarity index 91% rename from storage/sealmgr/advmgr/worker_local.go rename to storage/sectorstorage/worker_local.go index f62ad58c9..d691f150e 100644 --- a/storage/sealmgr/advmgr/worker_local.go +++ b/storage/sectorstorage/worker_local.go @@ -1,4 +1,4 @@ -package advmgr +package sectorstorage import ( "context" @@ -14,16 +14,16 @@ import ( storage2 "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/storage/sealmgr" - "github.com/filecoin-project/lotus/storage/sealmgr/sectorutil" - "github.com/filecoin-project/lotus/storage/sealmgr/stores" + "github.com/filecoin-project/lotus/storage/sectorstorage/sealtasks" + "github.com/filecoin-project/lotus/storage/sectorstorage/sectorutil" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) var pathTypes = []sectorbuilder.SectorFileType{sectorbuilder.FTUnsealed, sectorbuilder.FTSealed, sectorbuilder.FTCache} type WorkerConfig struct { SealProof abi.RegisteredProof - TaskTypes []sealmgr.TaskType + TaskTypes []sealtasks.TaskType } type LocalWorker struct { @@ -32,7 +32,7 @@ type LocalWorker struct { localStore *stores.Local sindex stores.SectorIndex - acceptTasks map[sealmgr.TaskType]struct{} + acceptTasks map[sealtasks.TaskType]struct{} } func NewLocalWorker(wcfg WorkerConfig, store stores.Store, local *stores.Local, sindex stores.SectorIndex) *LocalWorker { @@ -41,7 +41,7 @@ func NewLocalWorker(wcfg WorkerConfig, store stores.Store, local *stores.Local, panic(err) } - acceptTasks := map[sealmgr.TaskType]struct{}{} + acceptTasks := map[sealtasks.TaskType]struct{}{} for _, taskType := range wcfg.TaskTypes { acceptTasks[taskType] = struct{}{} } @@ -155,7 +155,7 @@ func (l *LocalWorker) FinalizeSector(ctx context.Context, sector abi.SectorID) e return sb.FinalizeSector(ctx, sector) } -func (l *LocalWorker) TaskTypes(context.Context) (map[sealmgr.TaskType]struct{}, error) { +func (l *LocalWorker) TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) { return l.acceptTasks, nil } diff --git a/storage/sealmgr/advmgr/worker_remote.go b/storage/sectorstorage/worker_remote.go similarity index 98% rename from storage/sealmgr/advmgr/worker_remote.go rename to storage/sectorstorage/worker_remote.go index 734e14ebc..f49ea4dc6 100644 --- a/storage/sealmgr/advmgr/worker_remote.go +++ b/storage/sectorstorage/worker_remote.go @@ -1,4 +1,4 @@ -package advmgr +package sectorstorage import ( "context" From cb1e7bd1b13964859338f2aaf826f2c12d01add7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 23 Mar 2020 13:12:43 +0100 Subject: [PATCH 34/45] sector index: Don't add duplicate storage URLs --- storage/sectorstorage/stores/index.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/storage/sectorstorage/stores/index.go b/storage/sectorstorage/stores/index.go index e508171b7..da7c8212e 100644 --- a/storage/sectorstorage/stores/index.go +++ b/storage/sectorstorage/stores/index.go @@ -104,7 +104,17 @@ func (i *Index) StorageAttach(ctx context.Context, si StorageInfo, st FsStat) er } } - i.stores[si.ID].info.URLs = append(i.stores[si.ID].info.URLs, si.URLs...) + uloop: + for _, u := range si.URLs { + for _, l := range i.stores[si.ID].info.URLs { + if u == l { + continue uloop + } + } + + i.stores[si.ID].info.URLs = append(i.stores[si.ID].info.URLs, u) + } + return nil } i.stores[si.ID] = &storageEntry{ @@ -127,7 +137,7 @@ func (i *Index) StorageDeclareSector(ctx context.Context, storageId ID, s abi.Se for _, sid := range i.sectors[d] { if sid == storageId { - log.Warnf("sector %v redeclared in %s", storageId) + log.Warnf("sector %v redeclared in %s", s, storageId) return nil } } From 80d0f1f2c1f0c519f70c7b523ab231117274240e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 23 Mar 2020 13:29:24 +0100 Subject: [PATCH 35/45] storageminer: Cleanup CLI --- cli/cmd.go | 8 ++++++++ cmd/lotus-storage-miner/main.go | 3 +-- cmd/lotus-storage-miner/sectors.go | 27 ++++++++++++++------------- cmd/lotus-storage-miner/workers.go | 1 + 4 files changed, 24 insertions(+), 15 deletions(-) create mode 100644 cmd/lotus-storage-miner/workers.go diff --git a/cli/cmd.go b/cli/cmd.go index fc312f191..e5b8faa82 100644 --- a/cli/cmd.go +++ b/cli/cmd.go @@ -209,6 +209,14 @@ func ReqContext(cctx *cli.Context) context.Context { return ctx } +var CommonCommands = []*cli.Command{ + authCmd, + fetchParamCmd, + netCmd, + versionCmd, + logCmd, +} + var Commands = []*cli.Command{ authCmd, chainCmd, diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index 4549332e0..f0480de89 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -25,7 +25,6 @@ func main() { dealsCmd, infoCmd, initCmd, - pledgeSectorCmd, rewardsCmd, runCmd, sectorsCmd, @@ -71,7 +70,7 @@ func main() { }, }, - Commands: append(local, lcli.Commands...), + Commands: append(local, lcli.CommonCommands...), } app.Setup() app.Metadata["repoType"] = repo.StorageMiner diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 8e3d6a852..75add6ea7 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -18,8 +18,20 @@ import ( lcli "github.com/filecoin-project/lotus/cli" ) -var pledgeSectorCmd = &cli.Command{ - Name: "pledge-sector", +var sectorsCmd = &cli.Command{ + Name: "sectors", + Usage: "interact with sector store", + Subcommands: []*cli.Command{ + sectorsStatusCmd, + sectorsListCmd, + sectorsRefsCmd, + sectorsUpdateCmd, + sectorsPledgeCmd, + }, +} + +var sectorsPledgeCmd = &cli.Command{ + Name: "pledge", Usage: "store random data in a sector", Action: func(cctx *cli.Context) error { nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) @@ -33,17 +45,6 @@ var pledgeSectorCmd = &cli.Command{ }, } -var sectorsCmd = &cli.Command{ - Name: "sectors", - Usage: "interact with sector store", - Subcommands: []*cli.Command{ - sectorsStatusCmd, - sectorsListCmd, - sectorsRefsCmd, - sectorsUpdateCmd, - }, -} - var sectorsStatusCmd = &cli.Command{ Name: "status", Usage: "Get the seal status of a sector by its ID", diff --git a/cmd/lotus-storage-miner/workers.go b/cmd/lotus-storage-miner/workers.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/cmd/lotus-storage-miner/workers.go @@ -0,0 +1 @@ +package main From a3ba8eb0d7353e1aaccad7cded24efb108c33a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 23 Mar 2020 15:56:22 +0100 Subject: [PATCH 36/45] workers: Basic monitoring tools --- api/api_storage.go | 2 + api/api_worker.go | 9 ++++ api/apistruct/struct.go | 6 +++ cmd/lotus-storage-miner/main.go | 1 + cmd/lotus-storage-miner/storage.go | 20 ++++++-- cmd/lotus-storage-miner/workers.go | 80 ++++++++++++++++++++++++++++++ node/impl/storminer.go | 9 ++-- scripts/miner-mon.sh | 23 +++++++++ storage/sectorstorage/manager.go | 22 ++++---- storage/sectorstorage/sched.go | 8 +-- storage/sectorstorage/stats.go | 22 ++++++++ 11 files changed, 177 insertions(+), 25 deletions(-) create mode 100755 scripts/miner-mon.sh create mode 100644 storage/sectorstorage/stats.go diff --git a/api/api_storage.go b/api/api_storage.go index 9f032ecb2..d8c33f4f7 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -113,6 +113,8 @@ type StorageMiner interface { // WorkerConnect tells the node to connect to workers RPC WorkerConnect(context.Context, string) error + WorkerStats(context.Context) (map[uint64]WorkerStats, error) + stores.SectorIndex MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error diff --git a/api/api_worker.go b/api/api_worker.go index 999120643..113952adb 100644 --- a/api/api_worker.go +++ b/api/api_worker.go @@ -35,3 +35,12 @@ type WorkerInfo struct { Resources WorkerResources } + +type WorkerStats struct { + Info WorkerInfo + + MemUsedMin uint64 + MemUsedMax uint64 + GpuUsed bool + CpuUse int +} diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index b1a6b265e..0a2c297b7 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -182,6 +182,8 @@ type StorageMinerStruct struct { SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error `perm:"write"` WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm + WorkerStats func(context.Context) (map[uint64]api.WorkerStats, error) `perm:"admin"` + StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` @@ -658,6 +660,10 @@ func (c *StorageMinerStruct) WorkerConnect(ctx context.Context, url string) erro return c.Internal.WorkerConnect(ctx, url) } +func (c *StorageMinerStruct) WorkerStats(ctx context.Context) (map[uint64]api.WorkerStats, error) { + return c.Internal.WorkerStats(ctx) +} + func (c *StorageMinerStruct) StorageAttach(ctx context.Context, si stores.StorageInfo, st stores.FsStat) error { return c.Internal.StorageAttach(ctx, si, st) } diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index f0480de89..2ed147531 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -30,6 +30,7 @@ func main() { sectorsCmd, storageCmd, setPriceCmd, + workersCmd, } jaeger := tracing.SetupJaegerTracing("lotus") defer func() { diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 417c5b83a..397894edf 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -134,9 +134,19 @@ var storageListCmd = &cli.Command{ return err } - for id, sectors := range st { + sorted := make([]struct{stores.ID; sectors []stores.Decl}, 0, len(st)) + for id, decls := range st { + sorted = append(sorted, struct{stores.ID; sectors []stores.Decl}{id, decls}) + } + + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].ID < sorted[j].ID + }) + + for _, s := range sorted { + var cnt [3]int - for _, decl := range sectors { + for _, decl := range s.sectors { for i := range cnt { if decl.SectorFileType&(1< Date: Mon, 23 Mar 2020 16:40:36 +0100 Subject: [PATCH 37/45] sectorstorage: Fix remote sector deletion --- storage/sectorstorage/stores/local.go | 8 ++++++++ storage/sectorstorage/stores/remote.go | 16 ++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/storage/sectorstorage/stores/local.go b/storage/sectorstorage/stores/local.go index a8eb53ee8..64924ad00 100644 --- a/storage/sectorstorage/stores/local.go +++ b/storage/sectorstorage/stores/local.go @@ -280,6 +280,10 @@ func (st *Local) delete(ctx context.Context, sid abi.SectorID, typ sectorbuilder return xerrors.Errorf("finding existing sector %d(t:%d) failed: %w", sid, typ, err) } + if len(si) == 0 { + return xerrors.Errorf("can't delete sector %v(%d), not found", sid, typ) + } + for _, info := range si { p, ok := st.paths[info.ID] if !ok { @@ -290,6 +294,10 @@ func (st *Local) delete(ctx context.Context, sid abi.SectorID, typ sectorbuilder continue } + if err := st.index.StorageDropSector(ctx, info.ID, sid, typ); err != nil { + return xerrors.Errorf("dropping sector from index: %w", err) + } + spath := filepath.Join(p.local, typ.String(), sectorutil.SectorName(sid)) log.Infof("remove %s", spath) diff --git a/storage/sectorstorage/stores/remote.go b/storage/sectorstorage/stores/remote.go index 7bbd6d225..d2ce1626a 100644 --- a/storage/sectorstorage/stores/remote.go +++ b/storage/sectorstorage/stores/remote.go @@ -59,7 +59,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec continue } - ap, storageID, url, foundIn, rdone, err := r.acquireFromRemote(ctx, s, fileType, sealing) + ap, storageID, url, rdone, err := r.acquireFromRemote(ctx, s, fileType, sealing) if err != nil { done() return sectorbuilder.SectorPaths{}, sectorbuilder.SectorPaths{}, nil, err @@ -75,10 +75,6 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec } // TODO: some way to allow having duplicated sectors in the system for perf - if err := r.index.StorageDropSector(ctx, foundIn, s, fileType); err != nil { - log.Warnf("dropping sector %v from %s from sector index failed: %+v", s, storageID, err) - } - if err := r.deleteFromRemote(url); err != nil { log.Warnf("deleting sector %v from %s (delete %s): %+v", s, storageID, url, err) } @@ -87,10 +83,10 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec return paths, stores, done, nil } -func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, ID, string, ID, func(), error) { +func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType sectorbuilder.SectorFileType, sealing bool) (string, ID, string, func(), error) { si, err := r.index.StorageFindSector(ctx, s, fileType, false) if err != nil { - return "", "", "", "", nil, err + return "", "", "", nil, err } sort.Slice(si, func(i, j int) bool { @@ -99,7 +95,7 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType apaths, ids, done, err := r.local.AcquireSector(ctx, s, 0, fileType, sealing) if err != nil { - return "", "", "", "", nil, xerrors.Errorf("allocate local sector for fetching: %w", err) + return "", "", "", nil, xerrors.Errorf("allocate local sector for fetching: %w", err) } dest := sectorutil.PathByType(apaths, fileType) storageID := sectorutil.PathByType(ids, fileType) @@ -116,12 +112,12 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType if merr != nil { log.Warnw("acquireFromRemote encountered errors when fetching sector from remote", "errors", merr) } - return dest, ID(storageID), url, info.ID, done, nil + return dest, ID(storageID), url, done, nil } } done() - return "", "", "", "", nil, xerrors.Errorf("failed to acquire sector %v from remote (tried %v): %w", s, si, merr) + return "", "", "", nil, xerrors.Errorf("failed to acquire sector %v from remote (tried %v): %w", s, si, merr) } func (r *Remote) fetch(url, outname string) error { From 8d1fab1e05ed18407dd15e222a58f6365d219eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 23 Mar 2020 17:28:40 +0100 Subject: [PATCH 38/45] sectorstorage: Enable sealing on the local worker by default --- storage/sectorstorage/manager.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/storage/sectorstorage/manager.go b/storage/sectorstorage/manager.go index d465948b2..6a38c3f4d 100644 --- a/storage/sectorstorage/manager.go +++ b/storage/sectorstorage/manager.go @@ -115,7 +115,10 @@ func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Confi err = m.AddWorker(ctx, NewLocalWorker(WorkerConfig{ SealProof: cfg.SealProofType, - TaskTypes: []sealtasks.TaskType{sealtasks.TTAddPiece, sealtasks.TTCommit1, sealtasks.TTFinalize}, + TaskTypes: []sealtasks.TaskType{ + sealtasks.TTAddPiece, sealtasks.TTCommit1, sealtasks.TTFinalize, + sealtasks.TTPreCommit1, sealtasks.TTPreCommit2, sealtasks.TTCommit2, // TODO: Config + }, }, stor, lstor, si)) if err != nil { return nil, xerrors.Errorf("adding local worker: %w", err) From 8559c895602e150899b04dfba7f9f2b958cc2508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 23 Mar 2020 23:43:38 +0100 Subject: [PATCH 39/45] storageminer: More storage stats in storage list --- api/api_storage.go | 1 + api/apistruct/struct.go | 9 ++- cmd/lotus-storage-miner/storage.go | 33 +++++++-- cmd/lotus-storage-miner/workers.go | 20 ++++-- node/impl/storminer.go | 5 +- storage/sectorstorage/manager.go | 4 ++ storage/sectorstorage/stores/http_handler.go | 25 +++++++ storage/sectorstorage/stores/index.go | 4 +- storage/sectorstorage/stores/interface.go | 16 +++-- storage/sectorstorage/stores/local.go | 4 +- storage/sectorstorage/stores/remote.go | 76 ++++++++++++++++++-- 11 files changed, 169 insertions(+), 28 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index d8c33f4f7..0f3b07bd5 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -110,6 +110,7 @@ type StorageMiner interface { StorageList(ctx context.Context) (map[stores.ID][]stores.Decl, error) StorageLocal(ctx context.Context) (map[stores.ID]string, error) + StorageStat(ctx context.Context, id stores.ID) (stores.FsStat, error) // WorkerConnect tells the node to connect to workers RPC WorkerConnect(context.Context, string) error diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 0a2c297b7..b6bc3f1b3 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -181,11 +181,12 @@ type StorageMinerStruct struct { SectorsRefs func(context.Context) (map[string][]api.SealedRef, error) `perm:"read"` SectorsUpdate func(context.Context, abi.SectorNumber, api.SectorState) error `perm:"write"` - WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm - WorkerStats func(context.Context) (map[uint64]api.WorkerStats, error) `perm:"admin"` + WorkerConnect func(context.Context, string) error `perm:"admin"` // TODO: worker perm + WorkerStats func(context.Context) (map[uint64]api.WorkerStats, error) `perm:"admin"` StorageList func(context.Context) (map[stores.ID][]stores.Decl, error) `perm:"admin"` StorageLocal func(context.Context) (map[stores.ID]string, error) `perm:"admin"` + StorageStat func(context.Context, stores.ID) (stores.FsStat, error) `perm:"admin"` StorageAttach func(context.Context, stores.StorageInfo, stores.FsStat) error `perm:"admin"` StorageDeclareSector func(context.Context, stores.ID, abi.SectorID, sectorbuilder.SectorFileType) error `perm:"admin"` StorageDropSector func(context.Context, stores.ID, abi.SectorID, sectorbuilder.SectorFileType) error `perm:"admin"` @@ -688,6 +689,10 @@ func (c *StorageMinerStruct) StorageLocal(ctx context.Context) (map[stores.ID]st return c.Internal.StorageLocal(ctx) } +func (c *StorageMinerStruct) StorageStat(ctx context.Context, id stores.ID) (stores.FsStat, error) { + return c.Internal.StorageStat(ctx, id) +} + func (c *StorageMinerStruct) StorageInfo(ctx context.Context, id stores.ID) (stores.StorageInfo, error) { return c.Internal.StorageInfo(ctx, id) } diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 397894edf..98cc96754 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sort" "strconv" + "time" "github.com/google/uuid" "github.com/mitchellh/go-homedir" @@ -18,6 +19,7 @@ import ( "github.com/filecoin-project/go-sectorbuilder" "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) @@ -134,9 +136,15 @@ var storageListCmd = &cli.Command{ return err } - sorted := make([]struct{stores.ID; sectors []stores.Decl}, 0, len(st)) + sorted := make([]struct { + stores.ID + sectors []stores.Decl + }, 0, len(st)) for id, decls := range st { - sorted = append(sorted, struct{stores.ID; sectors []stores.Decl}{id, decls}) + sorted = append(sorted, struct { + stores.ID + sectors []stores.Decl + }{id, decls}) } sort.Slice(sorted, func(i, j int) bool { @@ -154,8 +162,20 @@ var storageListCmd = &cli.Command{ } } + pingStart := time.Now() + st, err := nodeApi.StorageStat(ctx, s.ID) + if err != nil { + return err + } + ping := time.Now().Sub(pingStart) + fmt.Printf("%s:\n", s.ID) fmt.Printf("\tUnsealed: %d; Sealed: %d; Caches: %d\n", cnt[0], cnt[1], cnt[2]) + fmt.Printf("\tSpace Used: %s/%s %d%% (%s avail)\n", + types.SizeStr(types.NewInt(st.Capacity-st.Available)), + types.SizeStr(types.NewInt(st.Capacity)), + (st.Capacity-st.Available)*100/st.Capacity, + types.SizeStr(types.NewInt(st.Available))) si, err := nodeApi.StorageInfo(ctx, s.ID) if err != nil { @@ -179,8 +199,13 @@ var storageListCmd = &cli.Command{ if localPath, ok := local[s.ID]; ok { fmt.Printf("\tLocal: %s\n", localPath) } - for _, l := range si.URLs { - fmt.Printf("\tURL: %s\n", l) // TODO; try pinging maybe?? print latency? + for i, l := range si.URLs { + var rtt string + if _, ok := local[s.ID]; !ok && i == 0 { + rtt = " (latency: " + ping.Truncate(time.Microsecond*100).String() + ")" + } + + fmt.Printf("\tURL: %s%s\n", l, rtt) // TODO; try pinging maybe?? print latency? } } diff --git a/cmd/lotus-storage-miner/workers.go b/cmd/lotus-storage-miner/workers.go index 77f3cc929..a29de1c6b 100644 --- a/cmd/lotus-storage-miner/workers.go +++ b/cmd/lotus-storage-miner/workers.go @@ -35,9 +35,15 @@ var workersListCmd = &cli.Command{ return err } - st := make([]struct{id uint64; api.WorkerStats}, 0, len(stats)) + st := make([]struct { + id uint64 + api.WorkerStats + }, 0, len(stats)) for id, stat := range stats { - st = append(st, struct{id uint64; api.WorkerStats}{id, stat}) + st = append(st, struct { + id uint64 + api.WorkerStats + }{id, stat}) } sort.Slice(st, func(i, j int) bool { @@ -66,16 +72,16 @@ var workersListCmd = &cli.Command{ types.SizeStr(types.NewInt(stat.Info.Resources.MemPhysical)), types.SizeStr(types.NewInt(stat.Info.Resources.MemSwap)), types.SizeStr(types.NewInt(stat.Info.Resources.MemReserved)), - stat.Info.Resources.MemReserved * 100 / stat.Info.Resources.MemPhysical) + stat.Info.Resources.MemReserved*100/stat.Info.Resources.MemPhysical) fmt.Printf("\t\tUsed: Physical %s (%d%% phys), Virtual %s (%d%% phys, %d%% virt)\n", types.SizeStr(types.NewInt(stat.MemUsedMin)), - stat.MemUsedMin * 100 / stat.Info.Resources.MemPhysical, + stat.MemUsedMin*100/stat.Info.Resources.MemPhysical, types.SizeStr(types.NewInt(stat.MemUsedMax)), - stat.MemUsedMax * 100 / stat.Info.Resources.MemPhysical, - stat.MemUsedMax * 100 / (stat.Info.Resources.MemPhysical + stat.Info.Resources.MemSwap)) + stat.MemUsedMax*100/stat.Info.Resources.MemPhysical, + stat.MemUsedMax*100/(stat.Info.Resources.MemPhysical+stat.Info.Resources.MemSwap)) } return nil }, -} \ No newline at end of file +} diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 2c29766a3..24027bba5 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -51,7 +51,6 @@ func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { sm.StorageMgr.ServeHTTP(w, r) } - func (sm *StorageMinerAPI) WorkerStats(context.Context) (map[uint64]api.WorkerStats, error) { return sm.StorageMgr.WorkerStats(), nil } @@ -142,6 +141,10 @@ func (sm *StorageMinerAPI) SectorsRefs(context.Context) (map[string][]api.Sealed return out, nil } +func (sm *StorageMinerAPI) StorageStat(ctx context.Context, id stores.ID) (stores.FsStat, error) { + return sm.StorageMgr.FsStat(ctx, id) +} + func (sm *StorageMinerAPI) SectorsUpdate(ctx context.Context, id abi.SectorNumber, state api.SectorState) error { return sm.Miner.ForceSectorState(ctx, id, state) } diff --git a/storage/sectorstorage/manager.go b/storage/sectorstorage/manager.go index 6a38c3f4d..f3aa90964 100644 --- a/storage/sectorstorage/manager.go +++ b/storage/sectorstorage/manager.go @@ -410,4 +410,8 @@ func (m *Manager) StorageLocal(ctx context.Context) (map[stores.ID]string, error return out, nil } +func (m *Manager) FsStat(ctx context.Context, id stores.ID) (stores.FsStat, error) { + return m.storage.FsStat(ctx, id) +} + var _ SectorManager = &Manager{} diff --git a/storage/sectorstorage/stores/http_handler.go b/storage/sectorstorage/stores/http_handler.go index daa81061e..97c6c34f6 100644 --- a/storage/sectorstorage/stores/http_handler.go +++ b/storage/sectorstorage/stores/http_handler.go @@ -1,6 +1,7 @@ package stores import ( + "encoding/json" "io" "net/http" "os" @@ -23,12 +24,36 @@ type FetchHandler struct { func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // /remote/ mux := mux.NewRouter() + mux.HandleFunc("/remote/stat/{id}", handler.remoteStatFs).Methods("GET") mux.HandleFunc("/remote/{type}/{id}", handler.remoteGetSector).Methods("GET") mux.HandleFunc("/remote/{type}/{id}", handler.remoteDeleteSector).Methods("DELETE") mux.ServeHTTP(w, r) } +func (handler *FetchHandler) remoteStatFs(w http.ResponseWriter, r *http.Request) { + log.Debugf("SERVE STAT %s", r.URL) + vars := mux.Vars(r) + id := ID(vars["id"]) + + st, err := handler.Local.FsStat(id) + switch err { + case errPathNotFound: + w.WriteHeader(404) + return + case nil: + break + default: + w.WriteHeader(500) + log.Errorf("%+v", err) + return + } + + if err := json.NewEncoder(w).Encode(&st); err != nil { + log.Warnf("error writing stat response: %+v", err) + } +} + func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Request) { log.Infof("SERVE GET %s", r.URL) vars := mux.Vars(r) diff --git a/storage/sectorstorage/stores/index.go b/storage/sectorstorage/stores/index.go index da7c8212e..ccad8ba7f 100644 --- a/storage/sectorstorage/stores/index.go +++ b/storage/sectorstorage/stores/index.go @@ -295,8 +295,8 @@ func (i *Index) StorageBestAlloc(ctx context.Context, allocate sectorbuilder.Sec } sort.Slice(candidates, func(i, j int) bool { - iw := big.Mul(big.NewInt(int64(candidates[i].fsi.Free)), big.NewInt(int64(candidates[i].info.Weight))) - jw := big.Mul(big.NewInt(int64(candidates[j].fsi.Free)), big.NewInt(int64(candidates[j].info.Weight))) + iw := big.Mul(big.NewInt(int64(candidates[i].fsi.Available)), big.NewInt(int64(candidates[i].info.Weight))) + jw := big.Mul(big.NewInt(int64(candidates[j].fsi.Available)), big.NewInt(int64(candidates[j].info.Weight))) return iw.GreaterThan(jw) }) diff --git a/storage/sectorstorage/stores/interface.go b/storage/sectorstorage/stores/interface.go index 67c18b16e..149cb9e5f 100644 --- a/storage/sectorstorage/stores/interface.go +++ b/storage/sectorstorage/stores/interface.go @@ -12,11 +12,7 @@ import ( type Store interface { AcquireSector(ctx context.Context, s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (paths sectorbuilder.SectorPaths, stores sectorbuilder.SectorPaths, done func(), err error) -} - -type FsStat struct { - Capacity uint64 - Free uint64 // Free to use for sector storage + FsStat(ctx context.Context, id ID) (FsStat, error) } func Stat(path string) (FsStat, error) { @@ -26,7 +22,13 @@ func Stat(path string) (FsStat, error) { } return FsStat{ - Capacity: stat.Blocks * uint64(stat.Bsize), - Free: stat.Bavail * uint64(stat.Bsize), + Capacity: stat.Blocks * uint64(stat.Bsize), + Available: stat.Bavail * uint64(stat.Bsize), }, nil } + +type FsStat struct { + Capacity uint64 + Available uint64 // Available to use for sector storage + Used uint64 +} diff --git a/storage/sectorstorage/stores/local.go b/storage/sectorstorage/stores/local.go index 64924ad00..581afb13f 100644 --- a/storage/sectorstorage/stores/local.go +++ b/storage/sectorstorage/stores/local.go @@ -309,13 +309,15 @@ func (st *Local) delete(ctx context.Context, sid abi.SectorID, typ sectorbuilder return nil } +var errPathNotFound = xerrors.Errorf("fsstat: path not found") + func (st *Local) FsStat(id ID) (FsStat, error) { st.localLk.RLock() defer st.localLk.RUnlock() p, ok := st.paths[id] if !ok { - return FsStat{}, xerrors.Errorf("fsstat: path not found") + return FsStat{}, errPathNotFound } return Stat(p.local) diff --git a/storage/sectorstorage/stores/remote.go b/storage/sectorstorage/stores/remote.go index d2ce1626a..a0648d972 100644 --- a/storage/sectorstorage/stores/remote.go +++ b/storage/sectorstorage/stores/remote.go @@ -2,9 +2,13 @@ package stores import ( "context" + "encoding/json" + "io/ioutil" "mime" "net/http" + "net/url" "os" + gopath "path" "sort" "sync" @@ -75,7 +79,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing sec } // TODO: some way to allow having duplicated sectors in the system for perf - if err := r.deleteFromRemote(url); err != nil { + if err := r.deleteFromRemote(ctx, url); err != nil { log.Warnf("deleting sector %v from %s (delete %s): %+v", s, storageID, url, err) } } @@ -103,7 +107,7 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType var merr error for _, info := range si { for _, url := range info.URLs { - err := r.fetch(url, dest) + err := r.fetch(ctx, url, dest) if err != nil { merr = multierror.Append(merr, xerrors.Errorf("fetch error %s (storage %s) -> %s: %w", url, info.ID, dest, err)) continue @@ -120,7 +124,7 @@ func (r *Remote) acquireFromRemote(ctx context.Context, s abi.SectorID, fileType return "", "", "", nil, xerrors.Errorf("failed to acquire sector %v from remote (tried %v): %w", s, si, merr) } -func (r *Remote) fetch(url, outname string) error { +func (r *Remote) fetch(ctx context.Context, url, outname string) error { log.Infof("Fetch %s -> %s", url, outname) req, err := http.NewRequest("GET", url, nil) @@ -128,6 +132,7 @@ func (r *Remote) fetch(url, outname string) error { return xerrors.Errorf("request: %w", err) } req.Header = r.auth + req = req.WithContext(ctx) resp, err := http.DefaultClient.Do(req) if err != nil { @@ -168,7 +173,7 @@ func (r *Remote) fetch(url, outname string) error { } } -func (r *Remote) deleteFromRemote(url string) error { +func (r *Remote) deleteFromRemote(ctx context.Context, url string) error { log.Infof("Delete %s", url) req, err := http.NewRequest("DELETE", url, nil) @@ -176,6 +181,7 @@ func (r *Remote) deleteFromRemote(url string) error { return xerrors.Errorf("request: %w", err) } req.Header = r.auth + req = req.WithContext(ctx) resp, err := http.DefaultClient.Do(req) if err != nil { @@ -190,6 +196,68 @@ func (r *Remote) deleteFromRemote(url string) error { return nil } +func (r *Remote) FsStat(ctx context.Context, id ID) (FsStat, error) { + st, err := r.local.FsStat(id) + switch err { + case nil: + return st, nil + case errPathNotFound: + break + default: + return FsStat{}, xerrors.Errorf("local stat: %w", err) + } + + si, err := r.index.StorageInfo(ctx, id) + if err != nil { + return FsStat{}, xerrors.Errorf("getting remote storage info: %w", err) + } + + if len(si.URLs) == 0 { + return FsStat{}, xerrors.Errorf("no known URLs for remote storage %s", id) + } + + rl, err := url.Parse(si.URLs[0]) + if err != nil { + return FsStat{}, xerrors.Errorf("failed to parse url: %w", err) + } + + rl.Path = gopath.Join(rl.Path, "stat", string(id)) + + req, err := http.NewRequest("GET", rl.String(), nil) + if err != nil { + return FsStat{}, xerrors.Errorf("request: %w", err) + } + req.Header = r.auth + req = req.WithContext(ctx) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return FsStat{}, xerrors.Errorf("do request: %w", err) + } + switch resp.StatusCode { + case 200: + break + case 404: + return FsStat{}, errPathNotFound + case 500: + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return FsStat{}, xerrors.Errorf("fsstat: got http 500, then failed to read the error: %w", err) + } + + return FsStat{}, xerrors.New(string(b)) + } + + var out FsStat + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return FsStat{}, xerrors.Errorf("decoding fsstat: %w", err) + } + + defer resp.Body.Close() + + return out, nil +} + func mergeDone(a func(), b func()) func() { return func() { a() From 726c9c1fe7f2261a43c5da35a2684ee1eb32e706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 24 Mar 2020 19:00:08 +0100 Subject: [PATCH 40/45] workers: Address review --- api/api_storage.go | 2 +- api/apistruct/struct.go | 6 +++--- api/client/client.go | 12 +++++++++--- cmd/lotus-seal-worker/main.go | 2 -- cmd/lotus-storage-miner/init.go | 2 +- cmd/lotus-storage-miner/market.go | 2 +- cmd/lotus-storage-miner/run.go | 11 +++++++++-- node/builder.go | 2 +- node/impl/storminer.go | 2 +- node/modules/storageminer.go | 7 +++++++ storage/sectorstorage/manager.go | 23 +++++++++++++++-------- 11 files changed, 48 insertions(+), 23 deletions(-) diff --git a/api/api_storage.go b/api/api_storage.go index 0f3b07bd5..4712ed98c 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -121,7 +121,7 @@ type StorageMiner interface { MarketImportDealData(ctx context.Context, propcid cid.Cid, path string) error MarketListDeals(ctx context.Context) ([]storagemarket.StorageDeal, error) MarketListIncompleteDeals(ctx context.Context) ([]storagemarket.MinerDeal, error) - SetPrice(context.Context, types.BigInt) error + MarketSetPrice(context.Context, types.BigInt) error DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error DealsList(ctx context.Context) ([]storagemarket.StorageDeal, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index b6bc3f1b3..1f6f90b37 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -172,7 +172,7 @@ type StorageMinerStruct struct { MarketImportDealData func(context.Context, cid.Cid, string) error `perm:"write"` MarketListDeals func(ctx context.Context) ([]storagemarket.StorageDeal, error) `perm:"read"` MarketListIncompleteDeals func(ctx context.Context) ([]storagemarket.MinerDeal, error) `perm:"read"` - /* Market */ SetPrice func(context.Context, types.BigInt) error `perm:"admin"` + MarketSetPrice func(context.Context, types.BigInt) error `perm:"admin"` PledgeSector func(context.Context) error `perm:"write"` @@ -713,8 +713,8 @@ func (c *StorageMinerStruct) MarketListIncompleteDeals(ctx context.Context) ([]s return c.Internal.MarketListIncompleteDeals(ctx) } -func (c *StorageMinerStruct) SetPrice(ctx context.Context, p types.BigInt) error { - return c.Internal.SetPrice(ctx, p) +func (c *StorageMinerStruct) MarketSetPrice(ctx context.Context, p types.BigInt) error { + return c.Internal.MarketSetPrice(ctx, p) } func (c *StorageMinerStruct) DealsImportData(ctx context.Context, dealPropCid cid.Cid, file string) error { diff --git a/api/client/client.go b/api/client/client.go index 0380d45b5..d0facc55d 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -14,7 +14,9 @@ func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.C closer, err := jsonrpc.NewMergeClient(addr, "Filecoin", []interface{}{ &res.Internal, - }, requestHeader) + }, + requestHeader, + ) return &res, closer, err } @@ -38,7 +40,9 @@ func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMine []interface{}{ &res.CommonStruct.Internal, &res.Internal, - }, requestHeader) + }, + requestHeader, + ) return &res, closer, err } @@ -48,7 +52,9 @@ func NewWorkerRPC(addr string, requestHeader http.Header) (api.WorkerApi, jsonrp closer, err := jsonrpc.NewMergeClient(addr, "Filecoin", []interface{}{ &res.Internal, - }, requestHeader) + }, + requestHeader, + ) return &res, closer, err } diff --git a/cmd/lotus-seal-worker/main.go b/cmd/lotus-seal-worker/main.go index c3717735d..dd0972099 100644 --- a/cmd/lotus-seal-worker/main.go +++ b/cmd/lotus-seal-worker/main.go @@ -277,8 +277,6 @@ var runCmd = &cli.Command{ } }() - // todo go register - return srv.Serve(nl) }, } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 5a62c4e19..8bf135479 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -404,7 +404,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, return xerrors.Errorf("getting id address: %w", err) } - smgr, err := sectorstorage.New(lr, stores.NewIndex(), §orbuilder.Config{ + smgr, err := sectorstorage.New(ctx, lr, stores.NewIndex(), §orbuilder.Config{ SealProofType: spt, PoStProofType: ppt, }, nil, api) diff --git a/cmd/lotus-storage-miner/market.go b/cmd/lotus-storage-miner/market.go index ea5de6345..13713ee2a 100644 --- a/cmd/lotus-storage-miner/market.go +++ b/cmd/lotus-storage-miner/market.go @@ -32,7 +32,7 @@ var setPriceCmd = &cli.Command{ return err } - return api.SetPrice(ctx, types.BigInt(fp)) + return api.MarketSetPrice(ctx, types.BigInt(fp)) }, } diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 897e60f27..3d2a21c87 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -40,6 +40,11 @@ var runCmd = &cli.Command{ Name: "nosync", Usage: "don't check full-node sync status", }, + &cli.BoolFlag{ + Name: "manage-fdlimit", + Usage: "manage open file limit", + Value: true, + }, }, Action: func(cctx *cli.Context) error { if !cctx.Bool("enable-gpu-proving") { @@ -58,8 +63,10 @@ var runCmd = &cli.Command{ return err } - if _, _, err := ulimit.ManageFdLimit(); err != nil { - log.Errorf("setting file descriptor limit: %s", err) + if cctx.Bool("manage-fdlimit") { + if _, _, err := ulimit.ManageFdLimit(); err != nil { + log.Errorf("setting file descriptor limit: %s", err) + } } if v.APIVersion != build.APIVersion { diff --git a/node/builder.go b/node/builder.go index ec8c11f63..730b1b9ef 100644 --- a/node/builder.go +++ b/node/builder.go @@ -272,7 +272,7 @@ func Online() Option { Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig), Override(new(stores.LocalStorage), From(new(repo.LockedRepo))), Override(new(sealing.SectorIDCounter), modules.SectorIDCounter), - Override(new(*sectorstorage.Manager), sectorstorage.New), + Override(new(*sectorstorage.Manager), modules.SectorStorage), Override(new(sectorstorage.SectorManager), From(new(*sectorstorage.Manager))), Override(new(storage2.Prover), From(new(sectorstorage.SectorManager))), diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 24027bba5..8579bdcf9 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -178,7 +178,7 @@ func (sm *StorageMinerAPI) MarketListIncompleteDeals(ctx context.Context) ([]sto return sm.StorageProvider.ListIncompleteDeals() } -func (sm *StorageMinerAPI) SetPrice(ctx context.Context, p types.BigInt) error { +func (sm *StorageMinerAPI) MarketSetPrice(ctx context.Context, p types.BigInt) error { return sm.StorageProvider.AddAsk(abi.TokenAmount(p), 60*60*24*100) // lasts for 100 days? } diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 2c6a4c805..16713e745 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -50,6 +50,7 @@ import ( "github.com/filecoin-project/lotus/storage" "github.com/filecoin-project/lotus/storage/sealing" "github.com/filecoin-project/lotus/storage/sectorstorage" + "github.com/filecoin-project/lotus/storage/sectorstorage/stores" ) func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) { @@ -336,3 +337,9 @@ func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.S network := rmnet.NewFromLibp2pHost(h) return retrievalimpl.NewProvider(address, adapter, network, pieceStore, ibs, ds) } + +func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls sectorstorage.URLs, ca lapi.Common) (*sectorstorage.Manager, error) { + ctx := helpers.LifecycleCtx(mctx, lc) + + return sectorstorage.New(ctx, ls, si, cfg, urls, ca) +} diff --git a/storage/sectorstorage/manager.go b/storage/sectorstorage/manager.go index f3aa90964..d198c88ec 100644 --- a/storage/sectorstorage/manager.go +++ b/storage/sectorstorage/manager.go @@ -3,6 +3,7 @@ package sectorstorage import ( "container/list" "context" + "errors" "io" "net/http" "sync" @@ -24,6 +25,8 @@ import ( var log = logging.Logger("advmgr") +var ErrNoWorkers = errors.New("no suitable workers found") + type URLs []string type Worker interface { @@ -71,9 +74,7 @@ type Manager struct { schedQueue *list.List // List[*workerRequest] } -func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls URLs, ca api.Common) (*Manager, error) { - ctx := context.TODO() - +func New(ctx context.Context, ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls URLs, ca api.Common) (*Manager, error) { lstor, err := stores.NewLocal(ctx, ls, si, urls) if err != nil { return nil, err @@ -84,7 +85,7 @@ func New(ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Confi return nil, xerrors.Errorf("creating prover instance: %w", err) } - token, err := ca.AuthNew(context.TODO(), []api.Permission{"admin"}) + token, err := ca.AuthNew(ctx, []api.Permission{"admin"}) headers := http.Header{} headers.Add("Authorization", "Bearer "+string(token)) stor := stores.NewRemote(lstor, si, headers) @@ -272,7 +273,7 @@ func (m *Manager) AddPiece(ctx context.Context, sector abi.SectorID, existingPie candidateWorkers, _ := m.getWorkersByPaths(sealtasks.TTAddPiece, best) if len(candidateWorkers) == 0 { - return abi.PieceInfo{}, xerrors.New("no worker found") + return abi.PieceInfo{}, ErrNoWorkers } worker, done, err := m.getWorker(ctx, sealtasks.TTAddPiece, candidateWorkers) @@ -296,7 +297,7 @@ func (m *Manager) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticke candidateWorkers, _ := m.getWorkersByPaths(sealtasks.TTPreCommit1, best) if len(candidateWorkers) == 0 { - return nil, xerrors.New("no suitable workers found") + return nil, ErrNoWorkers } worker, done, err := m.getWorker(ctx, sealtasks.TTPreCommit1, candidateWorkers) @@ -320,7 +321,7 @@ func (m *Manager) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase candidateWorkers, _ := m.getWorkersByPaths(sealtasks.TTPreCommit2, best) if len(candidateWorkers) == 0 { - return storage.SectorCids{}, xerrors.New("no suitable workers found") + return storage.SectorCids{}, ErrNoWorkers } worker, done, err := m.getWorker(ctx, sealtasks.TTPreCommit2, candidateWorkers) @@ -342,7 +343,7 @@ func (m *Manager) SealCommit1(ctx context.Context, sector abi.SectorID, ticket a candidateWorkers, _ := m.getWorkersByPaths(sealtasks.TTCommit1, best) if len(candidateWorkers) == 0 { - return nil, xerrors.New("no suitable workers found") // TODO: wait? + return nil, ErrNoWorkers } // TODO: Try very hard to execute on worker with access to the sectors @@ -373,6 +374,9 @@ func (m *Manager) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Ou candidateWorkers = append(candidateWorkers, id) } m.workersLk.Unlock() + if len(candidateWorkers) == 0 { + return nil, ErrNoWorkers + } worker, done, err := m.getWorker(ctx, sealtasks.TTCommit2, candidateWorkers) if err != nil { @@ -390,6 +394,9 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector abi.SectorID) error } candidateWorkers, _ := m.getWorkersByPaths(sealtasks.TTFinalize, best) + if len(candidateWorkers) == 0 { + return ErrNoWorkers + } // TODO: Remove sector from sealing stores // TODO: Move the sector to long-term storage From 62cf9a5dd7dbb66cb4404980d20db82619b27913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 24 Mar 2020 19:19:41 +0100 Subject: [PATCH 41/45] Update FFI --- Makefile | 2 +- extern/filecoin-ffi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c12095f53..e7e69589c 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ GOFLAGS+=-ldflags=-X="github.com/filecoin-project/lotus/build".CurrentCommit="+g ## FFI FFI_PATH:=extern/filecoin-ffi/ -FFI_DEPS:=libfilecoin.a filecoin.pc filecoin.h +FFI_DEPS:=.install-filcrypto FFI_DEPS:=$(addprefix $(FFI_PATH),$(FFI_DEPS)) $(FFI_DEPS): build/.filecoin-install ; diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index f20cfbe28..eb91e8c46 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit f20cfbe28d99beda69e5416c6829927945116428 +Subproject commit eb91e8c461452a685ba0d0765f996d2117dbd314 From 5de17e903cc3f68dbbf672cd40f920206fc6f613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 24 Mar 2020 20:38:00 +0100 Subject: [PATCH 42/45] storageminer: Config for local worker task types --- cmd/lotus-storage-miner/init.go | 2 +- node/builder.go | 6 +++++- node/config/def.go | 10 +++++++++- node/modules/storageminer.go | 5 +++-- storage/sectorstorage/manager.go | 14 +++++++++----- 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 8bf135479..aa23f56ca 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -407,7 +407,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api lapi.FullNode, smgr, err := sectorstorage.New(ctx, lr, stores.NewIndex(), §orbuilder.Config{ SealProofType: spt, PoStProofType: ppt, - }, nil, api) + }, config.Storage{true, true, true}, nil, api) if err != nil { return err } diff --git a/node/builder.go b/node/builder.go index 730b1b9ef..baac693a1 100644 --- a/node/builder.go +++ b/node/builder.go @@ -382,7 +382,11 @@ func ConfigStorageMiner(c interface{}) Option { return Error(xerrors.Errorf("invalid config from repo, got: %T", c)) } - return Options(ConfigCommon(&cfg.Common)) + return Options( + ConfigCommon(&cfg.Common), + + Override(new(config.Storage), cfg.Storage), + ) } func Repo(r repo.Repo) Option { diff --git a/node/config/def.go b/node/config/def.go index 69355aadd..edb00aa5e 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -53,6 +53,10 @@ type Metrics struct { // // Storage Miner type Storage struct { + // Local worker config + AllowPreCommit1 bool + AllowPreCommit2 bool + AllowCommit bool } func defCommon() Common { @@ -86,7 +90,11 @@ func DefaultStorageMiner() *StorageMiner { cfg := &StorageMiner{ Common: defCommon(), - Storage: Storage{}, + Storage: Storage{ + AllowPreCommit1: true, + AllowPreCommit2: true, + AllowCommit: true, + }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" return cfg diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 16713e745..010201de1 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -44,6 +44,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/markets/retrievaladapter" "github.com/filecoin-project/lotus/miner" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" "github.com/filecoin-project/lotus/node/repo" @@ -338,8 +339,8 @@ func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.S return retrievalimpl.NewProvider(address, adapter, network, pieceStore, ibs, ds) } -func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls sectorstorage.URLs, ca lapi.Common) (*sectorstorage.Manager, error) { +func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, sc config.Storage, urls sectorstorage.URLs, ca lapi.Common) (*sectorstorage.Manager, error) { ctx := helpers.LifecycleCtx(mctx, lc) - return sectorstorage.New(ctx, ls, si, cfg, urls, ca) + return sectorstorage.New(ctx, ls, si, cfg, sc, urls, ca) } diff --git a/storage/sectorstorage/manager.go b/storage/sectorstorage/manager.go index d198c88ec..843d841cb 100644 --- a/storage/sectorstorage/manager.go +++ b/storage/sectorstorage/manager.go @@ -74,7 +74,7 @@ type Manager struct { schedQueue *list.List // List[*workerRequest] } -func New(ctx context.Context, ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, urls URLs, ca api.Common) (*Manager, error) { +func New(ctx context.Context, ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, sc config.Storage, urls URLs, ca api.Common) (*Manager, error) { lstor, err := stores.NewLocal(ctx, ls, si, urls) if err != nil { return nil, err @@ -114,12 +114,16 @@ func New(ctx context.Context, ls stores.LocalStorage, si stores.SectorIndex, cfg go m.runSched() + localTasks := []sealtasks.TaskType{ + sealtasks.TTAddPiece, sealtasks.TTCommit1, sealtasks.TTFinalize, + } + if sc.AllowPreCommit1 { localTasks = append(localTasks, sealtasks.TTPreCommit1)} + if sc.AllowPreCommit2 { localTasks = append(localTasks, sealtasks.TTPreCommit2)} + if sc.AllowCommit { localTasks = append(localTasks, sealtasks.TTCommit2)} + err = m.AddWorker(ctx, NewLocalWorker(WorkerConfig{ SealProof: cfg.SealProofType, - TaskTypes: []sealtasks.TaskType{ - sealtasks.TTAddPiece, sealtasks.TTCommit1, sealtasks.TTFinalize, - sealtasks.TTPreCommit1, sealtasks.TTPreCommit2, sealtasks.TTCommit2, // TODO: Config - }, + TaskTypes: localTasks, }, stor, lstor, si)) if err != nil { return nil, xerrors.Errorf("adding local worker: %w", err) From 146bfe75f89913ed57e7758b511964c80c6e57f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 24 Mar 2020 21:28:07 +0100 Subject: [PATCH 43/45] sectorstorage: Remove unsealed sector in FinalizeSector --- storage/sectorstorage/stores/http_handler.go | 4 +-- storage/sectorstorage/stores/interface.go | 1 + storage/sectorstorage/stores/local.go | 6 ++-- storage/sectorstorage/stores/remote.go | 30 +++++++++++++++++++- storage/sectorstorage/worker_local.go | 10 ++++++- 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/storage/sectorstorage/stores/http_handler.go b/storage/sectorstorage/stores/http_handler.go index 97c6c34f6..657da0f9b 100644 --- a/storage/sectorstorage/stores/http_handler.go +++ b/storage/sectorstorage/stores/http_handler.go @@ -36,7 +36,7 @@ func (handler *FetchHandler) remoteStatFs(w http.ResponseWriter, r *http.Request vars := mux.Vars(r) id := ID(vars["id"]) - st, err := handler.Local.FsStat(id) + st, err := handler.Local.FsStat(r.Context(), id) switch err { case errPathNotFound: w.WriteHeader(404) @@ -129,7 +129,7 @@ func (handler *FetchHandler) remoteDeleteSector(w http.ResponseWriter, r *http.R return } - if err := handler.delete(r.Context(), id, ft); err != nil { + if err := handler.Remove(r.Context(), id, ft); err != nil { log.Error("%+v", err) w.WriteHeader(500) return diff --git a/storage/sectorstorage/stores/interface.go b/storage/sectorstorage/stores/interface.go index 149cb9e5f..e56a6d74a 100644 --- a/storage/sectorstorage/stores/interface.go +++ b/storage/sectorstorage/stores/interface.go @@ -12,6 +12,7 @@ import ( type Store interface { AcquireSector(ctx context.Context, s abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (paths sectorbuilder.SectorPaths, stores sectorbuilder.SectorPaths, done func(), err error) + Remove(ctx context.Context, s abi.SectorID, types sectorbuilder.SectorFileType) error FsStat(ctx context.Context, id ID) (FsStat, error) } diff --git a/storage/sectorstorage/stores/local.go b/storage/sectorstorage/stores/local.go index 581afb13f..d891f86fa 100644 --- a/storage/sectorstorage/stores/local.go +++ b/storage/sectorstorage/stores/local.go @@ -270,7 +270,7 @@ func (st *Local) Local(ctx context.Context) ([]StoragePath, error) { return out, nil } -func (st *Local) delete(ctx context.Context, sid abi.SectorID, typ sectorbuilder.SectorFileType) error { +func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ sectorbuilder.SectorFileType) error { if bits.OnesCount(uint(typ)) != 1 { return xerrors.New("delete expects one file type") } @@ -311,7 +311,7 @@ func (st *Local) delete(ctx context.Context, sid abi.SectorID, typ sectorbuilder var errPathNotFound = xerrors.Errorf("fsstat: path not found") -func (st *Local) FsStat(id ID) (FsStat, error) { +func (st *Local) FsStat(ctx context.Context, id ID) (FsStat, error) { st.localLk.RLock() defer st.localLk.RUnlock() @@ -322,3 +322,5 @@ func (st *Local) FsStat(id ID) (FsStat, error) { return Stat(p.local) } + +var _ Store = &Local{} diff --git a/storage/sectorstorage/stores/remote.go b/storage/sectorstorage/stores/remote.go index a0648d972..ab7d465b3 100644 --- a/storage/sectorstorage/stores/remote.go +++ b/storage/sectorstorage/stores/remote.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "io/ioutil" + "math/bits" "mime" "net/http" "net/url" @@ -173,6 +174,33 @@ func (r *Remote) fetch(ctx context.Context, url, outname string) error { } } +func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ sectorbuilder.SectorFileType) error { + if bits.OnesCount(uint(typ)) != 1 { + return xerrors.New("delete expects one file type") + } + + if err := r.local.Remove(ctx, sid, typ); err != nil { + return xerrors.Errorf("remove from local: %w", err) + } + + si, err := r.index.StorageFindSector(ctx, sid, typ, false) + if err != nil { + return xerrors.Errorf("finding existing sector %d(t:%d) failed: %w", sid, typ, err) + } + + for _, info := range si { + for _, url := range info.URLs { + if err := r.deleteFromRemote(ctx, url); err != nil { + log.Warnf("remove %s: %+v", url, err) + continue + } + break + } + } + + return nil +} + func (r *Remote) deleteFromRemote(ctx context.Context, url string) error { log.Infof("Delete %s", url) @@ -197,7 +225,7 @@ func (r *Remote) deleteFromRemote(ctx context.Context, url string) error { } func (r *Remote) FsStat(ctx context.Context, id ID) (FsStat, error) { - st, err := r.local.FsStat(id) + st, err := r.local.FsStat(ctx, id) switch err { case nil: return st, nil diff --git a/storage/sectorstorage/worker_local.go b/storage/sectorstorage/worker_local.go index d691f150e..18a0305fd 100644 --- a/storage/sectorstorage/worker_local.go +++ b/storage/sectorstorage/worker_local.go @@ -152,7 +152,15 @@ func (l *LocalWorker) FinalizeSector(ctx context.Context, sector abi.SectorID) e return err } - return sb.FinalizeSector(ctx, sector) + if err := sb.FinalizeSector(ctx, sector); err != nil { + return xerrors.Errorf("finalizing sector: %w", err) + } + + if err := l.storage.Remove(ctx, sector, sectorbuilder.FTUnsealed); err != nil { + return xerrors.Errorf("removing unsealed data: %w", err) + } + + return nil } func (l *LocalWorker) TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) { From ecf53f88ce1786fcb02414e6a1a2ef34647c9f2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 25 Mar 2020 00:37:40 +0100 Subject: [PATCH 44/45] workers: More review related fixes --- cmd/lotus-seal-worker/rpc.go | 2 +- storage/sectorstorage/resources.go | 2 +- storage/sectorstorage/roprov.go | 2 +- storage/sectorstorage/sectorutil/utils.go | 2 +- storage/sectorstorage/stores/http_handler.go | 3 +++ storage/sectorstorage/stores/remote.go | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-seal-worker/rpc.go b/cmd/lotus-seal-worker/rpc.go index 5d8ce3cb7..254cd47f7 100644 --- a/cmd/lotus-seal-worker/rpc.go +++ b/cmd/lotus-seal-worker/rpc.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/lotus/storage/sectorstorage" ) -type worker struct { // TODO: use advmgr.LocalWorker here +type worker struct { *sectorstorage.LocalWorker } diff --git a/storage/sectorstorage/resources.go b/storage/sectorstorage/resources.go index 3587b41ea..ebb32f165 100644 --- a/storage/sectorstorage/resources.go +++ b/storage/sectorstorage/resources.go @@ -20,7 +20,7 @@ var FsOverheadFinalized = map[sectorbuilder.SectorFileType]int{ type Resources struct { MinMemory uint64 // What Must be in RAM for decent perf - MaxMemory uint64 // Mamory required (swap + ram) + MaxMemory uint64 // Memory required (swap + ram) MultiThread bool CanGPU bool diff --git a/storage/sectorstorage/roprov.go b/storage/sectorstorage/roprov.go index 99723e181..4b0dfbe2b 100644 --- a/storage/sectorstorage/roprov.go +++ b/storage/sectorstorage/roprov.go @@ -15,7 +15,7 @@ type readonlyProvider struct { } func (l *readonlyProvider) AcquireSector(ctx context.Context, id abi.SectorID, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) { - if allocate != 0 { + if allocate != 0 { // 0 - don't allocate anything return sectorbuilder.SectorPaths{}, nil, xerrors.New("read-only storage") } diff --git a/storage/sectorstorage/sectorutil/utils.go b/storage/sectorstorage/sectorutil/utils.go index 01862b7b4..ede59410b 100644 --- a/storage/sectorstorage/sectorutil/utils.go +++ b/storage/sectorstorage/sectorutil/utils.go @@ -14,7 +14,7 @@ func ParseSectorID(baseName string) (abi.SectorID, error) { var mid abi.ActorID read, err := fmt.Sscanf(baseName, "s-t0%d-%d", &mid, &n) if err != nil { - return abi.SectorID{}, xerrors.Errorf(": %w", err) + return abi.SectorID{}, xerrors.Errorf("sscanf sector name ('%s'): %w", baseName, err) } if read != 2 { diff --git a/storage/sectorstorage/stores/http_handler.go b/storage/sectorstorage/stores/http_handler.go index 657da0f9b..b242d1159 100644 --- a/storage/sectorstorage/stores/http_handler.go +++ b/storage/sectorstorage/stores/http_handler.go @@ -68,11 +68,13 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ ft, err := ftFromString(vars["type"]) if err != nil { log.Error("%+v", err) + w.WriteHeader(500) return } paths, _, done, err := handler.Local.AcquireSector(r.Context(), id, ft, 0, false) if err != nil { log.Error("%+v", err) + w.WriteHeader(500) return } defer done() @@ -126,6 +128,7 @@ func (handler *FetchHandler) remoteDeleteSector(w http.ResponseWriter, r *http.R ft, err := ftFromString(vars["type"]) if err != nil { log.Error("%+v", err) + w.WriteHeader(500) return } diff --git a/storage/sectorstorage/stores/remote.go b/storage/sectorstorage/stores/remote.go index ab7d465b3..66b818434 100644 --- a/storage/sectorstorage/stores/remote.go +++ b/storage/sectorstorage/stores/remote.go @@ -273,7 +273,7 @@ func (r *Remote) FsStat(ctx context.Context, id ID) (FsStat, error) { return FsStat{}, xerrors.Errorf("fsstat: got http 500, then failed to read the error: %w", err) } - return FsStat{}, xerrors.New(string(b)) + return FsStat{}, xerrors.Errorf("fsstat: got http 500: %s", string(b)) } var out FsStat From cfc65f525ab0e328a22cfe9da489b40a9894eaf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 25 Mar 2020 00:49:45 +0100 Subject: [PATCH 45/45] sectorstorage: wire up closing logic --- cmd/lotus-storage-miner/workers.go | 11 +++++------ extern/filecoin-ffi | 2 +- node/modules/storageminer.go | 17 ++++++++++++++++- storage/sectorstorage/manager.go | 7 +++++++ storage/sectorstorage/sched.go | 14 ++++++++++++++ storage/sectorstorage/worker_local.go | 4 ++++ storage/sectorstorage/worker_remote.go | 12 +++++++++--- 7 files changed, 56 insertions(+), 11 deletions(-) diff --git a/cmd/lotus-storage-miner/workers.go b/cmd/lotus-storage-miner/workers.go index a29de1c6b..71a6431ab 100644 --- a/cmd/lotus-storage-miner/workers.go +++ b/cmd/lotus-storage-miner/workers.go @@ -35,15 +35,14 @@ var workersListCmd = &cli.Command{ return err } - st := make([]struct { + type sortableStat struct { id uint64 api.WorkerStats - }, 0, len(stats)) + } + + st := make([]sortableStat, 0, len(stats)) for id, stat := range stats { - st = append(st, struct { - id uint64 - api.WorkerStats - }{id, stat}) + st = append(st, sortableStat{id, stat}) } sort.Slice(st, func(i, j int) bool { diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index eb91e8c46..41b20ed16 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit eb91e8c461452a685ba0d0765f996d2117dbd314 +Subproject commit 41b20ed16500eb5b4bacd07ec8aee386257e56da diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 010201de1..4c39f89d2 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -342,5 +342,20 @@ func RetrievalProvider(h host.Host, miner *storage.Miner, sealer sectorstorage.S func SectorStorage(mctx helpers.MetricsCtx, lc fx.Lifecycle, ls stores.LocalStorage, si stores.SectorIndex, cfg *sectorbuilder.Config, sc config.Storage, urls sectorstorage.URLs, ca lapi.Common) (*sectorstorage.Manager, error) { ctx := helpers.LifecycleCtx(mctx, lc) - return sectorstorage.New(ctx, ls, si, cfg, sc, urls, ca) + sst, err := sectorstorage.New(ctx, ls, si, cfg, sc, urls, ca) + if err != nil { + return nil, err + } + + lc.Append(fx.Hook{ + OnStop: func(_ context.Context) error { + if err := sst.Close(); err != nil { + log.Errorf("%+v", err) + } + + return nil + }, + }) + + return sst, nil } diff --git a/storage/sectorstorage/manager.go b/storage/sectorstorage/manager.go index 843d841cb..3ef8e30c6 100644 --- a/storage/sectorstorage/manager.go +++ b/storage/sectorstorage/manager.go @@ -38,6 +38,8 @@ type Worker interface { Paths(context.Context) ([]stores.StoragePath, error) Info(context.Context) (api.WorkerInfo, error) + + Close() error } type SectorManager interface { @@ -425,4 +427,9 @@ func (m *Manager) FsStat(ctx context.Context, id stores.ID) (stores.FsStat, erro return m.storage.FsStat(ctx, id) } +func (m *Manager) Close() error { + close(m.closing) + return nil +} + var _ SectorManager = &Manager{} diff --git a/storage/sectorstorage/sched.go b/storage/sectorstorage/sched.go index a38707f74..d8e3d35a0 100644 --- a/storage/sectorstorage/sched.go +++ b/storage/sectorstorage/sched.go @@ -67,6 +67,9 @@ func (m *Manager) runSched() { m.schedQueue.PushBack(req) case wid := <-m.workerFree: m.onWorkerFreed(wid) + case <-m.closing: + m.schedClose() + return } } } @@ -240,3 +243,14 @@ func (m *Manager) schedNewWorker(w *workerHandle) { m.workers[id] = w m.nextWorker++ } + +func (m *Manager) schedClose() { + m.workersLk.Lock() + defer m.workersLk.Unlock() + + for i, w := range m.workers { + if err := w.w.Close(); err != nil { + log.Errorf("closing worker %d: %+v", i, err) + } + } +} diff --git a/storage/sectorstorage/worker_local.go b/storage/sectorstorage/worker_local.go index 18a0305fd..175106bad 100644 --- a/storage/sectorstorage/worker_local.go +++ b/storage/sectorstorage/worker_local.go @@ -203,4 +203,8 @@ func (l *LocalWorker) Info(context.Context) (api.WorkerInfo, error) { }, nil } +func (l *LocalWorker) Close() error { + return nil +} + var _ Worker = &LocalWorker{} diff --git a/storage/sectorstorage/worker_remote.go b/storage/sectorstorage/worker_remote.go index f49ea4dc6..ffd96f188 100644 --- a/storage/sectorstorage/worker_remote.go +++ b/storage/sectorstorage/worker_remote.go @@ -10,10 +10,12 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/lib/jsonrpc" ) type remote struct { api.WorkerApi + closer jsonrpc.ClientCloser } func (r *remote) NewSector(ctx context.Context, sector abi.SectorID) error { @@ -33,13 +35,17 @@ func ConnectRemote(ctx context.Context, fa api.Common, url string) (*remote, err headers := http.Header{} headers.Add("Authorization", "Bearer "+string(token)) - wapi, close, err := client.NewWorkerRPC(url, headers) + wapi, closer, err := client.NewWorkerRPC(url, headers) if err != nil { return nil, xerrors.Errorf("creating jsonrpc client: %w", err) } - _ = close // TODO - return &remote{wapi}, nil + return &remote{wapi, closer}, nil +} + +func (r *remote) Close() error { + r.closer() + return nil } var _ Worker = &remote{}