workers: RPC scaffolding

2020-03-11 02:58:02 +01:00
parent da52596d79
commit eb61a36fd7
15 changed files with 299 additions and 214 deletions
@@ -4,9 +4,15 @@ import (
 	"os"

 	logging "github.com/ipfs/go-log/v2"
+	manet "github.com/multiformats/go-multiaddr-net"
+	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"

+	paramfetch "github.com/filecoin-project/go-paramfetch"
+
+	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
+	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/lib/lotuslog"
 	"github.com/filecoin-project/lotus/node/repo"
 )
@@ -75,60 +81,53 @@ var runCmd = &cli.Command{
 	Name:  "run",
 	Usage: "Start lotus worker",
 	Action: func(cctx *cli.Context) error {
-		/*	if !cctx.Bool("enable-gpu-proving") {
-				os.Setenv("BELLMAN_NO_GPU", "true")
-			}
+		if !cctx.Bool("enable-gpu-proving") {
+			os.Setenv("BELLMAN_NO_GPU", "true")
+		}

-			nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
-			if err != nil {
-				return xerrors.Errorf("getting miner api: %w", err)
-			}
-			defer closer()
-			ctx := lcli.ReqContext(cctx)
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return xerrors.Errorf("getting miner api: %w", err)
+		}
+		defer closer()
+		ctx := lcli.ReqContext(cctx)

-			ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner)
-			if err != nil {
-				return xerrors.Errorf("could not get api info: %w", err)
-			}
-			_, storageAddr, err := manet.DialArgs(ainfo.Addr)
+		v, err := nodeApi.Version(ctx)
+		if err != nil {
+			return err
+		}
+		if v.APIVersion != build.APIVersion {
+			return xerrors.Errorf("lotus-storage-miner API version doesn't match: local: ", api.Version{APIVersion: build.APIVersion})
+		}

-			r, err := homedir.Expand(cctx.String("repo"))
-			if err != nil {
-				return err
-			}
+		go func() {
+			<-ctx.Done()
+			log.Warn("Shutting down..")
+		}()

-			v, err := nodeApi.Version(ctx)
-			if err != nil {
-				return err
-			}
-			if v.APIVersion != build.APIVersion {
-				return xerrors.Errorf("lotus-storage-miner API version doesn't match: local: ", api.Version{APIVersion: build.APIVersion})
-			}
+		act, err := nodeApi.ActorAddress(ctx)
+		if err != nil {
+			return err
+		}
+		ssize, err := nodeApi.ActorSectorSize(ctx, act)
+		if err != nil {
+			return err
+		}

-			go func() {
-				<-ctx.Done()
-				log.Warn("Shutting down..")
-			}()
+		if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil {
+			return xerrors.Errorf("get params: %w", err)
+		}

-			limiter := &limits{
-				workLimit:     make(chan struct{}, workers),
-				transferLimit: make(chan struct{}, transfers),
-			}
+		ainfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner)
+		if err != nil {
+			return xerrors.Errorf("could not get api info: %w", err)
+		}
+		_, storageAddr, err := manet.DialArgs(ainfo.Addr)

-			act, err := nodeApi.ActorAddress(ctx)
-			if err != nil {
-				return err
-			}
-			ssize, err := nodeApi.ActorSectorSize(ctx, act)
-			if err != nil {
-				return err
-			}

-			if err := paramfetch.GetParams(build.ParametersJson(), uint64(ssize)); err != nil {
-				return xerrors.Errorf("get params: %w", err)
-			}

-			/*ppt, spt, err := api.ProofTypeFromSectorSize(ssize)
+		/*
+		/*ppt, spt, err := api.ProofTypeFromSectorSize(ssize)
 			if err != nil {
 				return err
 			}
@@ -0,0 +1,35 @@
+package main
+
+import (
+	"context"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-storage/storage"
+	"github.com/ipfs/go-cid"
+)
+
+type worker struct {
+
+}
+
+func (w *worker) SealPreCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, pieces []abi.PieceInfo) (storage.PreCommit1Out, error) {
+	panic("implement me")
+}
+
+func (w *worker) SealPreCommit2(context.Context, abi.SectorNumber, storage.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) {
+	panic("implement me")
+}
+
+func (w *worker) SealCommit1(ctx context.Context, sectorNum abi.SectorNumber, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, sealedCID cid.Cid, unsealedCID cid.Cid) (storage.Commit1Out, error) {
+	panic("implement me")
+}
+
+func (w *worker) SealCommit2(context.Context, abi.SectorNumber, storage.Commit1Out) (storage.Proof, error) {
+	panic("implement me")
+}
+
+func (w *worker) FinalizeSector(context.Context, abi.SectorNumber) error {
+	panic("implement me")
+}
+
+var _ storage.Sealer = &worker{}
@@ -0,0 +1,22 @@
+package main
+
+import (
+	"context"
+
+	"github.com/filecoin-project/specs-actors/actors/abi"
+
+	"github.com/filecoin-project/go-sectorbuilder"
+	"github.com/filecoin-project/lotus/api"
+)
+
+type workerStorage struct {
+	path string // TODO: multi-path support
+
+	api api.StorageMiner
+}
+
+func (w *workerStorage) AcquireSector(ctx context.Context, id abi.SectorNumber, existing sectorbuilder.SectorFileType, allocate sectorbuilder.SectorFileType, sealing bool) (sectorbuilder.SectorPaths, func(), error) {
+	w.api.WorkerFindSector()
+}
+
+var _ sectorbuilder.SectorProvider = &workerStorage{}
@@ -1,132 +0,0 @@
-package main
-
-/*
-import (
-	"context"
-	"net/http"
-
-	"github.com/filecoin-project/go-sectorbuilder"
-	"golang.org/x/xerrors"
-
-	lapi "github.com/filecoin-project/lotus/api"
-)
-
-type worker struct {
-	api           lapi.StorageMiner
-	minerEndpoint string
-	repo          string
-	auth          http.Header
-
-	limiter *limits
-	sb      *sectorbuilder.SectorBuilder
-}
-
-func acceptJobs(ctx context.Context, api lapi.StorageMiner, sb *sectorbuilder.SectorBuilder, limiter *limits, endpoint string, auth http.Header, repo string, noprecommit, nocommit bool) error {
-	w := &worker{
-		api:           api,
-		minerEndpoint: endpoint,
-		auth:          auth,
-		repo:          repo,
-
-		limiter: limiter,
-		sb:      sb,
-	}
-
-	tasks, err := api.WorkerQueue(ctx, sectorbuilder.WorkerCfg{
-		NoPreCommit: noprecommit,
-		NoCommit:    nocommit,
-	})
-	if err != nil {
-		return err
-	}
-
-loop:
-	for {
-		log.Infof("Waiting for new task")
-
-		select {
-		case task := <-tasks:
-			log.Infof("New task: %d, sector %d, action: %d", task.TaskID, task.SectorNum, task.Type)
-
-			res := w.processTask(ctx, task)
-
-			log.Infof("Task %d done, err: %+v", task.TaskID, res.GoErr)
-
-			if err := api.WorkerDone(ctx, task.TaskID, res); err != nil {
-				log.Error(err)
-			}
-		case <-ctx.Done():
-			break loop
-		}
-	}
-
-	log.Warn("acceptJobs exit")
-	return nil
-}
-
-func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask) sectorbuilder.SealRes {
-	switch task.Type {
-	case sectorbuilder.WorkerPreCommit:
-	case sectorbuilder.WorkerCommit:
-	default:
-		return errRes(xerrors.Errorf("unknown task type %d", task.Type))
-	}
-
-	if err := w.fetchSector(task.SectorNum, task.Type); err != nil {
-		return errRes(xerrors.Errorf("fetching sector: %w", err))
-	}
-
-	log.Infof("Data fetched, starting computation")
-
-	var res sectorbuilder.SealRes
-
-	switch task.Type {
-	case sectorbuilder.WorkerPreCommit:
-		w.limiter.workLimit <- struct{}{}
-		sealedCid, unsealedCid, err := w.sb.SealPreCommit(ctx, task.SectorNum, task.SealTicket, task.Pieces)
-		<-w.limiter.workLimit
-
-		if err != nil {
-			return errRes(xerrors.Errorf("precomitting: %w", err))
-		}
-		res.Rspco.CommD = unsealedCid
-		res.Rspco.CommR = sealedCid
-
-		if err := w.push("sealed", task.SectorNum); err != nil {
-			return errRes(xerrors.Errorf("pushing precommited data: %w", err))
-		}
-
-		if err := w.push("cache", task.SectorNum); err != nil {
-			return errRes(xerrors.Errorf("pushing precommited data: %w", err))
-		}
-
-		if err := w.remove("staging", task.SectorNum); err != nil {
-			return errRes(xerrors.Errorf("cleaning up staged sector: %w", err))
-		}
-	case sectorbuilder.WorkerCommit:
-		w.limiter.workLimit <- struct{}{}
-		proof, err := w.sb.SealCommit(ctx, task.SectorNum, task.SealTicket, task.SealSeed, task.Pieces, task.SealedCID, task.UnsealedCID)
-		<-w.limiter.workLimit
-
-		if err != nil {
-			return errRes(xerrors.Errorf("comitting: %w", err))
-		}
-
-		res.Proof = proof
-
-		if err := w.push("cache", task.SectorNum); err != nil {
-			return errRes(xerrors.Errorf("pushing precommited data: %w", err))
-		}
-
-		if err := w.remove("sealed", task.SectorNum); err != nil {
-			return errRes(xerrors.Errorf("cleaning up sealed sector: %w", err))
-		}
-	}
-
-	return res
-}
-
-func errRes(err error) sectorbuilder.SealRes {
-	return sectorbuilder.SealRes{Err: err.Error(), GoErr: err}
-}
-*/