From b83ff6b9dcacdbf07ce40457c42520767d8b17e2 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sat, 27 Jul 2019 14:08:10 -0700 Subject: [PATCH] Wire up sector builder commands through the api --- api/api.go | 10 +++ api/struct.go | 20 ++++++ cmd/lotus-storage-miner/main.go | 1 + cmd/lotus-storage-miner/run.go | 21 ------ cmd/lotus-storage-miner/sectors.go | 112 +++++++++++++++++++++++++++++ lib/sectorbuilder/sectorbuilder.go | 12 +++- node/impl/storminer.go | 22 +++++- node/modules/core.go | 12 +++- 8 files changed, 183 insertions(+), 27 deletions(-) create mode 100644 cmd/lotus-storage-miner/sectors.go diff --git a/api/api.go b/api/api.go index 598777695..d5ba98f64 100644 --- a/api/api.go +++ b/api/api.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/go-lotus/chain/store" "github.com/filecoin-project/go-lotus/chain/types" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/ipfs/go-filestore" ) @@ -118,4 +119,13 @@ type StorageMiner interface { // Temp api for testing StoreGarbageData(context.Context) (uint64, error) + + // Get the status of a given sector by ID + SectorsStatus(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) + + // List all staged sectors + SectorsStagedList(context.Context) ([]sectorbuilder.StagedSectorMetadata, error) + + // Seal all staged sectors + SectorsStagedSeal(context.Context) error } diff --git a/api/struct.go b/api/struct.go index f6394f678..82379f278 100644 --- a/api/struct.go +++ b/api/struct.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-lotus/chain/address" "github.com/filecoin-project/go-lotus/chain/store" "github.com/filecoin-project/go-lotus/chain/types" + sectorbuilder "github.com/filecoin-project/go-sectorbuilder" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p-core/peer" @@ -69,6 +70,10 @@ type StorageMinerStruct struct { Internal struct { StoreGarbageData func(context.Context) (uint64, error) `perm:"write"` + + SectorsStatus func(context.Context, uint64) (sectorbuilder.SectorSealingStatus, error) `perm:"read"` + SectorsStagedList func(context.Context) ([]sectorbuilder.StagedSectorMetadata, error) `perm:"read"` + SectorsStagedSeal func(context.Context) error `perm:"write"` } } @@ -190,6 +195,21 @@ func (c *StorageMinerStruct) StoreGarbageData(ctx context.Context) (uint64, erro return c.Internal.StoreGarbageData(ctx) } +// Get the status of a given sector by ID +func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (sectorbuilder.SectorSealingStatus, error) { + return c.Internal.SectorsStatus(ctx, sid) +} + +// List all staged sectors +func (c *StorageMinerStruct) SectorsStagedList(ctx context.Context) ([]sectorbuilder.StagedSectorMetadata, error) { + return c.Internal.SectorsStagedList(ctx) +} + +// Seal all staged sectors +func (c *StorageMinerStruct) SectorsStagedSeal(ctx context.Context) error { + return c.Internal.SectorsStagedSeal(ctx) +} + var _ Common = &CommonStruct{} var _ FullNode = &FullNodeStruct{} var _ StorageMiner = &StorageMinerStruct{} diff --git a/cmd/lotus-storage-miner/main.go b/cmd/lotus-storage-miner/main.go index 10142d3f8..d91745783 100644 --- a/cmd/lotus-storage-miner/main.go +++ b/cmd/lotus-storage-miner/main.go @@ -22,6 +22,7 @@ func main() { runCmd, initCmd, storeGarbageCmd, + sectorsCmd, } jaeger := tracing.SetupJaegerTracing("lotus") defer func() { diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 752d48d9a..80df5015b 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net/http" "os" @@ -96,23 +95,3 @@ var runCmd = &cli.Command{ return http.ListenAndServe("127.0.0.1:"+cctx.String("api"), http.DefaultServeMux) }, } - -var storeGarbageCmd = &cli.Command{ - Name: "store-garbage", - Usage: "store random data in a sector", - Action: func(cctx *cli.Context) error { - nodeApi, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return err - } - ctx := lcli.ReqContext(cctx) - - sectorId, err := nodeApi.StoreGarbageData(ctx) - if err != nil { - return err - } - - fmt.Println(sectorId) - return nil - }, -} diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go new file mode 100644 index 000000000..82eef4a33 --- /dev/null +++ b/cmd/lotus-storage-miner/sectors.go @@ -0,0 +1,112 @@ +package main + +import ( + "fmt" + "strconv" + + "gopkg.in/urfave/cli.v2" + + lcli "github.com/filecoin-project/go-lotus/cli" +) + +var storeGarbageCmd = &cli.Command{ + Name: "store-garbage", + Usage: "store random data in a sector", + Action: func(cctx *cli.Context) error { + nodeApi, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + ctx := lcli.ReqContext(cctx) + + sectorId, err := nodeApi.StoreGarbageData(ctx) + if err != nil { + return err + } + + fmt.Println(sectorId) + return nil + }, +} + +var sectorsCmd = &cli.Command{ + Name: "sectors", + Usage: "interact with sector store", + Subcommands: []*cli.Command{ + sectorsStatusCmd, + sectorsStagedListCmd, + sectorsStagedSealCmd, + }, +} + +var sectorsStatusCmd = &cli.Command{ + Name: "status", + Usage: "Get the seal status of a sector by its ID", + Action: func(cctx *cli.Context) error { + nodeApi, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + ctx := lcli.ReqContext(cctx) + + if !cctx.Args().Present() { + return fmt.Errorf("must specify sector ID to get status of") + } + + id, err := strconv.ParseUint(cctx.Args().First(), 10, 64) + if err != nil { + return err + } + + status, err := nodeApi.SectorsStatus(ctx, id) + if err != nil { + return err + } + + fmt.Printf("SectorID:\t%d\n", status.SectorID) + fmt.Printf("SealStatusCode:\t%d\n", status.SealStatusCode) + fmt.Printf("SealErrorMsg:\t%q\n", status.SealErrorMsg) + fmt.Printf("CommD:\t\t%x\n", status.CommD) + fmt.Printf("CommR:\t\t%x\n", status.CommR) + fmt.Printf("CommR*:\t\t%x\n", status.CommRStar) + fmt.Printf("Proof:\t\t%x\n", status.Proof) + fmt.Printf("Pieces:\t\t%s\n", status.Pieces) + return nil + }, +} + +var sectorsStagedListCmd = &cli.Command{ + Name: "list-staged", // TODO: nest this under a 'staged' subcommand? idk + Usage: "List staged sectors", + Action: func(cctx *cli.Context) error { + nodeApi, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + ctx := lcli.ReqContext(cctx) + + staged, err := nodeApi.SectorsStagedList(ctx) + if err != nil { + return err + } + + for _, s := range staged { + fmt.Println(s) + } + return nil + }, +} + +var sectorsStagedSealCmd = &cli.Command{ + Name: "seal-staged", // TODO: nest this under a 'staged' subcommand? idk + Usage: "Seal staged sectors", + Action: func(cctx *cli.Context) error { + nodeApi, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + ctx := lcli.ReqContext(cctx) + + return nodeApi.SectorsStagedSeal(ctx) + }, +} diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index bc50ff841..20bc8fb27 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -7,6 +7,10 @@ import ( sectorbuilder "github.com/filecoin-project/go-sectorbuilder" ) +type SectorSealingStatus = sectorbuilder.SectorSealingStatus + +type StagedSectorMetadata = sectorbuilder.StagedSectorMetadata + const CommLen = sectorbuilder.CommitmentBytesLen type SectorBuilder struct { @@ -52,16 +56,22 @@ func (sb *SectorBuilder) SealAllStagedSectors() error { return sectorbuilder.SealAllStagedSectors(sb.handle) } -func (sb *SectorBuilder) SealStatus(sector uint64) (sectorbuilder.SectorSealingStatus, error) { +func (sb *SectorBuilder) SealStatus(sector uint64) (SectorSealingStatus, error) { return sectorbuilder.GetSectorSealingStatusByID(sb.handle, sector) } +func (sb *SectorBuilder) GetAllStagedSectors() ([]StagedSectorMetadata, error) { + return sectorbuilder.GetAllStagedSectors(sb.handle) +} + func (sb *SectorBuilder) GeneratePoSt(sortedCommRs [][CommLen]byte, challengeSeed [CommLen]byte) ([][]byte, []uint64, error) { // Wait, this is a blocking method with no way of interrupting it? // does it checkpoint itself? return sectorbuilder.GeneratePoSt(sb.handle, sortedCommRs, challengeSeed) } +var UserBytesForSectorSize = sectorbuilder.GetMaxUserBytesPerStagedSector + func VerifySeal(sectorSize uint64, commR, commD, commRStar [CommLen]byte, proverID address.Address, sectorID uint64, proof []byte) (bool, error) { panic("TODO") // return sectorbuilder.VerifySeal(sectorSize, commR, commD, commRStar, providerID, sectorID, proof) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index cd41b2531..013d04f04 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -2,7 +2,9 @@ package impl import ( "context" + "fmt" "io/ioutil" + "math/rand" "github.com/filecoin-project/go-lotus/api" "github.com/filecoin-project/go-lotus/lib/sectorbuilder" @@ -15,7 +17,8 @@ type StorageMinerAPI struct { } func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) (uint64, error) { - data := make([]byte, 1024) + maxSize := uint64(1016) // this is the most data we can fit in a 1024 byte sector + data := make([]byte, maxSize) fi, err := ioutil.TempFile("", "lotus-garbage") if err != nil { return 0, err @@ -26,7 +29,8 @@ func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) (uint64, error) } fi.Close() - sectorId, err := sm.SectorBuilder.AddPiece("foo", 1024, fi.Name()) + name := fmt.Sprintf("fake-file-%d", rand.Intn(100000000)) + sectorId, err := sm.SectorBuilder.AddPiece(name, maxSize, fi.Name()) if err != nil { return 0, err } @@ -34,4 +38,18 @@ func (sm *StorageMinerAPI) StoreGarbageData(ctx context.Context) (uint64, error) return sectorId, err } +func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid uint64) (sectorbuilder.SectorSealingStatus, error) { + return sm.SectorBuilder.SealStatus(sid) +} + +// List all staged sectors +func (sm *StorageMinerAPI) SectorsStagedList(context.Context) ([]sectorbuilder.StagedSectorMetadata, error) { + return sm.SectorBuilder.GetAllStagedSectors() +} + +// Seal all staged sectors +func (sm *StorageMinerAPI) SectorsStagedSeal(context.Context) error { + return sm.SectorBuilder.SealAllStagedSectors() +} + var _ api.StorageMiner = &StorageMinerAPI{} diff --git a/node/modules/core.go b/node/modules/core.go index b6678d46a..96accfb74 100644 --- a/node/modules/core.go +++ b/node/modules/core.go @@ -26,6 +26,7 @@ import ( "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/routing" record "github.com/libp2p/go-libp2p-record" + "github.com/mitchellh/go-homedir" "go.uber.org/fx" "golang.org/x/xerrors" @@ -221,9 +222,14 @@ func LoadGenesis(genBytes []byte) func(blockstore.Blockstore) Genesis { func SectorBuilderConfig(storagePath string) func() (*sectorbuilder.SectorBuilderConfig, error) { return func() (*sectorbuilder.SectorBuilderConfig, error) { - metadata := filepath.Join(storagePath, "meta") - sealed := filepath.Join(storagePath, "sealed") - staging := filepath.Join(storagePath, "staging") + sp, err := homedir.Expand(storagePath) + if err != nil { + return nil, err + } + + metadata := filepath.Join(sp, "meta") + sealed := filepath.Join(sp, "sealed") + staging := filepath.Join(sp, "staging") // TODO: get the address of the miner actor minerAddr, err := address.NewIDAddress(42)