sectorstorage: Address review

.circleci/config.yml

@@ -42,13 +42,13 @@ commands:
       - restore_cache:
           name: Restore parameters cache
           keys:
-            - 'v20-1k-lotus-params'
+            - 'v24-2k-lotus-params'
           paths:
             - /var/tmp/filecoin-proof-parameters/
-      - run:  ./lotus fetch-params --proving-params 1024
+      - run:  ./lotus fetch-params --proving-params 2048
       - save_cache:
           name: Save parameters cache
-          key: 'v20-1k-lotus-params'
+          key: 'v24-2k-lotus-params'
           paths:
             - /var/tmp/filecoin-proof-parameters/
   install_ipfs:

storage/sectorstorage/ffiwrapper/basicfs/fs.go

@@ -2,7 +2,6 @@ package basicfs
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"path/filepath"
 	"sync"
@@ -25,14 +24,24 @@ type Provider struct {
 }
 
 func (b *Provider) AcquireSector(ctx context.Context, id abi.SectorID, existing stores.SectorFileType, allocate stores.SectorFileType, sealing bool) (stores.SectorPaths, func(), error) {
-	os.Mkdir(filepath.Join(b.Root, stores.FTUnsealed.String()), 0755)
+	if err := os.Mkdir(filepath.Join(b.Root, stores.FTUnsealed.String()), 0755); err != nil && !os.IsExist(err) {
-	os.Mkdir(filepath.Join(b.Root, stores.FTSealed.String()), 0755)
+		return stores.SectorPaths{}, nil, err
-	os.Mkdir(filepath.Join(b.Root, stores.FTCache.String()), 0755)
+	}
+	if err := os.Mkdir(filepath.Join(b.Root, stores.FTSealed.String()), 0755); err != nil && !os.IsExist(err) {
+		return stores.SectorPaths{}, nil, err
+	}
+	if err := os.Mkdir(filepath.Join(b.Root, stores.FTCache.String()), 0755); err != nil && !os.IsExist(err) {
+		return stores.SectorPaths{}, nil, err
+	}
 
 	done := func() {}
 
-	for i := 0; i < 3; i++ {
+	out := stores.SectorPaths{
-		if (existing|allocate)&(1<<i) == 0 {
+		Id: id,
+	}
+
+	for _, fileType := range stores.PathTypes {
+		if !existing.Has(fileType) && !allocate.Has(fileType) {
 			continue
 		}
 
@@ -40,10 +49,10 @@ func (b *Provider) AcquireSector(ctx context.Context, id abi.SectorID, existing
 		if b.waitSector == nil {
 			b.waitSector = map[sectorFile]chan struct{}{}
 		}
-		ch, found := b.waitSector[sectorFile{id, 1 << i}]
+		ch, found := b.waitSector[sectorFile{id, fileType}]
 		if !found {
 			ch = make(chan struct{}, 1)
-			b.waitSector[sectorFile{id, 1 << i}] = ch
+			b.waitSector[sectorFile{id, fileType}] = ch
 		}
 		b.lk.Unlock()
 
@@ -59,12 +68,9 @@ func (b *Provider) AcquireSector(ctx context.Context, id abi.SectorID, existing
 			prevDone()
 			<-ch
 		}
+
+		stores.SetPathByType(&out, fileType, filepath.Join(b.Root, fileType.String(), stores.SectorName(id)))
 	}
 
-	return stores.SectorPaths{
+	return out, done, nil
-		Id:       id,
-		Unsealed: filepath.Join(b.Root, stores.FTUnsealed.String(), fmt.Sprintf("s-t0%d-%d", id.Miner, id.Number)),
-		Sealed:   filepath.Join(b.Root, stores.FTSealed.String(), fmt.Sprintf("s-t0%d-%d", id.Miner, id.Number)),
-		Cache:    filepath.Join(b.Root, stores.FTCache.String(), fmt.Sprintf("s-t0%d-%d", id.Miner, id.Number)),
-	}, done, nil
 }

storage/sectorstorage/ffiwrapper/config.go

@@ -23,12 +23,12 @@ func sizeFromConfig(cfg Config) (abi.SectorSize, error) {
 		return abi.SectorSize(0), xerrors.New("must specify a PoSt proof type from abi.RegisteredProof")
 	}
 
-	s1, err := sizeFromProofType(cfg.SealProofType)
+	s1, err := SectorSizeForRegisteredProof(cfg.SealProofType)
 	if err != nil {
 		return abi.SectorSize(0), err
 	}
 
-	s2, err := sizeFromProofType(cfg.PoStProofType)
+	s2, err := SectorSizeForRegisteredProof(cfg.PoStProofType)
 	if err != nil {
 		return abi.SectorSize(0), err
 	}
@@ -40,28 +40,6 @@ func sizeFromConfig(cfg Config) (abi.SectorSize, error) {
 	return s1, nil
 }
 
-func sizeFromProofType(p abi.RegisteredProof) (abi.SectorSize, error) {
-	x, err := p.RegisteredPoStProof()
-	if err != nil {
-		return 0, err
-	}
-
-	// values taken from https://github.com/filecoin-project/rust-fil-proofs/blob/master/filecoin-proofs/src/constants.rs#L11
-
-	switch x {
-	case abi.RegisteredProof_StackedDRG32GiBPoSt:
-		return 1 << 35, nil
-	case abi.RegisteredProof_StackedDRG2KiBPoSt:
-		return 2048, nil
-	case abi.RegisteredProof_StackedDRG8MiBPoSt:
-		return 1 << 23, nil
-	case abi.RegisteredProof_StackedDRG512MiBPoSt:
-		return 1 << 29, nil
-	default:
-		return abi.SectorSize(0), xerrors.Errorf("unsupported proof type: %+v", p)
-	}
-}
-
 // TODO: remove this method after implementing it along side the registered proofs and importing it from there.
 func SectorSizeForRegisteredProof(p abi.RegisteredProof) (abi.SectorSize, error) {
 	switch p {

storage/sectorstorage/ffiwrapper/sealer_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-storage/storage"
 
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper/basicfs"
 )
 
@@ -149,12 +150,9 @@ func post(t *testing.T, sb *Sealer, seals ...seal) time.Time {
 }
 
 func getGrothParamFileAndVerifyingKeys(s abi.SectorSize) {
-	dat, err := ioutil.ReadFile("./parameters.json")
+	dat := build.ParametersJson()
-	if err != nil {
-		panic(xerrors.Errorf("failed to read contents of ./parameters.json: %w", err))
-	}
 
-	err = paramfetch.GetParams(dat, uint64(s))
+	err := paramfetch.GetParams(dat, uint64(s))
 	if err != nil {
 		panic(xerrors.Errorf("failed to acquire Groth parameters for 2KiB sectors: %w", err))
 	}

storage/sectorstorage/stores/filetype.go

@@ -32,6 +32,10 @@ func (t SectorFileType) String() string {
 	}
 }
 
+func (t SectorFileType) Has(singleType SectorFileType) bool {
+	return t&singleType == singleType
+}
+
 type SectorPaths struct {
 	Id abi.SectorID
 

storage/sectorstorage/stores/index.go

@@ -125,7 +125,7 @@ func (i *Index) StorageDeclareSector(ctx context.Context, storageId ID, s abi.Se
 	i.lk.Lock()
 	defer i.lk.Unlock()
 
-	for _, fileType := range pathTypes {
+	for _, fileType := range PathTypes {
 		if fileType&ft == 0 {
 			continue
 		}
@@ -149,7 +149,7 @@ func (i *Index) StorageDropSector(ctx context.Context, storageId ID, s abi.Secto
 	i.lk.Lock()
 	defer i.lk.Unlock()
 
-	for _, fileType := range pathTypes {
+	for _, fileType := range PathTypes {
 		if fileType&ft == 0 {
 			continue
 		}
@@ -185,7 +185,7 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft Sector
 
 	storageIDs := map[ID]uint64{}
 
-	for _, pathType := range pathTypes {
+	for _, pathType := range PathTypes {
 		if ft&pathType == 0 {
 			continue
 		}

storage/sectorstorage/stores/local.go

@@ -41,7 +41,7 @@ type LocalStorage interface {
 
 const MetaFile = "sectorstore.json"
 
-var pathTypes = []SectorFileType{FTUnsealed, FTSealed, FTCache}
+var PathTypes = []SectorFileType{FTUnsealed, FTSealed, FTCache}
 
 type Local struct {
 	localStorage LocalStorage
@@ -104,7 +104,7 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
 		return xerrors.Errorf("declaring storage in index: %w", err)
 	}
 
-	for _, t := range pathTypes {
+	for _, t := range PathTypes {
 		ents, err := ioutil.ReadDir(filepath.Join(p, t.String()))
 		if err != nil {
 			if os.IsNotExist(err) {
@@ -160,7 +160,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing S
 	var out SectorPaths
 	var storageIDs SectorPaths
 
-	for _, fileType := range pathTypes {
+	for _, fileType := range PathTypes {
 		if fileType&existing == 0 {
 			continue
 		}
@@ -190,7 +190,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing S
 		}
 	}
 
-	for _, fileType := range pathTypes {
+	for _, fileType := range PathTypes {
 		if fileType&allocate == 0 {
 			continue
 		}
@@ -320,7 +320,7 @@ func (st *Local) MoveStorage(ctx context.Context, s abi.SectorID, types SectorFi
 	}
 	defer ddone()
 
-	for _, fileType := range pathTypes {
+	for _, fileType := range PathTypes {
 		if fileType&types == 0 {
 			continue
 		}

storage/sectorstorage/stores/remote.go

@@ -53,7 +53,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing Sec
 		return SectorPaths{}, SectorPaths{}, nil, xerrors.Errorf("local acquire error: %w", err)
 	}
 
-	for _, fileType := range pathTypes {
+	for _, fileType := range PathTypes {
 		if fileType&existing == 0 {
 			continue
 		}