cerc-io/lotus
15
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

sectorstorage: Address review

Browse Source
This commit is contained in:
Łukasz Magiera 2020-03-27 18:21:32 +01:00
parent e050e0307e
commit f882c5b229
8 changed files with 41 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.circleci/config.yml
View File

@ -42,13 +42,13 @@ commands:
- restore_cache:
name: Restore parameters cache
keys:
- 'v20-1k-lotus-params'
- 'v24-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
- run: ./lotus fetch-params --proving-params 1024
- run: ./lotus fetch-params --proving-params 2048
- save_cache:
name: Save parameters cache
key: 'v20-1k-lotus-params'
key: 'v24-2k-lotus-params'
paths:
- /var/tmp/filecoin-proof-parameters/
install_ipfs:

34
storage/sectorstorage/ffiwrapper/basicfs/fs.go
View File

@ -2,7 +2,6 @@ package basicfs
import (
"context"
"fmt"
"os"
"path/filepath"
"sync"
@ -25,14 +24,24 @@ type Provider struct {
}
func (b *Provider) AcquireSector(ctx context.Context, id abi.SectorID, existing stores.SectorFileType, allocate stores.SectorFileType, sealing bool) (stores.SectorPaths, func(), error) {
os.Mkdir(filepath.Join(b.Root, stores.FTUnsealed.String()), 0755)
os.Mkdir(filepath.Join(b.Root, stores.FTSealed.String()), 0755)
os.Mkdir(filepath.Join(b.Root, stores.FTCache.String()), 0755)
if err := os.Mkdir(filepath.Join(b.Root, stores.FTUnsealed.String()), 0755); err != nil && !os.IsExist(err) {
return stores.SectorPaths{}, nil, err
}
if err := os.Mkdir(filepath.Join(b.Root, stores.FTSealed.String()), 0755); err != nil && !os.IsExist(err) {
return stores.SectorPaths{}, nil, err
}
if err := os.Mkdir(filepath.Join(b.Root, stores.FTCache.String()), 0755); err != nil && !os.IsExist(err) {
return stores.SectorPaths{}, nil, err
}
done := func() {}
for i := 0; i < 3; i++ {
if (existing|allocate)&(1<<i) == 0 {
out := stores.SectorPaths{
Id: id,
}
for _, fileType := range stores.PathTypes {
if !existing.Has(fileType) && !allocate.Has(fileType) {
continue
}
@ -40,10 +49,10 @@ func (b *Provider) AcquireSector(ctx context.Context, id abi.SectorID, existing
if b.waitSector == nil {
b.waitSector = map[sectorFile]chan struct{}{}
}
ch, found := b.waitSector[sectorFile{id, 1 << i}]
ch, found := b.waitSector[sectorFile{id, fileType}]
if !found {
ch = make(chan struct{}, 1)
b.waitSector[sectorFile{id, 1 << i}] = ch
b.waitSector[sectorFile{id, fileType}] = ch
}
b.lk.Unlock()
@ -59,12 +68,9 @@ func (b *Provider) AcquireSector(ctx context.Context, id abi.SectorID, existing
prevDone()
<-ch
}
stores.SetPathByType(&out, fileType, filepath.Join(b.Root, fileType.String(), stores.SectorName(id)))
}
return stores.SectorPaths{
Id: id,
Unsealed: filepath.Join(b.Root, stores.FTUnsealed.String(), fmt.Sprintf("s-t0%d-%d", id.Miner, id.Number)),
Sealed: filepath.Join(b.Root, stores.FTSealed.String(), fmt.Sprintf("s-t0%d-%d", id.Miner, id.Number)),
Cache: filepath.Join(b.Root, stores.FTCache.String(), fmt.Sprintf("s-t0%d-%d", id.Miner, id.Number)),
}, done, nil
return out, done, nil
}

26
storage/sectorstorage/ffiwrapper/config.go
View File

@ -23,12 +23,12 @@ func sizeFromConfig(cfg Config) (abi.SectorSize, error) {
return abi.SectorSize(0), xerrors.New("must specify a PoSt proof type from abi.RegisteredProof")
}
s1, err := sizeFromProofType(cfg.SealProofType)
s1, err := SectorSizeForRegisteredProof(cfg.SealProofType)
if err != nil {
return abi.SectorSize(0), err
}
s2, err := sizeFromProofType(cfg.PoStProofType)
s2, err := SectorSizeForRegisteredProof(cfg.PoStProofType)
if err != nil {
return abi.SectorSize(0), err
}
@ -40,28 +40,6 @@ func sizeFromConfig(cfg Config) (abi.SectorSize, error) {
return s1, nil
}
func sizeFromProofType(p abi.RegisteredProof) (abi.SectorSize, error) {
x, err := p.RegisteredPoStProof()
if err != nil {
return 0, err
}
// values taken from https://github.com/filecoin-project/rust-fil-proofs/blob/master/filecoin-proofs/src/constants.rs#L11
switch x {
case abi.RegisteredProof_StackedDRG32GiBPoSt:
return 1 << 35, nil
case abi.RegisteredProof_StackedDRG2KiBPoSt:
return 2048, nil
case abi.RegisteredProof_StackedDRG8MiBPoSt:
return 1 << 23, nil
case abi.RegisteredProof_StackedDRG512MiBPoSt:
return 1 << 29, nil
default:
return abi.SectorSize(0), xerrors.Errorf("unsupported proof type: %+v", p)
}
}
// TODO: remove this method after implementing it along side the registered proofs and importing it from there.
func SectorSizeForRegisteredProof(p abi.RegisteredProof) (abi.SectorSize, error) {
switch p {

8
storage/sectorstorage/ffiwrapper/sealer_test.go
View File

@ -19,6 +19,7 @@ import (
"github.com/filecoin-project/specs-actors/actors/abi"
"github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/storage/sectorstorage/ffiwrapper/basicfs"
)
@ -149,12 +150,9 @@ func post(t *testing.T, sb *Sealer, seals ...seal) time.Time {
}
func getGrothParamFileAndVerifyingKeys(s abi.SectorSize) {
dat, err := ioutil.ReadFile("./parameters.json")
if err != nil {
panic(xerrors.Errorf("failed to read contents of ./parameters.json: %w", err))
}
dat := build.ParametersJson()
err = paramfetch.GetParams(dat, uint64(s))
err := paramfetch.GetParams(dat, uint64(s))
if err != nil {
panic(xerrors.Errorf("failed to acquire Groth parameters for 2KiB sectors: %w", err))
}

4
storage/sectorstorage/stores/filetype.go
View File

@ -32,6 +32,10 @@ func (t SectorFileType) String() string {
}
}
func (t SectorFileType) Has(singleType SectorFileType) bool {
return t&singleType == singleType
}
type SectorPaths struct {
Id abi.SectorID

6
storage/sectorstorage/stores/index.go
View File

@ -125,7 +125,7 @@ func (i *Index) StorageDeclareSector(ctx context.Context, storageId ID, s abi.Se
i.lk.Lock()
defer i.lk.Unlock()
for _, fileType := range pathTypes {
for _, fileType := range PathTypes {
if fileType&ft == 0 {
continue
}
@ -149,7 +149,7 @@ func (i *Index) StorageDropSector(ctx context.Context, storageId ID, s abi.Secto
i.lk.Lock()
defer i.lk.Unlock()
for _, fileType := range pathTypes {
for _, fileType := range PathTypes {
if fileType&ft == 0 {
continue
}
@ -185,7 +185,7 @@ func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft Sector
storageIDs := map[ID]uint64{}
for _, pathType := range pathTypes {
for _, pathType := range PathTypes {
if ft&pathType == 0 {
continue
}

10
storage/sectorstorage/stores/local.go
View File

@ -41,7 +41,7 @@ type LocalStorage interface {
const MetaFile = "sectorstore.json"
var pathTypes = []SectorFileType{FTUnsealed, FTSealed, FTCache}
var PathTypes = []SectorFileType{FTUnsealed, FTSealed, FTCache}
type Local struct {
localStorage LocalStorage
@ -104,7 +104,7 @@ func (st *Local) OpenPath(ctx context.Context, p string) error {
return xerrors.Errorf("declaring storage in index: %w", err)
}
for _, t := range pathTypes {
for _, t := range PathTypes {
ents, err := ioutil.ReadDir(filepath.Join(p, t.String()))
if err != nil {
if os.IsNotExist(err) {
@ -160,7 +160,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing S
var out SectorPaths
var storageIDs SectorPaths
for _, fileType := range pathTypes {
for _, fileType := range PathTypes {
if fileType&existing == 0 {
continue
}
@ -190,7 +190,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, existing S
}
}
for _, fileType := range pathTypes {
for _, fileType := range PathTypes {
if fileType&allocate == 0 {
continue
}
@ -320,7 +320,7 @@ func (st *Local) MoveStorage(ctx context.Context, s abi.SectorID, types SectorFi
}
defer ddone()
for _, fileType := range pathTypes {
for _, fileType := range PathTypes {
if fileType&types == 0 {
continue
}

2
storage/sectorstorage/stores/remote.go
View File

@ -53,7 +53,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s abi.SectorID, existing Sec
return SectorPaths{}, SectorPaths{}, nil, xerrors.Errorf("local acquire error: %w", err)
}
for _, fileType := range pathTypes {
for _, fileType := range PathTypes {
if fileType&existing == 0 {
continue
}