cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

curio: feat: break trees task into TreeD(prefetch) and TreeRC (#11895)

Browse Source 
* break trees task

* fix TreeD reservation

* fix nil pointer err

* apply suggestions

* fix allocate file types

* fix dbIndex inserts

* set resource, move release func

* refactor func(), update memory

* remove extra release
This commit is contained in:
LexLuthr 2024-05-02 14:47:33 +05:30 committed by GitHub
parent ecc82d4526
commit 00edad4e4d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 450 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/curio/tasks/tasks.go
View File

@ -114,9 +114,10 @@ func StartTasks(ctx context.Context, dependencies *deps.Deps) (*harmonytask.Task
activeTasks = append(activeTasks, sdrTask)
}
if cfg.Subsystems.EnableSealSDRTrees {
treesTask := seal.NewTreesTask(sp, db, slr, cfg.Subsystems.SealSDRTreesMaxTasks)
treeDTask := seal.NewTreeDTask(sp, db, slr, cfg.Subsystems.SealSDRTreesMaxTasks)
treeRCTask := seal.NewTreeRCTask(sp, db, slr, cfg.Subsystems.SealSDRTreesMaxTasks)
finalizeTask := seal.NewFinalizeTask(cfg.Subsystems.FinalizeMaxTasks, sp, slr, db)
activeTasks = append(activeTasks, treesTask, finalizeTask)
activeTasks = append(activeTasks, treeDTask, treeRCTask, finalizeTask)
}
if cfg.Subsystems.EnableSendPrecommitMsg {
precommitTask := seal.NewSubmitPrecommitTask(sp, db, full, sender, as, cfg.Fees.MaxPreCommitGasFee)

146
curiosrc/ffi/sdr_funcs.go
View File

@ -61,7 +61,7 @@ type storageProvider struct {
}
func (l *storageProvider) AcquireSector(ctx context.Context, taskID *harmonytask.TaskID, sector storiface.SectorRef, existing, allocate storiface.SectorFileType, sealing storiface.PathType) (fspaths, ids storiface.SectorPaths, release func(), err error) {
var paths, storageIDs storiface.SectorPaths
var sectorPaths, storageIDs storiface.SectorPaths
var releaseStorage func()
var ok bool
@ -77,7 +77,7 @@ func (l *storageProvider) AcquireSector(ctx context.Context, taskID *harmonytask
log.Debugw("using existing storage reservation", "task", taskID, "sector", sector, "existing", existing, "allocate", allocate)
paths = resv.Paths
sectorPaths = resv.Paths
storageIDs = resv.PathIDs
releaseStorage = resv.Release
@ -87,7 +87,7 @@ func (l *storageProvider) AcquireSector(ctx context.Context, taskID *harmonytask
// present locally. Note that we do not care about 'allocate' reqeuests, those files don't exist, and are just
// proposed paths with a reservation of space.
_, checkPathIDs, err := l.storage.AcquireSector(ctx, sector, existing, storiface.FTNone, sealing, storiface.AcquireMove, storiface.AcquireInto(storiface.PathsWithIDs{Paths: paths, IDs: storageIDs}))
_, checkPathIDs, err := l.storage.AcquireSector(ctx, sector, existing, storiface.FTNone, sealing, storiface.AcquireMove, storiface.AcquireInto(storiface.PathsWithIDs{Paths: sectorPaths, IDs: storageIDs}))
if err != nil {
return storiface.SectorPaths{}, storiface.SectorPaths{}, nil, xerrors.Errorf("acquire reserved existing files: %w", err)
}
@ -101,20 +101,20 @@ func (l *storageProvider) AcquireSector(ctx context.Context, taskID *harmonytask
// No related reservation, acquire storage as usual
var err error
paths, storageIDs, err = l.storage.AcquireSector(ctx, sector, existing, allocate, sealing, storiface.AcquireMove)
sectorPaths, storageIDs, err = l.storage.AcquireSector(ctx, sector, existing, allocate, sealing, storiface.AcquireMove)
if err != nil {
return storiface.SectorPaths{}, storiface.SectorPaths{}, nil, err
}
releaseStorage, err = l.localStore.Reserve(ctx, sector, allocate, storageIDs, storiface.FSOverheadSeal)
releaseStorage, err = l.localStore.Reserve(ctx, sector, allocate, storageIDs, storiface.FSOverheadSeal, paths.MinFreeStoragePercentage)
if err != nil {
return storiface.SectorPaths{}, storiface.SectorPaths{}, nil, xerrors.Errorf("reserving storage space: %w", err)
}
}
log.Debugf("acquired sector %d (e:%d; a:%d): %v", sector, existing, allocate, paths)
log.Debugf("acquired sector %d (e:%d; a:%d): %v", sector, existing, allocate, sectorPaths)
return paths, storageIDs, func() {
return sectorPaths, storageIDs, func() {
releaseStorage()
for _, fileType := range storiface.PathTypes {
@ -194,13 +194,13 @@ func (sb *SealCalls) ensureOneCopy(ctx context.Context, sid abi.SectorID, pathID
return nil
}
func (sb *SealCalls) TreeDRC(ctx context.Context, task *harmonytask.TaskID, sector storiface.SectorRef, unsealed cid.Cid, size abi.PaddedPieceSize, data io.Reader, unpaddedData bool) (scid cid.Cid, ucid cid.Cid, err error) {
func (sb *SealCalls) TreeRC(ctx context.Context, task *harmonytask.TaskID, sector storiface.SectorRef, unsealed cid.Cid) (scid cid.Cid, ucid cid.Cid, err error) {
p1o, err := sb.makePhase1Out(unsealed, sector.ProofType)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("make phase1 output: %w", err)
}
paths, pathIDs, releaseSector, err := sb.sectors.AcquireSector(ctx, task, sector, storiface.FTCache, storiface.FTSealed, storiface.PathSealing)
fspaths, pathIDs, releaseSector, err := sb.sectors.AcquireSector(ctx, task, sector, storiface.FTCache, storiface.FTSealed, storiface.PathSealing)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("acquiring sector paths: %w", err)
}
@ -208,66 +208,55 @@ func (sb *SealCalls) TreeDRC(ctx context.Context, task *harmonytask.TaskID, sect
defer func() {
if err != nil {
clerr := removeDRCTrees(paths.Cache)
clerr := removeDRCTrees(fspaths.Cache, false)
if clerr != nil {
log.Errorw("removing tree files after TreeDRC error", "error", clerr, "exec-error", err, "sector", sector, "cache", paths.Cache)
log.Errorw("removing tree files after TreeDRC error", "error", clerr, "exec-error", err, "sector", sector, "cache", fspaths.Cache)
}
}
}()
treeDUnsealed, err := proof.BuildTreeD(data, unpaddedData, filepath.Join(paths.Cache, proofpaths.TreeDName), size)
// create sector-sized file at paths.Sealed; PC2 transforms it into a sealed sector in-place
ssize, err := sector.ProofType.SectorSize()
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("building tree-d: %w", err)
}
if treeDUnsealed != unsealed {
return cid.Undef, cid.Undef, xerrors.Errorf("tree-d cid mismatch with supplied unsealed cid")
return cid.Undef, cid.Undef, xerrors.Errorf("getting sector size: %w", err)
}
{
// create sector-sized file at paths.Sealed; PC2 transforms it into a sealed sector in-place
ssize, err := sector.ProofType.SectorSize()
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("getting sector size: %w", err)
}
// copy TreeD prefix to sealed sector, SealPreCommitPhase2 will mutate it in place into the sealed sector
{
// copy TreeD prefix to sealed sector, SealPreCommitPhase2 will mutate it in place into the sealed sector
// first try reflink + truncate, that should be way faster
err := reflink.Always(filepath.Join(fspaths.Cache, proofpaths.TreeDName), fspaths.Sealed)
if err == nil {
err = os.Truncate(fspaths.Sealed, int64(ssize))
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("truncating reflinked sealed file: %w", err)
}
} else {
log.Errorw("reflink treed -> sealed failed, falling back to slow copy, use single scratch btrfs or xfs filesystem", "error", err, "sector", sector, "cache", fspaths.Cache, "sealed", fspaths.Sealed)
// first try reflink + truncate, that should be way faster
err := reflink.Always(filepath.Join(paths.Cache, proofpaths.TreeDName), paths.Sealed)
if err == nil {
err = os.Truncate(paths.Sealed, int64(ssize))
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("truncating reflinked sealed file: %w", err)
}
} else {
log.Errorw("reflink treed -> sealed failed, falling back to slow copy, use single scratch btrfs or xfs filesystem", "error", err, "sector", sector, "cache", paths.Cache, "sealed", paths.Sealed)
// fallback to slow copy, copy ssize bytes from treed to sealed
dst, err := os.OpenFile(fspaths.Sealed, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("opening sealed sector file: %w", err)
}
src, err := os.Open(filepath.Join(fspaths.Cache, proofpaths.TreeDName))
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("opening treed sector file: %w", err)
}
// fallback to slow copy, copy ssize bytes from treed to sealed
dst, err := os.OpenFile(paths.Sealed, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("opening sealed sector file: %w", err)
}
src, err := os.Open(filepath.Join(paths.Cache, proofpaths.TreeDName))
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("opening treed sector file: %w", err)
}
_, err = io.CopyN(dst, src, int64(ssize))
derr := dst.Close()
_ = src.Close()
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("copying treed -> sealed: %w", err)
}
if derr != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("closing sealed file: %w", derr)
}
_, err = io.CopyN(dst, src, int64(ssize))
derr := dst.Close()
_ = src.Close()
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("copying treed -> sealed: %w", err)
}
if derr != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("closing sealed file: %w", derr)
}
}
}
sl, uns, err := ffi.SealPreCommitPhase2(p1o, paths.Cache, paths.Sealed)
sl, uns, err := ffi.SealPreCommitPhase2(p1o, fspaths.Cache, fspaths.Sealed)
if err != nil {
return cid.Undef, cid.Undef, xerrors.Errorf("computing seal proof: %w", err)
}
@ -283,22 +272,28 @@ func (sb *SealCalls) TreeDRC(ctx context.Context, task *harmonytask.TaskID, sect
return sl, uns, nil
}
func removeDRCTrees(cache string) error {
// list files in cache
func removeDRCTrees(cache string, isDTree bool) error {
files, err := os.ReadDir(cache)
if err != nil {
return xerrors.Errorf("listing cache: %w", err)
}
var testFunc func(string) bool
if isDTree {
testFunc = proofpaths.IsTreeDFile
} else {
testFunc = proofpaths.IsTreeRCFile
}
for _, file := range files {
if proofpaths.IsTreeFile(file.Name()) {
if testFunc(file.Name()) {
err := os.Remove(filepath.Join(cache, file.Name()))
if err != nil {
return xerrors.Errorf("removing tree file: %w", err)
}
}
}
return nil
}
@ -625,3 +620,40 @@ func (sb *SealCalls) sectorStorageType(ctx context.Context, sector storiface.Sec
return true, storiface.PathStorage, nil
}
// PreFetch fetches the sector file to local storage before SDR and TreeRC Tasks
func (sb *SealCalls) PreFetch(ctx context.Context, sector storiface.SectorRef, task *harmonytask.TaskID) (fsPath, pathID storiface.SectorPaths, releaseSector func(), err error) {
fsPath, pathID, releaseSector, err = sb.sectors.AcquireSector(ctx, task, sector, storiface.FTCache, storiface.FTNone, storiface.PathSealing)
if err != nil {
return storiface.SectorPaths{}, storiface.SectorPaths{}, nil, xerrors.Errorf("acquiring sector paths: %w", err)
}
// Don't release the storage locks. They will be released in TreeD func()
return
}
func (sb *SealCalls) TreeD(ctx context.Context, sector storiface.SectorRef, unsealed cid.Cid, size abi.PaddedPieceSize, data io.Reader, unpaddedData bool, fspaths, pathIDs storiface.SectorPaths) error {
var err error
defer func() {
if err != nil {
clerr := removeDRCTrees(fspaths.Cache, true)
if clerr != nil {
log.Errorw("removing tree files after TreeDRC error", "error", clerr, "exec-error", err, "sector", sector, "cache", fspaths.Cache)
}
}
}()
treeDUnsealed, err := proof.BuildTreeD(data, unpaddedData, filepath.Join(fspaths.Cache, proofpaths.TreeDName), size)
if err != nil {
return xerrors.Errorf("building tree-d: %w", err)
}
if treeDUnsealed != unsealed {
return xerrors.Errorf("tree-d cid mismatch with supplied unsealed cid")
}
if err := sb.ensureOneCopy(ctx, sector.ID, pathIDs, storiface.FTCache); err != nil {
return xerrors.Errorf("ensure one copy: %w", err)
}
return nil
}

20
curiosrc/ffi/task_storage.go
View File

@ -43,6 +43,9 @@ type TaskStorage struct {
pathType storiface.PathType
taskToSectorRef func(taskID harmonytask.TaskID) (SectorRef, error)
// Minimum free storage percentage cutoff for reservation rejection
MinFreeStoragePercentage float64
}
type ReleaseStorageFunc func() // free storage reservation
@ -56,14 +59,15 @@ type StorageReservation struct {
Alloc, Existing storiface.SectorFileType
}
func (sb *SealCalls) Storage(taskToSectorRef func(taskID harmonytask.TaskID) (SectorRef, error), alloc, existing storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) *TaskStorage {
func (sb *SealCalls) Storage(taskToSectorRef func(taskID harmonytask.TaskID) (SectorRef, error), alloc, existing storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType, MinFreeStoragePercentage float64) *TaskStorage {
return &TaskStorage{
sc: sb,
alloc: alloc,
existing: existing,
ssize: ssize,
pathType: pathType,
taskToSectorRef: taskToSectorRef,
sc: sb,
alloc: alloc,
existing: existing,
ssize: ssize,
pathType: pathType,
taskToSectorRef: taskToSectorRef,
MinFreeStoragePercentage: MinFreeStoragePercentage,
}
}
@ -166,7 +170,7 @@ func (t *TaskStorage) Claim(taskID int) error {
}
// reserve the space
release, err := t.sc.sectors.localStore.Reserve(ctx, sectorRef.Ref(), requestedTypes, pathIDs, storiface.FSOverheadSeal)
release, err := t.sc.sectors.localStore.Reserve(ctx, sectorRef.Ref(), requestedTypes, pathIDs, storiface.FSOverheadSeal, t.MinFreeStoragePercentage)
if err != nil {
return err
}

1
curiosrc/gc/storage_endpoint_gc.go
View File

@ -209,6 +209,7 @@ func (s *StorageEndpointGC) Do(taskID harmonytask.TaskID, stillOwned func() bool
// Remove dead URLs from storage_path entries and handle path cleanup
for _, du := range deadURLs {
du := du
// Fetch the current URLs for the storage path
var URLs string
err = tx.QueryRow("SELECT urls FROM storage_path WHERE storage_id = $1", du.StorageID).Scan(&URLs)

3
curiosrc/piece/task_park_piece.go
View File

@ -15,6 +15,7 @@ import (
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/lib/harmony/resources"
"github.com/filecoin-project/lotus/lib/promise"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
@ -185,7 +186,7 @@ func (p *ParkPieceTask) TypeDetails() harmonytask.TaskTypeDetails {
Cpu: 1,
Gpu: 0,
Ram: 64 << 20,
Storage: p.sc.Storage(p.taskToRef, storiface.FTPiece, storiface.FTNone, maxSizePiece, storiface.PathSealing),
Storage: p.sc.Storage(p.taskToRef, storiface.FTPiece, storiface.FTNone, maxSizePiece, storiface.PathSealing, paths.MinFreeStoragePercentage),
},
MaxFailures: 10,
}

45
curiosrc/seal/poller.go
View File

@ -21,7 +21,8 @@ var log = logging.Logger("lpseal")
const (
pollerSDR = iota
pollerTrees
pollerTreeD
pollerTreeRC
pollerPrecommitMsg
pollerPoRep
pollerCommitMsg
@ -154,7 +155,8 @@ func (s *SealPoller) poll(ctx context.Context) error {
}
s.pollStartSDR(ctx, task)
s.pollStartSDRTrees(ctx, task)
s.pollStartSDRTreeD(ctx, task)
s.pollStartSDRTreeRC(ctx, task)
s.pollStartPrecommitMsg(ctx, task)
s.mustPoll(s.pollPrecommitMsgLanded(ctx, task))
s.pollStartPoRep(ctx, task, ts)
@ -187,14 +189,10 @@ func (t pollTask) afterSDR() bool {
return t.AfterSDR
}
func (s *SealPoller) pollStartSDRTrees(ctx context.Context, task pollTask) {
if !task.AfterTreeD && !task.AfterTreeC && !task.AfterTreeR &&
task.TaskTreeD == nil && task.TaskTreeC == nil && task.TaskTreeR == nil &&
s.pollers[pollerTrees].IsSet() && task.AfterSDR {
s.pollers[pollerTrees].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) {
n, err := tx.Exec(`UPDATE sectors_sdr_pipeline SET task_id_tree_d = $1, task_id_tree_c = $1, task_id_tree_r = $1
WHERE sp_id = $2 AND sector_number = $3 AND after_sdr = TRUE AND task_id_tree_d IS NULL AND task_id_tree_c IS NULL AND task_id_tree_r IS NULL`, id, task.SpID, task.SectorNumber)
func (s *SealPoller) pollStartSDRTreeD(ctx context.Context, task pollTask) {
if !task.AfterTreeD && task.TaskTreeD == nil && s.pollers[pollerTreeD].IsSet() && task.afterSDR() {
s.pollers[pollerTreeD].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) {
n, err := tx.Exec(`UPDATE sectors_sdr_pipeline SET task_id_tree_d = $1 WHERE sp_id = $2 AND sector_number = $3 AND after_sdr = TRUE AND task_id_tree_d IS NULL`, id, task.SpID, task.SectorNumber)
if err != nil {
return false, xerrors.Errorf("update sectors_sdr_pipeline: %w", err)
}
@ -207,12 +205,33 @@ func (s *SealPoller) pollStartSDRTrees(ctx context.Context, task pollTask) {
}
}
func (t pollTask) afterTrees() bool {
return t.AfterTreeD && t.AfterTreeC && t.AfterTreeR && t.afterSDR()
func (t pollTask) afterTreeD() bool {
return t.AfterTreeD && t.afterSDR()
}
func (s *SealPoller) pollStartSDRTreeRC(ctx context.Context, task pollTask) {
if !task.AfterTreeC && !task.AfterTreeR && task.TaskTreeC == nil && task.TaskTreeR == nil && s.pollers[pollerTreeRC].IsSet() && task.afterTreeD() {
s.pollers[pollerTreeRC].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) {
n, err := tx.Exec(`UPDATE sectors_sdr_pipeline SET task_id_tree_c = $1, task_id_tree_r = $1
WHERE sp_id = $2 AND sector_number = $3 AND after_tree_d = TRUE AND task_id_tree_c IS NULL AND task_id_tree_r IS NULL`, id, task.SpID, task.SectorNumber)
if err != nil {
return false, xerrors.Errorf("update sectors_sdr_pipeline: %w", err)
}
if n != 1 {
return false, xerrors.Errorf("expected to update 1 row, updated %d", n)
}
return true, nil
})
}
}
func (t pollTask) afterTreeRC() bool {
return t.AfterTreeC && t.AfterTreeR && t.afterTreeD()
}
func (t pollTask) afterPrecommitMsg() bool {
return t.AfterPrecommitMsg && t.afterTrees()
return t.AfterPrecommitMsg && t.afterTreeRC()
}
func (t pollTask) afterPrecommitMsgSuccess() bool {

2
curiosrc/seal/poller_precommit_msg.go
View File

@ -16,7 +16,7 @@ import (
)
func (s *SealPoller) pollStartPrecommitMsg(ctx context.Context, task pollTask) {
if task.TaskPrecommitMsg == nil && !task.AfterPrecommitMsg && task.afterTrees() && s.pollers[pollerPrecommitMsg].IsSet() {
if task.TaskPrecommitMsg == nil && !task.AfterPrecommitMsg && task.afterTreeRC() && s.pollers[pollerPrecommitMsg].IsSet() {
s.pollers[pollerPrecommitMsg].Val(ctx)(func(id harmonytask.TaskID, tx *harmonydb.Tx) (shouldCommit bool, seriousError error) {
n, err := tx.Exec(`UPDATE sectors_sdr_pipeline SET task_id_precommit_msg = $1 WHERE sp_id = $2 AND sector_number = $3 AND task_id_precommit_msg IS NULL AND after_tree_r = TRUE AND after_tree_d = TRUE`, id, task.SpID, task.SectorNumber)
if err != nil {

3
curiosrc/seal/task_movestorage.go
View File

@ -11,6 +11,7 @@ import (
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/lib/harmony/resources"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
@ -148,7 +149,7 @@ func (m *MoveStorageTask) TypeDetails() harmonytask.TaskTypeDetails {
Cpu: 1,
Gpu: 0,
Ram: 128 << 20,
Storage: m.sc.Storage(m.taskToSector, storiface.FTNone, storiface.FTCache|storiface.FTSealed|storiface.FTUnsealed, ssize, storiface.PathStorage),
Storage: m.sc.Storage(m.taskToSector, storiface.FTNone, storiface.FTCache|storiface.FTSealed|storiface.FTUnsealed, ssize, storiface.PathStorage, paths.MinFreeStoragePercentage),
},
MaxFailures: 10,
}

3
curiosrc/seal/task_sdr.go
View File

@ -20,6 +20,7 @@ import (
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/lib/harmony/resources"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
@ -204,7 +205,7 @@ func (s *SDRTask) TypeDetails() harmonytask.TaskTypeDetails {
Cpu: 4, // todo multicore sdr
Gpu: 0,
Ram: 54 << 30,
Storage: s.sc.Storage(s.taskToSector, storiface.FTCache, storiface.FTNone, ssize, storiface.PathSealing),
Storage: s.sc.Storage(s.taskToSector, storiface.FTCache, storiface.FTNone, ssize, storiface.PathSealing, paths.MinFreeStoragePercentage),
},
MaxFailures: 2,
Follows: nil,

144
curiosrc/seal/task_trees.go → curiosrc/seal/task_treed.go
View File

@ -23,7 +23,7 @@ import (
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
type TreesTask struct {
type TreeDTask struct {
sp *SealPoller
db *harmonydb.DB
sc *ffi.SealCalls
@ -31,8 +31,54 @@ type TreesTask struct {
max int
}
func NewTreesTask(sp *SealPoller, db *harmonydb.DB, sc *ffi.SealCalls, maxTrees int) *TreesTask {
return &TreesTask{
func (t *TreeDTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
if engine.Resources().Gpu > 0 {
return &ids[0], nil
}
return nil, nil
}
func (t *TreeDTask) TypeDetails() harmonytask.TaskTypeDetails {
ssize := abi.SectorSize(32 << 30) // todo task details needs taskID to get correct sector size
if isDevnet {
ssize = abi.SectorSize(2 << 20)
}
return harmonytask.TaskTypeDetails{
Max: t.max,
Name: "TreeD",
Cost: resources.Resources{
Cpu: 1,
Ram: 1 << 30,
Gpu: 0,
Storage: t.sc.Storage(t.taskToSector, storiface.FTNone, storiface.FTCache, ssize, storiface.PathSealing, 1.0),
},
MaxFailures: 3,
Follows: nil,
}
}
func (t *TreeDTask) taskToSector(id harmonytask.TaskID) (ffi.SectorRef, error) {
var refs []ffi.SectorRef
err := t.db.Select(context.Background(), &refs, `SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_tree_d = $1`, id)
if err != nil {
return ffi.SectorRef{}, xerrors.Errorf("getting sector ref: %w", err)
}
if len(refs) != 1 {
return ffi.SectorRef{}, xerrors.Errorf("expected 1 sector ref, got %d", len(refs))
}
return refs[0], nil
}
func (t *TreeDTask) Adder(taskFunc harmonytask.AddTaskFunc) {
t.sp.pollers[pollerTreeD].Set(taskFunc)
}
func NewTreeDTask(sp *SealPoller, db *harmonydb.DB, sc *ffi.SealCalls, maxTrees int) *TreeDTask {
return &TreeDTask{
sp: sp,
db: db,
sc: sc,
@ -41,7 +87,7 @@ func NewTreesTask(sp *SealPoller, db *harmonydb.DB, sc *ffi.SealCalls, maxTrees
}
}
func (t *TreesTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
func (t *TreeDTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
ctx := context.Background()
var sectorParamsArr []struct {
@ -53,7 +99,7 @@ func (t *TreesTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done
err = t.db.Select(ctx, &sectorParamsArr, `
SELECT sp_id, sector_number, reg_seal_proof
FROM sectors_sdr_pipeline
WHERE task_id_tree_r = $1 AND task_id_tree_c = $1 AND task_id_tree_d = $1`, taskID)
WHERE task_id_tree_d = $1`, taskID)
if err != nil {
return false, xerrors.Errorf("getting sector params: %w", err)
}
@ -63,6 +109,21 @@ func (t *TreesTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done
}
sectorParams := sectorParamsArr[0]
sref := storiface.SectorRef{
ID: abi.SectorID{
Miner: abi.ActorID(sectorParams.SpID),
Number: abi.SectorNumber(sectorParams.SectorNumber),
},
ProofType: sectorParams.RegSealProof,
}
// Fetch the Sector to local storage
fsPaths, pathIds, release, err := t.sc.PreFetch(ctx, sref, &taskID)
if err != nil {
return false, xerrors.Errorf("failed to prefetch sectors: %w", err)
}
defer release()
var pieces []struct {
PieceIndex int64 `db:"piece_index"`
PieceCID string `db:"piece_cid"`
@ -178,82 +239,25 @@ func (t *TreesTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done
unpaddedData = false // nullreader includes fr32 zero bits
}
sref := storiface.SectorRef{
ID: abi.SectorID{
Miner: abi.ActorID(sectorParams.SpID),
Number: abi.SectorNumber(sectorParams.SectorNumber),
},
ProofType: sectorParams.RegSealProof,
}
// D / R / C
sealed, unsealed, err := t.sc.TreeDRC(ctx, &taskID, sref, commd, abi.PaddedPieceSize(ssize), dataReader, unpaddedData)
// Generate Tree D
err = t.sc.TreeD(ctx, sref, commd, abi.PaddedPieceSize(ssize), dataReader, unpaddedData, fsPaths, pathIds)
if err != nil {
return false, xerrors.Errorf("computing tree d, r and c: %w", err)
return false, xerrors.Errorf("failed to generate TreeD: %w", err)
}
// todo synth porep
// todo porep challenge check
n, err := t.db.Exec(ctx, `UPDATE sectors_sdr_pipeline
SET after_tree_r = true, after_tree_c = true, after_tree_d = true, tree_r_cid = $3, tree_d_cid = $4
WHERE sp_id = $1 AND sector_number = $2`,
sectorParams.SpID, sectorParams.SectorNumber, sealed, unsealed)
SET after_tree_d = true, tree_d_cid = $3 WHERE sp_id = $1 AND sector_number = $2`,
sectorParams.SpID, sectorParams.SectorNumber, commd)
if err != nil {
return false, xerrors.Errorf("store sdr-trees success: updating pipeline: %w", err)
return false, xerrors.Errorf("store TreeD success: updating pipeline: %w", err)
}
if n != 1 {
return false, xerrors.Errorf("store sdr-trees success: updated %d rows", n)
return false, xerrors.Errorf("store TreeD success: updated %d rows", n)
}
return true, nil
}
func (t *TreesTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
id := ids[0]
return &id, nil
}
func (t *TreesTask) TypeDetails() harmonytask.TaskTypeDetails {
ssize := abi.SectorSize(32 << 30) // todo task details needs taskID to get correct sector size
if isDevnet {
ssize = abi.SectorSize(2 << 20)
}
return harmonytask.TaskTypeDetails{
Max: t.max,
Name: "SDRTrees",
Cost: resources.Resources{
Cpu: 1,
Gpu: 1,
Ram: 8000 << 20, // todo
Storage: t.sc.Storage(t.taskToSector, storiface.FTSealed, storiface.FTCache, ssize, storiface.PathSealing),
},
MaxFailures: 3,
Follows: nil,
}
}
func (t *TreesTask) Adder(taskFunc harmonytask.AddTaskFunc) {
t.sp.pollers[pollerTrees].Set(taskFunc)
}
func (t *TreesTask) taskToSector(id harmonytask.TaskID) (ffi.SectorRef, error) {
var refs []ffi.SectorRef
err := t.db.Select(context.Background(), &refs, `SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_tree_r = $1`, id)
if err != nil {
return ffi.SectorRef{}, xerrors.Errorf("getting sector ref: %w", err)
}
if len(refs) != 1 {
return ffi.SectorRef{}, xerrors.Errorf("expected 1 sector ref, got %d", len(refs))
}
return refs[0], nil
}
type UrlPieceReader struct {
Url string
RawSize int64 // the exact number of bytes read, if we read more or less that's an error
@ -323,4 +327,4 @@ func (u *UrlPieceReader) Close() error {
return nil
}
var _ harmonytask.TaskInterface = &TreesTask{}
var _ harmonytask.TaskInterface = &TreeDTask{}

0
curiosrc/seal/task_trees_test.go → curiosrc/seal/task_treed_test.go
View File

190
curiosrc/seal/task_treerc.go Normal file
View File

@ -0,0 +1,190 @@
package seal
import (
"context"
"github.com/ipfs/go-cid"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/curiosrc/ffi"
"github.com/filecoin-project/lotus/lib/harmony/harmonydb"
"github.com/filecoin-project/lotus/lib/harmony/harmonytask"
"github.com/filecoin-project/lotus/lib/harmony/resources"
"github.com/filecoin-project/lotus/storage/paths"
"github.com/filecoin-project/lotus/storage/sealer/storiface"
)
type TreeRCTask struct {
sp *SealPoller
db *harmonydb.DB
sc *ffi.SealCalls
max int
}
func NewTreeRCTask(sp *SealPoller, db *harmonydb.DB, sc *ffi.SealCalls, maxTrees int) *TreeRCTask {
return &TreeRCTask{
sp: sp,
db: db,
sc: sc,
max: maxTrees,
}
}
func (t *TreeRCTask) Do(taskID harmonytask.TaskID, stillOwned func() bool) (done bool, err error) {
ctx := context.Background()
var sectorParamsArr []struct {
SpID int64 `db:"sp_id"`
SectorNumber int64 `db:"sector_number"`
RegSealProof abi.RegisteredSealProof `db:"reg_seal_proof"`
CommD string `db:"tree_d_cid"`
}
err = t.db.Select(ctx, &sectorParamsArr, `
SELECT sp_id, sector_number, reg_seal_proof, tree_d_cid
FROM sectors_sdr_pipeline
WHERE task_id_tree_c = $1 AND task_id_tree_r = $1`, taskID)
if err != nil {
return false, xerrors.Errorf("getting sector params: %w", err)
}
if len(sectorParamsArr) != 1 {
return false, xerrors.Errorf("expected 1 sector params, got %d", len(sectorParamsArr))
}
sectorParams := sectorParamsArr[0]
commd, err := cid.Parse(sectorParams.CommD)
if err != nil {
return false, xerrors.Errorf("parsing unsealed CID: %w", err)
}
sref := storiface.SectorRef{
ID: abi.SectorID{
Miner: abi.ActorID(sectorParams.SpID),
Number: abi.SectorNumber(sectorParams.SectorNumber),
},
ProofType: sectorParams.RegSealProof,
}
// R / C
sealed, _, err := t.sc.TreeRC(ctx, &taskID, sref, commd)
if err != nil {
return false, xerrors.Errorf("computing tree r and c: %w", err)
}
// todo synth porep
// todo porep challenge check
n, err := t.db.Exec(ctx, `UPDATE sectors_sdr_pipeline
SET after_tree_r = true, after_tree_c = true, tree_r_cid = $3
WHERE sp_id = $1 AND sector_number = $2`,
sectorParams.SpID, sectorParams.SectorNumber, sealed)
if err != nil {
return false, xerrors.Errorf("store sdr-trees success: updating pipeline: %w", err)
}
if n != 1 {
return false, xerrors.Errorf("store sdr-trees success: updated %d rows", n)
}
return true, nil
}
func (t *TreeRCTask) CanAccept(ids []harmonytask.TaskID, engine *harmonytask.TaskEngine) (*harmonytask.TaskID, error) {
var tasks []struct {
TaskID harmonytask.TaskID `db:"task_id_tree_c"`
SpID int64 `db:"sp_id"`
SectorNumber int64 `db:"sector_number"`
StorageID string `db:"storage_id"`
}
if storiface.FTCache != 4 {
panic("storiface.FTCache != 4")
}
ctx := context.Background()
indIDs := make([]int64, len(ids))
for i, id := range ids {
indIDs[i] = int64(id)
}
err := t.db.Select(ctx, &tasks, `
SELECT p.task_id_tree_c, p.sp_id, p.sector_number, l.storage_id FROM sectors_sdr_pipeline p
INNER JOIN sector_location l ON p.sp_id = l.miner_id AND p.sector_number = l.sector_num
WHERE task_id_tree_r = ANY ($1) AND l.sector_filetype = 4
`, indIDs)
if err != nil {
return nil, xerrors.Errorf("getting tasks: %w", err)
}
ls, err := t.sc.LocalStorage(ctx)
if err != nil {
return nil, xerrors.Errorf("getting local storage: %w", err)
}
acceptables := map[harmonytask.TaskID]bool{}
for _, t := range ids {
acceptables[t] = true
}
for _, t := range tasks {
if _, ok := acceptables[t.TaskID]; !ok {
continue
}
for _, l := range ls {
if string(l.ID) == t.StorageID {
return &t.TaskID, nil
}
}
}
return nil, nil
}
func (t *TreeRCTask) TypeDetails() harmonytask.TaskTypeDetails {
ssize := abi.SectorSize(32 << 30) // todo task details needs taskID to get correct sector size
if isDevnet {
ssize = abi.SectorSize(2 << 20)
}
return harmonytask.TaskTypeDetails{
Max: t.max,
Name: "TreeRC",
Cost: resources.Resources{
Cpu: 1,
Gpu: 1,
Ram: 8 << 30,
Storage: t.sc.Storage(t.taskToSector, storiface.FTSealed, storiface.FTCache, ssize, storiface.PathSealing, paths.MinFreeStoragePercentage),
},
MaxFailures: 3,
Follows: nil,
}
}
func (t *TreeRCTask) Adder(taskFunc harmonytask.AddTaskFunc) {
t.sp.pollers[pollerTreeRC].Set(taskFunc)
}
func (t *TreeRCTask) taskToSector(id harmonytask.TaskID) (ffi.SectorRef, error) {
var refs []ffi.SectorRef
err := t.db.Select(context.Background(), &refs, `SELECT sp_id, sector_number, reg_seal_proof FROM sectors_sdr_pipeline WHERE task_id_tree_r = $1`, id)
if err != nil {
return ffi.SectorRef{}, xerrors.Errorf("getting sector ref: %w", err)
}
if len(refs) != 1 {
return ffi.SectorRef{}, xerrors.Errorf("expected 1 sector ref, got %d", len(refs))
}
return refs[0], nil
}
var _ harmonytask.TaskInterface = &TreeRCTask{}

5
lib/harmony/harmonytask/harmonytask.go
View File

@ -385,3 +385,8 @@ func (e *TaskEngine) ResourcesAvailable() resources.Resources {
}
return tmp
}
// Resources returns the resources available in the TaskEngine's registry.
func (e *TaskEngine) Resources() resources.Resources {
return e.reg.Resources
}

6
storage/paths/db_index.go
View File

@ -226,8 +226,8 @@ func (dbi *DBIndex) StorageAttach(ctx context.Context, si storiface.StorageInfo,
// Insert storage id
_, err = tx.Exec(
"INSERT INTO storage_path "+
"Values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW(), $16, $17)",
"INSERT INTO storage_path (storage_id, urls, weight, max_storage, can_seal, can_store, groups, allow_to, allow_types, deny_types, capacity, available, fs_available, reserved, used, last_heartbeat, heartbeat_err, allow_miners, deny_miners)"+
"Values($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, NOW(), NULL, $16, $17)",
si.ID,
strings.Join(si.URLs, ","),
si.Weight,
@ -406,7 +406,7 @@ func (dbi *DBIndex) StorageDeclareSector(ctx context.Context, storageID storifac
}
} else {
_, err = tx.Exec(
"INSERT INTO sector_location "+
"INSERT INTO sector_location (miner_id, sector_num, sector_filetype, storage_id, is_primary)"+
"values($1, $2, $3, $4, $5)",
s.Miner, s.Number, ft, storageID, primary)
if err != nil {

2
storage/paths/interface.go
View File

@ -47,7 +47,7 @@ type Store interface {
FsStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error)
Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error)
Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int, minFreePercentage float64) (func(), error)
GenerateSingleVanillaProof(ctx context.Context, minerID abi.ActorID, si storiface.PostSectorChallenge, ppt abi.RegisteredPoStProof) ([]byte, error)
GeneratePoRepVanillaProof(ctx context.Context, sr storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error)

20
storage/paths/local.go
View File

@ -36,6 +36,8 @@ type LocalStorage interface {
const MetaFile = "sectorstore.json"
const MinFreeStoragePercentage = float64(0)
type Local struct {
localStorage LocalStorage
index SectorIndex
@ -460,13 +462,13 @@ func (st *Local) reportStorage(ctx context.Context) {
}
}
func (st *Local) Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (release func(), err error) {
func (st *Local) Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int, minFreePercentage float64) (userRelease func(), err error) {
var ssize abi.SectorSize
ssize, err = sid.ProofType.SectorSize()
if err != nil {
return nil, err
}
release = func() {}
release := func() {}
st.localLk.Lock()
@ -501,10 +503,18 @@ func (st *Local) Reserve(ctx context.Context, sid storiface.SectorRef, ft storif
resvOnDisk = overhead
}
if stat.Available < overhead-resvOnDisk {
overheadOnDisk := overhead - resvOnDisk
if stat.Available < overheadOnDisk {
return nil, storiface.Err(storiface.ErrTempAllocateSpace, xerrors.Errorf("can't reserve %d bytes in '%s' (id:%s), only %d available", overhead, p.local, id, stat.Available))
}
freePercentag := (float64(stat.Available-overheadOnDisk) / float64(stat.Available)) * 100.0
if freePercentag < minFreePercentage {
return nil, storiface.Err(storiface.ErrTempAllocateSpace, xerrors.Errorf("can't reserve %d bytes in '%s' (id:%s), free disk percentage %f will be lower than minimum %f", overhead, p.local, id, freePercentag, minFreePercentage))
}
resID := sectorFile{sid.ID, fileType}
log.Debugw("reserve add", "id", id, "sector", sid, "fileType", fileType, "overhead", overhead, "reserved-before", p.reserved, "reserved-after", p.reserved+overhead)
@ -523,7 +533,7 @@ func (st *Local) Reserve(ctx context.Context, sid storiface.SectorRef, ft storif
}
}
return
return release, nil
}
// DoubleCallWrap wraps a function to make sure it's not called twice
@ -533,7 +543,7 @@ func DoubleCallWrap(f func()) func() {
curStack := make([]byte, 20480)
curStack = curStack[:runtime.Stack(curStack, false)]
if len(stack) > 0 {
log.Warnf("double call from:\n%s\nBut originally from:", curStack, stack)
log.Warnf("double call from:\n%s\nBut originally from:\n%s", curStack, stack)
return
}
stack = curStack

8
storage/paths/mocks/store.go
View File

@ -154,16 +154,16 @@ func (mr *MockStoreMockRecorder) RemoveCopies(arg0, arg1, arg2 interface{}) *gom
}
// Reserve mocks base method.
func (m *MockStore) Reserve(arg0 context.Context, arg1 storiface.SectorRef, arg2 storiface.SectorFileType, arg3 storiface.SectorPaths, arg4 map[storiface.SectorFileType]int) (func(), error) {
func (m *MockStore) Reserve(arg0 context.Context, arg1 storiface.SectorRef, arg2 storiface.SectorFileType, arg3 storiface.SectorPaths, arg4 map[storiface.SectorFileType]int, arg5 float64) (func(), error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Reserve", arg0, arg1, arg2, arg3, arg4)
ret := m.ctrl.Call(m, "Reserve", arg0, arg1, arg2, arg3, arg4, arg5)
ret0, _ := ret[0].(func())
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Reserve indicates an expected call of Reserve.
func (mr *MockStoreMockRecorder) Reserve(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call {
func (mr *MockStoreMockRecorder) Reserve(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reserve", reflect.TypeOf((*MockStore)(nil).Reserve), arg0, arg1, arg2, arg3, arg4)
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reserve", reflect.TypeOf((*MockStore)(nil).Reserve), arg0, arg1, arg2, arg3, arg4, arg5)
}

4
storage/paths/remote.go
View File

@ -177,7 +177,7 @@ func (r *Remote) AcquireSector(ctx context.Context, s storiface.SectorRef, exist
// If any path types weren't found in local storage, try fetching them
// First reserve storage
releaseStorage, err := r.local.Reserve(ctx, s, toFetch, fetchIDs, overheadTable)
releaseStorage, err := r.local.Reserve(ctx, s, toFetch, fetchIDs, overheadTable, MinFreeStoragePercentage)
if err != nil {
return storiface.SectorPaths{}, storiface.SectorPaths{}, xerrors.Errorf("reserving storage space: %w", err)
}
@ -812,7 +812,7 @@ func (r *Remote) ReaderSeq(ctx context.Context, s storiface.SectorRef, ft storif
return nil, xerrors.Errorf("failed to read sector %v from remote(%d): %w", s, ft, storiface.ErrSectorNotFound)
}
func (r *Remote) Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error) {
func (r *Remote) Reserve(ctx context.Context, sid storiface.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int, minFreePercentage float64) (func(), error) {
log.Warnf("reserve called on remote store, sectorID: %v", sid.ID)
return func() {

8
storage/sealer/proofpaths/cachefiles.go
View File

@ -53,3 +53,11 @@ func SDRLayers(spt abi.RegisteredSealProof) (int, error) {
return 0, fmt.Errorf("unsupported proof type: %v", spt)
}
}
func IsTreeRCFile(baseName string) bool {
return IsFileTreeRLast(baseName) || IsFileTreeC(baseName)
}
func IsTreeDFile(baseName string) bool {
return IsFileTreeD(baseName)
}

8
storage/sealer/worker_local.go
View File

@ -150,19 +150,19 @@ type localWorkerPathProvider struct {
}
func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, sector storiface.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, sealing storiface.PathType) (storiface.SectorPaths, func(), error) {
paths, storageIDs, err := l.w.storage.AcquireSector(ctx, sector, existing, allocate, sealing, l.op)
spaths, storageIDs, err := l.w.storage.AcquireSector(ctx, sector, existing, allocate, sealing, l.op)
if err != nil {
return storiface.SectorPaths{}, nil, err
}
releaseStorage, err := l.w.localStore.Reserve(ctx, sector, allocate, storageIDs, storiface.FSOverheadSeal)
releaseStorage, err := l.w.localStore.Reserve(ctx, sector, allocate, storageIDs, storiface.FSOverheadSeal, paths.MinFreeStoragePercentage)
if err != nil {
return storiface.SectorPaths{}, nil, xerrors.Errorf("reserving storage space: %w", err)
}
log.Debugf("acquired sector %d (e:%d; a:%d): %v", sector, existing, allocate, paths)
log.Debugf("acquired sector %d (e:%d; a:%d): %v", sector, existing, allocate, spaths)
return paths, func() {
return spaths, func() {
releaseStorage()
for _, fileType := range storiface.PathTypes {