diff --git a/api/api_storage.go b/api/api_storage.go index bf7520d09..92117d2fb 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -127,6 +127,7 @@ type StorageMiner interface { ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storage.ReplicaUpdateOut, err *storiface.CallError) error //perm:admin retry:true ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, vanillaProofs storage.ReplicaVanillaProofs, err *storiface.CallError) error //perm:admin retry:true ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storage.ReplicaUpdateProof, err *storiface.CallError) error //perm:admin retry:true + ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error //perm:admin retry:true diff --git a/api/api_worker.go b/api/api_worker.go index 5e0b4f8c6..68d8e7baf 100644 --- a/api/api_worker.go +++ b/api/api_worker.go @@ -42,6 +42,7 @@ type Worker interface { ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error) //perm:admin ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (storiface.CallID, error) //perm:admin ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (storiface.CallID, error) //perm:admin + GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) (storiface.CallID, error) //perm:admin ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error) //perm:admin MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error) //perm:admin UnsealPiece(context.Context, storage.SectorRef, storiface.UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (storiface.CallID, error) //perm:admin diff --git a/api/proxy_gen.go b/api/proxy_gen.go index b78b40959..5733c3cc8 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -707,6 +707,8 @@ type StorageMinerStruct struct { ReturnFinalizeSector func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"` + ReturnGenerateSectorKeyFromData func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"` + ReturnMoveStorage func(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error `perm:"admin"` ReturnProveReplicaUpdate1 func(p0 context.Context, p1 storiface.CallID, p2 storage.ReplicaVanillaProofs, p3 *storiface.CallError) error `perm:"admin"` @@ -850,6 +852,8 @@ type WorkerStruct struct { FinalizeSector func(p0 context.Context, p1 storage.SectorRef, p2 []storage.Range) (storiface.CallID, error) `perm:"admin"` + GenerateSectorKeyFromData func(p0 context.Context, p1 storage.SectorRef, p2 cid.Cid) (storiface.CallID, error) `perm:"admin"` + Info func(p0 context.Context) (storiface.WorkerInfo, error) `perm:"admin"` MoveStorage func(p0 context.Context, p1 storage.SectorRef, p2 storiface.SectorFileType) (storiface.CallID, error) `perm:"admin"` @@ -4166,6 +4170,17 @@ func (s *StorageMinerStub) ReturnFinalizeSector(p0 context.Context, p1 storiface return ErrNotSupported } +func (s *StorageMinerStruct) ReturnGenerateSectorKeyFromData(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error { + if s.Internal.ReturnGenerateSectorKeyFromData == nil { + return ErrNotSupported + } + return s.Internal.ReturnGenerateSectorKeyFromData(p0, p1, p2) +} + +func (s *StorageMinerStub) ReturnGenerateSectorKeyFromData(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error { + return ErrNotSupported +} + func (s *StorageMinerStruct) ReturnMoveStorage(p0 context.Context, p1 storiface.CallID, p2 *storiface.CallError) error { if s.Internal.ReturnMoveStorage == nil { return ErrNotSupported @@ -4859,6 +4874,17 @@ func (s *WorkerStub) FinalizeSector(p0 context.Context, p1 storage.SectorRef, p2 return *new(storiface.CallID), ErrNotSupported } +func (s *WorkerStruct) GenerateSectorKeyFromData(p0 context.Context, p1 storage.SectorRef, p2 cid.Cid) (storiface.CallID, error) { + if s.Internal.GenerateSectorKeyFromData == nil { + return *new(storiface.CallID), ErrNotSupported + } + return s.Internal.GenerateSectorKeyFromData(p0, p1, p2) +} + +func (s *WorkerStub) GenerateSectorKeyFromData(p0 context.Context, p1 storage.SectorRef, p2 cid.Cid) (storiface.CallID, error) { + return *new(storiface.CallID), ErrNotSupported +} + func (s *WorkerStruct) Info(p0 context.Context) (storiface.WorkerInfo, error) { if s.Internal.Info == nil { return *new(storiface.WorkerInfo), ErrNotSupported diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index a5816ef4f..839e95e61 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 0c176bbef..f0eaa49ec 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 5209142d0..85161bcd1 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index c037a0af6..a498cddff 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -94,6 +94,7 @@ * [ReturnAddPiece](#ReturnAddPiece) * [ReturnFetch](#ReturnFetch) * [ReturnFinalizeSector](#ReturnFinalizeSector) + * [ReturnGenerateSectorKeyFromData](#ReturnGenerateSectorKeyFromData) * [ReturnMoveStorage](#ReturnMoveStorage) * [ReturnProveReplicaUpdate1](#ReturnProveReplicaUpdate1) * [ReturnProveReplicaUpdate2](#ReturnProveReplicaUpdate2) @@ -1443,6 +1444,30 @@ Response: `{}` ### ReturnFinalizeSector +Perms: admin + +Inputs: +```json +[ + { + "Sector": { + "Miner": 1000, + "Number": 9 + }, + "ID": "07070707-0707-0707-0707-070707070707" + }, + { + "Code": 0, + "Message": "string value" + } +] +``` + +Response: `{}` + +### ReturnGenerateSectorKeyFromData + + Perms: admin Inputs: diff --git a/documentation/en/api-v0-methods-worker.md b/documentation/en/api-v0-methods-worker.md index 7aee91d52..5d4b33a52 100644 --- a/documentation/en/api-v0-methods-worker.md +++ b/documentation/en/api-v0-methods-worker.md @@ -11,6 +11,8 @@ * [AddPiece](#AddPiece) * [Finalize](#Finalize) * [FinalizeSector](#FinalizeSector) +* [Generate](#Generate) + * [GenerateSectorKeyFromData](#GenerateSectorKeyFromData) * [Move](#Move) * [MoveStorage](#MoveStorage) * [Process](#Process) @@ -220,6 +222,41 @@ Response: } ``` +## Generate + + +### GenerateSectorKeyFromData + + +Perms: admin + +Inputs: +```json +[ + { + "ID": { + "Miner": 1000, + "Number": 9 + }, + "ProofType": 8 + }, + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Sector": { + "Miner": 1000, + "Number": 9 + }, + "ID": "07070707-0707-0707-0707-070707070707" +} +``` + ## Move diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index ce7083b3d..428503c87 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit ce7083b3d187ec3bc41a68ab66567bd4f3be6dfc +Subproject commit 428503c87d917cc5e3e637983b43b4c260863bf0 diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index 81dbfb426..65c6762fa 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -251,6 +251,23 @@ func (sb *Sealer) pieceCid(spt abi.RegisteredSealProof, in []byte) (cid.Cid, err return pieceCID, werr() } +func (sb *Sealer) tryDecodeUpdatedReplica(ctx context.Context, sector storage.SectorRef, commD cid.Cid, unsealedPath string) (bool, error) { + paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUpdate|storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage) + if xerrors.Is(err, storiface.ErrSectorNotFound) { + return false, nil + } else if err != nil { + return false, xerrors.Errorf("reading updated replica: %w", err) + } + defer done() + + // Sector data stored in replica update + updateProof, err := sector.ProofType.RegisteredUpdateProof() + if err != nil { + return false, err + } + return true, ffi.SectorUpdate.DecodeFrom(updateProof, unsealedPath, paths.Update, paths.Sealed, paths.Cache, commD) +} + func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd cid.Cid) error { ssize, err := sector.ProofType.SectorSize() if err != nil { @@ -301,6 +318,16 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off return nil } + // If piece data stored in updated replica decode whole sector + decoded, err := sb.tryDecodeUpdatedReplica(ctx, sector, commd, unsealedPath.Unsealed) + if err != nil { + return xerrors.Errorf("decoding sector from replica: %w", err) + } + if decoded { + return pf.MarkAllocated(0, maxPieceSize) + } + + // Piece data sealed in sector srcPaths, srcDone, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTCache|storiface.FTSealed, storiface.FTNone, storiface.PathStorage) if err != nil { return xerrors.Errorf("acquire sealed sector paths: %w", err) @@ -626,12 +653,6 @@ func (sb *Sealer) ReplicaUpdate(ctx context.Context, sector storage.SectorRef, p return empty, err } } - - // XXX: we want to keep the stuff at the end - if err := os.Truncate(paths.Unsealed, sealedSize); err != nil { - return empty, xerrors.Errorf("failed to truncate unsealed data file: %w", err) - } - sealed, unsealed, err := ffi.SectorUpdate.EncodeInto(updateProofType, paths.Update, paths.UpdateCache, paths.Sealed, paths.Cache, paths.Unsealed, pieces) if err != nil { return empty, xerrors.Errorf("failed to update replica %d with new deal data: %w", sector.ID.Number, err) @@ -661,6 +682,33 @@ func (sb *Sealer) ProveReplicaUpdate2(ctx context.Context, sector storage.Sector return ffi.SectorUpdate.GenerateUpdateProofWithVanilla(updateProofType, sectorKey, newSealed, newUnsealed, vanillaProofs) } +func (sb *Sealer) GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) error { + paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed|storiface.FTCache|storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTSealed, storiface.PathSealing) + defer done() + if err != nil { + return xerrors.Errorf("failed to acquire sector paths: %w", err) + } + + s, err := os.Stat(paths.Update) + if err != nil { + return xerrors.Errorf("measuring update file size: %w", err) + } + sealedSize := s.Size() + e, err := os.OpenFile(paths.Sealed, os.O_RDWR|os.O_CREATE, 0644) // nolint:gosec + if err != nil { + return xerrors.Errorf("ensuring sector key file exists: %w", err) + } + if err := fallocate.Fallocate(e, 0, sealedSize); err != nil { + return xerrors.Errorf("allocating space for sector key file: %w", err) + } + if err := e.Close(); err != nil { + return err + } + + updateProofType := abi.SealProofInfos[sector.ProofType].UpdateProof + return ffi.SectorUpdate.RemoveData(updateProofType, paths.Sealed, paths.Cache, paths.Update, paths.UpdateCache, paths.Unsealed, commD) +} + func (sb *Sealer) ReleaseSealed(ctx context.Context, sector storage.SectorRef) error { return xerrors.Errorf("not supported at this layer") } diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index e0abf1773..3deff4d6b 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -578,15 +578,72 @@ func (m *Manager) ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, return nil } -func (m *Manager) ReleaseSealed(ctx context.Context, sector storage.SectorRef) error { - return nil +func (m *Manager) ReleaseSectorKey(ctx context.Context, sector storage.SectorRef) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := m.index.StorageLock(ctx, sector.ID, storiface.FTNone, storiface.FTSealed); err != nil { + return xerrors.Errorf("acquiring sector lock: %w", err) + } + + return m.storage.Remove(ctx, sector.ID, storiface.FTSealed, true, nil) +} + +func (m *Manager) GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) error { + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + wk, wait, cancel, err := m.getWork(ctx, sealtasks.TTRegenSectorKey, sector, commD) + if err != nil { + return xerrors.Errorf("getWork: %w", err) + } + defer cancel() + + var waitErr error + waitRes := func() { + _, werr := m.waitWork(ctx, wk) + if werr != nil { + waitErr = werr + return + } + } + + if wait { // already in progress + waitRes() + return waitErr + } + + if err := m.index.StorageLock(ctx, sector.ID, storiface.FTUnsealed|storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTSealed|storiface.FTCache); err != nil { + return xerrors.Errorf("acquiring sector lock: %w", err) + } + + // NOTE: We set allowFetch to false in so that we always execute on a worker + // with direct access to the data. We want to do that because this step is + // generally very cheap / fast, and transferring data is not worth the effort + selector := newExistingSelector(m.index, sector.ID, storiface.FTUnsealed|storiface.FTUpdate|storiface.FTUpdateCache|storiface.FTCache, true) + + err = m.sched.Schedule(ctx, sector, sealtasks.TTRegenSectorKey, selector, m.schedFetch(sector, storiface.FTUpdate|storiface.FTUnsealed, storiface.PathSealing, storiface.AcquireMove), func(ctx context.Context, w Worker) error { + err := m.startWork(ctx, w, wk)(w.GenerateSectorKeyFromData(ctx, sector, commD)) + if err != nil { + return err + } + + waitRes() + return nil + }) + if err != nil { + return err + } + + return waitErr } func (m *Manager) Remove(ctx context.Context, sector storage.SectorRef) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - if err := m.index.StorageLock(ctx, sector.ID, storiface.FTNone, storiface.FTSealed|storiface.FTUnsealed|storiface.FTCache); err != nil { + if err := m.index.StorageLock(ctx, sector.ID, storiface.FTNone, storiface.FTSealed|storiface.FTUnsealed|storiface.FTCache|storiface.FTUpdate|storiface.FTUpdateCache); err != nil { return xerrors.Errorf("acquiring sector lock: %w", err) } @@ -601,6 +658,12 @@ func (m *Manager) Remove(ctx context.Context, sector storage.SectorRef) error { if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTUnsealed, true, nil); rerr != nil { err = multierror.Append(err, xerrors.Errorf("removing sector (unsealed): %w", rerr)) } + if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTUpdate, true, nil); rerr != nil { + err = multierror.Append(err, xerrors.Errorf("removing sector (unsealed): %w", rerr)) + } + if rerr := m.storage.Remove(ctx, sector.ID, storiface.FTUpdateCache, true, nil); rerr != nil { + err = multierror.Append(err, xerrors.Errorf("removing sector (unsealed): %w", rerr)) + } return err } @@ -790,6 +853,10 @@ func (m *Manager) ReturnProveReplicaUpdate2(ctx context.Context, callID storifac return m.returnResult(ctx, callID, proof, err) } +func (m *Manager) ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error { + return m.returnResult(ctx, callID, nil, err) +} + func (m *Manager) ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error { return m.returnResult(ctx, callID, nil, err) } diff --git a/extern/sector-storage/manager_test.go b/extern/sector-storage/manager_test.go index 77e185b93..cf54ccbf2 100644 --- a/extern/sector-storage/manager_test.go +++ b/extern/sector-storage/manager_test.go @@ -199,7 +199,8 @@ func TestSnapDeals(t *testing.T) { localTasks := []sealtasks.TaskType{ sealtasks.TTAddPiece, sealtasks.TTPreCommit1, sealtasks.TTPreCommit2, sealtasks.TTCommit1, sealtasks.TTCommit2, sealtasks.TTFinalize, - sealtasks.TTFetch, sealtasks.TTReplicaUpdate, sealtasks.TTProveReplicaUpdate1, sealtasks.TTProveReplicaUpdate2, + sealtasks.TTFetch, sealtasks.TTReplicaUpdate, sealtasks.TTProveReplicaUpdate1, sealtasks.TTProveReplicaUpdate2, sealtasks.TTUnseal, + sealtasks.TTRegenSectorKey, } wds := datastore.NewMapDatastore() @@ -245,14 +246,6 @@ func TestSnapDeals(t *testing.T) { require.NoError(t, err) fmt.Printf("PC2\n") pc2Out, err := m.SealPreCommit2(ctx, sid, pc1Out) - - require.NoError(t, err) - seed := abi.InteractiveSealRandomness{1, 1, 1, 1, 1, 1, 1} - fmt.Printf("C1\n") - c1Out, err := m.SealCommit1(ctx, sid, ticket, seed, nil, pc2Out) - require.NoError(t, err) - fmt.Printf("C2\n") - _, err = m.SealCommit2(ctx, sid, c1Out) require.NoError(t, err) // Now do a snap deals replica update @@ -270,19 +263,26 @@ func TestSnapDeals(t *testing.T) { pieces := []abi.PieceInfo{p1, p2} fmt.Printf("RU\n") + startRU := time.Now() out, err := m.ReplicaUpdate(ctx, sid, pieces) require.NoError(t, err) + fmt.Printf("RU duration (%s): %s\n", ss.ShortString(), time.Since(startRU)) + updateProofType, err := sid.ProofType.RegisteredUpdateProof() require.NoError(t, err) require.NotNil(t, out) fmt.Printf("PR1\n") + startPR1 := time.Now() vanillaProofs, err := m.ProveReplicaUpdate1(ctx, sid, sectorKey, out.NewSealed, out.NewUnsealed) require.NoError(t, err) require.NotNil(t, vanillaProofs) + fmt.Printf("PR1 duration (%s): %s\n", ss.ShortString(), time.Since(startPR1)) fmt.Printf("PR2\n") + startPR2 := time.Now() proof, err := m.ProveReplicaUpdate2(ctx, sid, sectorKey, out.NewSealed, out.NewUnsealed, vanillaProofs) require.NoError(t, err) require.NotNil(t, proof) + fmt.Printf("PR2 duration (%s): %s\n", ss.ShortString(), time.Since(startPR2)) vInfo := proof7.ReplicaUpdateInfo{ Proof: proof, @@ -294,6 +294,24 @@ func TestSnapDeals(t *testing.T) { pass, err := ffiwrapper.ProofVerifier.VerifyReplicaUpdate(vInfo) require.NoError(t, err) assert.True(t, pass) + + fmt.Printf("Decode\n") + // Remove unsealed data and decode for retrieval + require.NoError(t, m.FinalizeSector(ctx, sid, nil)) + startDecode := time.Now() + require.NoError(t, m.SectorsUnsealPiece(ctx, sid, 0, p1.Size.Unpadded(), ticket, &out.NewUnsealed)) + fmt.Printf("Decode duration (%s): %s\n", ss.ShortString(), time.Since(startDecode)) + + // Remove just the first piece and decode for retrieval + require.NoError(t, m.FinalizeSector(ctx, sid, []storage.Range{{Offset: p1.Size.Unpadded(), Size: p2.Size.Unpadded()}})) + require.NoError(t, m.SectorsUnsealPiece(ctx, sid, 0, p1.Size.Unpadded(), ticket, &out.NewUnsealed)) + + fmt.Printf("GSK\n") + require.NoError(t, m.ReleaseSectorKey(ctx, sid)) + startGSK := time.Now() + require.NoError(t, m.GenerateSectorKeyFromData(ctx, sid, out.NewUnsealed)) + fmt.Printf("GSK duration (%s): %s\n", ss.ShortString(), time.Since(startGSK)) + } func TestRedoPC1(t *testing.T) { diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 023904984..95b11c004 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -277,6 +277,10 @@ func (mgr *SectorMgr) ProveReplicaUpdate2(ctx context.Context, sector storage.Se return make([]byte, 0), nil } +func (mgr *SectorMgr) GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) error { + return nil +} + func (mgr *SectorMgr) ReleaseSealed(ctx context.Context, sid storage.SectorRef) error { return nil } @@ -534,6 +538,10 @@ func (mgr *SectorMgr) ReturnProveReplicaUpdate2(ctx context.Context, callID stor panic("not supported") } +func (mgr *SectorMgr) ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error { + panic("not supported") +} + func (m mockVerifProver) VerifySeal(svi proof5.SealVerifyInfo) (bool, error) { plen, err := svi.SealProof.ProofSize() if err != nil { diff --git a/extern/sector-storage/sched_test.go b/extern/sector-storage/sched_test.go index fd3c90295..a41e7c855 100644 --- a/extern/sector-storage/sched_test.go +++ b/extern/sector-storage/sched_test.go @@ -112,6 +112,10 @@ func (s *schedTestWorker) ProveReplicaUpdate2(ctx context.Context, sector storag panic("implement me") } +func (s *schedTestWorker) GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) (storiface.CallID, error) { + panic("implement me") +} + func (s *schedTestWorker) MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error) { panic("implement me") } diff --git a/extern/sector-storage/sealtasks/task.go b/extern/sector-storage/sealtasks/task.go index 3f2a9701f..f6104878b 100644 --- a/extern/sector-storage/sealtasks/task.go +++ b/extern/sector-storage/sealtasks/task.go @@ -17,10 +17,12 @@ const ( TTReplicaUpdate TaskType = "seal/v0/replicaupdate" TTProveReplicaUpdate1 TaskType = "seal/v0/provereplicaupdate/1" TTProveReplicaUpdate2 TaskType = "seal/v0/provereplicaupdate/2" + TTRegenSectorKey TaskType = "seal/v0/regensectorkey" ) var order = map[TaskType]int{ - TTAddPiece: 9, // least priority + TTRegenSectorKey: 10, // least priority + TTAddPiece: 9, TTReplicaUpdate: 8, TTProveReplicaUpdate2: 7, TTProveReplicaUpdate1: 6, @@ -49,6 +51,7 @@ var shortNames = map[TaskType]string{ TTReplicaUpdate: "RU", TTProveReplicaUpdate1: "PR1", TTProveReplicaUpdate2: "PR2", + TTRegenSectorKey: "GSK", } func (a TaskType) MuchLess(b TaskType) (bool, bool) { diff --git a/extern/sector-storage/storiface/worker.go b/extern/sector-storage/storiface/worker.go index 970e0ec33..ab8c484e6 100644 --- a/extern/sector-storage/storiface/worker.go +++ b/extern/sector-storage/storiface/worker.go @@ -95,6 +95,7 @@ type WorkerCalls interface { ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (CallID, error) ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (CallID, error) ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (CallID, error) + GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) (CallID, error) MoveStorage(ctx context.Context, sector storage.SectorRef, types SectorFileType) (CallID, error) UnsealPiece(context.Context, storage.SectorRef, UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (CallID, error) Fetch(context.Context, storage.SectorRef, SectorFileType, PathType, AcquireMode) (CallID, error) @@ -151,6 +152,7 @@ type WorkerReturn interface { ReturnReplicaUpdate(ctx context.Context, callID CallID, out storage.ReplicaUpdateOut, err *CallError) error ReturnProveReplicaUpdate1(ctx context.Context, callID CallID, proofs storage.ReplicaVanillaProofs, err *CallError) error ReturnProveReplicaUpdate2(ctx context.Context, callID CallID, proof storage.ReplicaUpdateProof, err *CallError) error + ReturnGenerateSectorKeyFromData(ctx context.Context, callID CallID, err *CallError) error ReturnMoveStorage(ctx context.Context, callID CallID, err *CallError) error ReturnUnsealPiece(ctx context.Context, callID CallID, err *CallError) error ReturnReadPiece(ctx context.Context, callID CallID, ok bool, err *CallError) error diff --git a/extern/sector-storage/teststorage_test.go b/extern/sector-storage/teststorage_test.go index 4061b48d9..9fdb3a913 100644 --- a/extern/sector-storage/teststorage_test.go +++ b/extern/sector-storage/teststorage_test.go @@ -75,6 +75,10 @@ func (t *testExec) ProveReplicaUpdate2(ctx context.Context, sector storage.Secto panic("implement me") } +func (t *testExec) GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) error { + panic("implement me") +} + func (t *testExec) NewSector(ctx context.Context, sector storage.SectorRef) error { panic("implement me") } diff --git a/extern/sector-storage/worker_local.go b/extern/sector-storage/worker_local.go index cc41e916e..387a47f8f 100644 --- a/extern/sector-storage/worker_local.go +++ b/extern/sector-storage/worker_local.go @@ -168,6 +168,7 @@ const ( ReplicaUpdate ReturnType = "ReplicaUpdate" ProveReplicaUpdate1 ReturnType = "ProveReplicaUpdate1" ProveReplicaUpdate2 ReturnType = "ProveReplicaUpdate2" + GenerateSectorKey ReturnType = "GenerateSectorKey" ReleaseUnsealed ReturnType = "ReleaseUnsealed" MoveStorage ReturnType = "MoveStorage" UnsealPiece ReturnType = "UnsealPiece" @@ -219,6 +220,7 @@ var returnFunc = map[ReturnType]func(context.Context, storiface.CallID, storifac ReplicaUpdate: rfunc(storiface.WorkerReturn.ReturnReplicaUpdate), ProveReplicaUpdate1: rfunc(storiface.WorkerReturn.ReturnProveReplicaUpdate1), ProveReplicaUpdate2: rfunc(storiface.WorkerReturn.ReturnProveReplicaUpdate2), + GenerateSectorKey: rfunc(storiface.WorkerReturn.ReturnGenerateSectorKeyFromData), MoveStorage: rfunc(storiface.WorkerReturn.ReturnMoveStorage), UnsealPiece: rfunc(storiface.WorkerReturn.ReturnUnsealPiece), Fetch: rfunc(storiface.WorkerReturn.ReturnFetch), @@ -419,6 +421,17 @@ func (l *LocalWorker) ProveReplicaUpdate2(ctx context.Context, sector storage.Se }) } +func (l *LocalWorker) GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) (storiface.CallID, error) { + sb, err := l.executor() + if err != nil { + return storiface.UndefCall, err + } + + return l.asyncCall(ctx, sector, GenerateSectorKey, func(ctx context.Context, ci storiface.CallID) (interface{}, error) { + return nil, sb.GenerateSectorKeyFromData(ctx, sector, commD) + }) +} + func (l *LocalWorker) FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error) { sb, err := l.executor() if err != nil { diff --git a/go.mod b/go.mod index e0823094d..95618417b 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/filecoin-project/specs-actors/v5 v5.0.4 github.com/filecoin-project/specs-actors/v6 v6.0.1 github.com/filecoin-project/specs-actors/v7 v7.0.0-20211118013026-3dce48197cec - github.com/filecoin-project/specs-storage v0.1.1-0.20211123153428-712cb8da07a3 + github.com/filecoin-project/specs-storage v0.1.1-0.20211202151826-2e51da61d454 github.com/filecoin-project/test-vectors/schema v0.0.5 github.com/gbrlsnchs/jwt/v3 v3.0.1 github.com/gdamore/tcell/v2 v2.2.0 diff --git a/go.sum b/go.sum index e404ca72e..38f772025 100644 --- a/go.sum +++ b/go.sum @@ -396,8 +396,8 @@ github.com/filecoin-project/specs-actors/v6 v6.0.1/go.mod h1:V1AYfi5GkHXipx1mnVi github.com/filecoin-project/specs-actors/v7 v7.0.0-20211117170924-fd07a4c7dff9/go.mod h1:p6LIOFezA1rgRLMewbvdi3Pp6SAu+q9FtJ9CAleSjrE= github.com/filecoin-project/specs-actors/v7 v7.0.0-20211118013026-3dce48197cec h1:KV9vE+Sl2Y3qKsrpba4HcE7wHwK7v6O5U/S0xHbje6A= github.com/filecoin-project/specs-actors/v7 v7.0.0-20211118013026-3dce48197cec/go.mod h1:p6LIOFezA1rgRLMewbvdi3Pp6SAu+q9FtJ9CAleSjrE= -github.com/filecoin-project/specs-storage v0.1.1-0.20211123153428-712cb8da07a3 h1:FLPxD2ksWwGc/sbnFLWep2p8ViP93VCAwFaVxrtVCyo= -github.com/filecoin-project/specs-storage v0.1.1-0.20211123153428-712cb8da07a3/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= +github.com/filecoin-project/specs-storage v0.1.1-0.20211202151826-2e51da61d454 h1:9II9Xf+jq5xAPQiS4rVoKIiALINa3loMC+ghyFYIrqQ= +github.com/filecoin-project/specs-storage v0.1.1-0.20211202151826-2e51da61d454/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g= github.com/filecoin-project/test-vectors/schema v0.0.5 h1:w3zHQhzM4pYxJDl21avXjOKBLF8egrvwUwjpT8TquDg= github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= diff --git a/itests/self_sent_txn_test.go b/itests/self_sent_txn_test.go index 846bcff05..b5ec2c0dc 100644 --- a/itests/self_sent_txn_test.go +++ b/itests/self_sent_txn_test.go @@ -15,8 +15,7 @@ import ( "github.com/stretchr/testify/require" ) - -// these tests check that the versioned code in vm.transfer is functioning correctly across versions! +// these tests check that the versioned code in vm.transfer is functioning correctly across versions! // we reordered the checks to make sure that a transaction with too much money in it sent to yourself will fail instead of succeeding as a noop // more info in this PR! https://github.com/filecoin-project/lotus/pull/7637 func TestSelfSentTxnV15(t *testing.T) {