From 69fae9dc52e1514dfb4c04bd0029ddc5d6d0b57b Mon Sep 17 00:00:00 2001 From: Aayush Date: Thu, 9 Jun 2022 17:09:17 -0400 Subject: [PATCH] Add new proofs version type --- extern/filecoin-ffi | 2 +- extern/sector-storage/manager_test.go | 144 ++++++++++++++++++++ extern/storage-sealing/commit_batch.go | 44 ++++-- extern/storage-sealing/commit_batch_test.go | 2 +- go.sum | 1 + testplans/lotus-soup/go.sum | 1 + 6 files changed, 179 insertions(+), 15 deletions(-) diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 3595f0c29..acda280d7 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 3595f0c2924f53ad802bd139d227609c907a85b1 +Subproject commit acda280d74047ee7b247ffb8463cf29f1f108560 diff --git a/extern/sector-storage/manager_test.go b/extern/sector-storage/manager_test.go index 9c844292e..e8cba2fa5 100644 --- a/extern/sector-storage/manager_test.go +++ b/extern/sector-storage/manager_test.go @@ -16,6 +16,9 @@ import ( "testing" "time" + ffi "github.com/filecoin-project/filecoin-ffi" + "github.com/filecoin-project/go-state-types/proof" + "github.com/google/uuid" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" @@ -317,6 +320,147 @@ func TestSnapDeals(t *testing.T) { } +func TestSnarkPackV2(t *testing.T) { + logging.SetAllLoggers(logging.LevelWarn) + ctx := context.Background() + m, lstor, stor, idx, cleanup := newTestMgr(ctx, t, datastore.NewMapDatastore()) + defer cleanup() + + localTasks := []sealtasks.TaskType{ + sealtasks.TTAddPiece, sealtasks.TTPreCommit1, sealtasks.TTPreCommit2, sealtasks.TTCommit1, sealtasks.TTCommit2, sealtasks.TTFinalize, + sealtasks.TTFetch, sealtasks.TTReplicaUpdate, sealtasks.TTProveReplicaUpdate1, sealtasks.TTProveReplicaUpdate2, sealtasks.TTUnseal, + sealtasks.TTRegenSectorKey, + } + wds := datastore.NewMapDatastore() + + w := NewLocalWorker(WorkerConfig{TaskTypes: localTasks}, stor, lstor, idx, m, statestore.New(wds)) + err := m.AddWorker(ctx, w) + require.NoError(t, err) + + proofType := abi.RegisteredSealProof_StackedDrg2KiBV1 + ptStr := os.Getenv("LOTUS_TEST_SNAP_DEALS_PROOF_TYPE") + switch ptStr { + case "2k": + case "8M": + proofType = abi.RegisteredSealProof_StackedDrg8MiBV1 + case "512M": + proofType = abi.RegisteredSealProof_StackedDrg512MiBV1 + case "32G": + proofType = abi.RegisteredSealProof_StackedDrg32GiBV1 + case "64G": + proofType = abi.RegisteredSealProof_StackedDrg64GiBV1 + default: + log.Warn("Unspecified proof type, make sure to set LOTUS_TEST_SNAP_DEALS_PROOF_TYPE to '2k', '8M', '512M', '32G' or '64G'") + log.Warn("Continuing test with 2k sectors") + } + + mid := abi.ActorID(1000) + + sid1 := storage.SectorRef{ + ID: abi.SectorID{Miner: mid, Number: 1}, + ProofType: proofType, + } + + sid2 := storage.SectorRef{ + ID: abi.SectorID{Miner: mid, Number: 2}, + ProofType: proofType, + } + + ss, err := proofType.SectorSize() + require.NoError(t, err) + + unpaddedSectorSize := abi.PaddedPieceSize(ss).Unpadded() + + // Pack sector with no pieces + p1, err := m.AddPiece(ctx, sid1, nil, unpaddedSectorSize, NewNullReader(unpaddedSectorSize)) + require.NoError(t, err) + ccPieces1 := []abi.PieceInfo{p1} + + p2, err := m.AddPiece(ctx, sid2, nil, unpaddedSectorSize, NewNullReader(unpaddedSectorSize)) + require.NoError(t, err) + ccPieces2 := []abi.PieceInfo{p2} + + // Precommit and Seal 2 CC sectors + fmt.Printf("PC1\n") + + ticket1 := abi.SealRandomness{9, 9, 9, 9, 9, 9, 9, 9} + ticket2 := abi.SealRandomness{1, 9, 8, 9, 1, 9, 8, 9} + interactiveRandomness1 := abi.InteractiveSealRandomness{1, 9, 2, 1, 2, 5, 3, 0} + interactiveRandomness2 := abi.InteractiveSealRandomness{1, 5, 2, 2, 1, 5, 2, 2} + + pc1Out1, err := m.SealPreCommit1(ctx, sid1, ticket1, ccPieces1) + require.NoError(t, err) + pc1Out2, err := m.SealPreCommit1(ctx, sid2, ticket2, ccPieces2) + require.NoError(t, err) + + fmt.Printf("PC2\n") + + pc2Out1, err := m.SealPreCommit2(ctx, sid1, pc1Out1) + require.NoError(t, err) + pc2Out2, err := m.SealPreCommit2(ctx, sid2, pc1Out2) + require.NoError(t, err) + + // Commit the sector + + fmt.Printf("C1\n") + + c1Out1, err := m.SealCommit1(ctx, sid1, ticket1, interactiveRandomness1, ccPieces1, pc2Out1) + require.NoError(t, err) + c1Out2, err := m.SealCommit1(ctx, sid2, ticket2, interactiveRandomness2, ccPieces2, pc2Out2) + require.NoError(t, err) + + fmt.Printf("C2\n") + + c2Out1, err := m.SealCommit2(ctx, sid1, c1Out1) + require.NoError(t, err) + c2Out2, err := m.SealCommit2(ctx, sid2, c1Out2) + require.NoError(t, err) + + fmt.Println("Aggregate") + agg, err := ffi.AggregateSealProofs(proof.AggregateSealVerifyProofAndInfos{ + Miner: mid, + SealProof: proofType, + AggregateProof: abi.RegisteredAggregationProof_SnarkPackV2, + Infos: []proof.AggregateSealVerifyInfo{{ + Number: sid1.ID.Number, + Randomness: ticket1, + InteractiveRandomness: interactiveRandomness1, + SealedCID: pc2Out1.Sealed, + UnsealedCID: pc2Out1.Unsealed, + }, { + Number: sid2.ID.Number, + Randomness: ticket2, + InteractiveRandomness: interactiveRandomness2, + SealedCID: pc2Out2.Sealed, + UnsealedCID: pc2Out2.Unsealed, + }}, + }, [][]byte{c2Out1, c2Out2}) + require.NoError(t, err) + + fmt.Println("Verifying aggregate") + ret, err := ffi.VerifyAggregateSeals(proof.AggregateSealVerifyProofAndInfos{ + Miner: mid, + SealProof: proofType, + AggregateProof: abi.RegisteredAggregationProof_SnarkPackV2, + Proof: agg, + Infos: []proof.AggregateSealVerifyInfo{{ + Number: sid1.ID.Number, + Randomness: ticket1, + InteractiveRandomness: interactiveRandomness1, + SealedCID: pc2Out1.Sealed, + UnsealedCID: pc2Out1.Unsealed, + }, { + Number: sid2.ID.Number, + Randomness: ticket2, + InteractiveRandomness: interactiveRandomness2, + SealedCID: pc2Out2.Sealed, + UnsealedCID: pc2Out2.Unsealed, + }}, + }) + require.NoError(t, err) + require.True(t, ret, "proof should be good") +} + func TestRedoPC1(t *testing.T) { logging.SetAllLoggers(logging.LevelDebug) diff --git a/extern/storage-sealing/commit_batch.go b/extern/storage-sealing/commit_batch.go index b364184fd..f2a665293 100644 --- a/extern/storage-sealing/commit_batch.go +++ b/extern/storage-sealing/commit_batch.go @@ -29,8 +29,6 @@ import ( "github.com/filecoin-project/lotus/node/config" ) -const arp = abi.RegisteredAggregationProof_SnarkPackV1 - var aggFeeNum = big.NewInt(110) var aggFeeDen = big.NewInt(100) @@ -206,13 +204,22 @@ func (b *CommitBatcher) maybeStartBatch(notif bool) ([]sealiface.CommitBatchRes, var res []sealiface.CommitBatchRes - individual := (total < cfg.MinCommitBatch) || (total < miner.MinAggregatedSectors) + tok, h, err := b.api.ChainHead(b.mctx) + if err != nil { + return nil, err + } + + blackedOut := func() bool { + const nv16BlackoutWindow = abi.ChainEpoch(20) // a magik number + if h <= build.UpgradeSkyrHeight && build.UpgradeSkyrHeight-h < nv16BlackoutWindow { + return true + } + return false + } + + individual := (total < cfg.MinCommitBatch) || (total < miner.MinAggregatedSectors) || blackedOut() if !individual && !cfg.AggregateAboveBaseFee.Equals(big.Zero()) { - tok, _, err := b.api.ChainHead(b.mctx) - if err != nil { - return nil, err - } bf, err := b.api.ChainBaseFee(b.mctx, tok) if err != nil { @@ -314,6 +321,17 @@ func (b *CommitBatcher) processBatch(cfg sealiface.Config) ([]sealiface.CommitBa return []sealiface.CommitBatchRes{res}, xerrors.Errorf("getting miner id: %w", err) } + nv, err := b.api.StateNetworkVersion(b.mctx, tok) + if err != nil { + log.Errorf("getting network version: %s", err) + return []sealiface.CommitBatchRes{res}, xerrors.Errorf("getting network version: %s", err) + } + + arp, err := b.aggregateProofType(nv) + if err != nil { + return []sealiface.CommitBatchRes{res}, xerrors.Errorf("getting aggregate proof type: %w", err) + } + params.AggregateProof, err = b.prover.AggregateSealProofs(proof.AggregateSealVerifyProofAndInfos{ Miner: abi.ActorID(mid), SealProof: b.todo[infos[0].Number].Spt, @@ -341,12 +359,6 @@ func (b *CommitBatcher) processBatch(cfg sealiface.Config) ([]sealiface.CommitBa return []sealiface.CommitBatchRes{res}, xerrors.Errorf("couldn't get base fee: %w", err) } - nv, err := b.api.StateNetworkVersion(b.mctx, tok) - if err != nil { - log.Errorf("getting network version: %s", err) - return []sealiface.CommitBatchRes{res}, xerrors.Errorf("getting network version: %s", err) - } - aggFeeRaw, err := policy.AggregateProveCommitNetworkFee(nv, len(infos), bf) if err != nil { log.Errorf("getting aggregate commit network fee: %s", err) @@ -625,3 +637,9 @@ func (b *CommitBatcher) getSectorCollateral(sn abi.SectorNumber, tok TipSetToken return collateral, nil } +func (b *CommitBatcher) aggregateProofType(nv network.Version) (abi.RegisteredAggregationProof, error) { + if nv < network.Version16 { + return abi.RegisteredAggregationProof_SnarkPackV1, nil + } + return abi.RegisteredAggregationProof_SnarkPackV2, nil +} diff --git a/extern/storage-sealing/commit_batch_test.go b/extern/storage-sealing/commit_batch_test.go index 4b2f56aec..17988ed92 100644 --- a/extern/storage-sealing/commit_batch_test.go +++ b/extern/storage-sealing/commit_batch_test.go @@ -167,8 +167,8 @@ func TestCommitBatcher(t *testing.T) { basefee = types.NanoFil } + s.EXPECT().ChainHead(gomock.Any()).Return(nil, abi.ChainEpoch(1), nil) if batch { - s.EXPECT().ChainHead(gomock.Any()).Return(nil, abi.ChainEpoch(1), nil) s.EXPECT().ChainBaseFee(gomock.Any(), gomock.Any()).Return(basefee, nil) } diff --git a/go.sum b/go.sum index 977846cfe..748f72835 100644 --- a/go.sum +++ b/go.sum @@ -372,6 +372,7 @@ github.com/filecoin-project/go-state-types v0.1.3/go.mod h1:ezYnPf0bNkTsDibL/psS github.com/filecoin-project/go-state-types v0.1.4/go.mod h1:xCA/WfKlC2zcn3fUmDv4IrzznwS98X5XW/irUP3Lhxg= github.com/filecoin-project/go-state-types v0.1.5/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= +github.com/filecoin-project/go-state-types v0.1.8/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.9 h1:7ffQu+arDAiW5dphWTl8WdgWTo9Kt3BsjGcNYr99crI= github.com/filecoin-project/go-state-types v0.1.9/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= diff --git a/testplans/lotus-soup/go.sum b/testplans/lotus-soup/go.sum index f90dfced7..1d54436d6 100644 --- a/testplans/lotus-soup/go.sum +++ b/testplans/lotus-soup/go.sum @@ -463,6 +463,7 @@ github.com/filecoin-project/go-state-types v0.1.3/go.mod h1:ezYnPf0bNkTsDibL/psS github.com/filecoin-project/go-state-types v0.1.4/go.mod h1:xCA/WfKlC2zcn3fUmDv4IrzznwS98X5XW/irUP3Lhxg= github.com/filecoin-project/go-state-types v0.1.5/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= +github.com/filecoin-project/go-state-types v0.1.8/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.9 h1:7ffQu+arDAiW5dphWTl8WdgWTo9Kt3BsjGcNYr99crI= github.com/filecoin-project/go-state-types v0.1.9/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=