From 6daaf6ac2e10cffbdcd8820298c555a8145f4c6a Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 7 Apr 2021 15:14:14 -0700 Subject: [PATCH 1/5] attempt to do better padding on pieces being written into sectors --- extern/sector-storage/ffiwrapper/sealer_cgo.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index dca8b44b5..51f70e132 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -195,12 +195,16 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi return piecePromises[0]() } + var payloadRoundedBytes abi.PaddedPieceSize pieceCids := make([]abi.PieceInfo, len(piecePromises)) for i, promise := range piecePromises { - pieceCids[i], err = promise() + pinfo, err := promise() if err != nil { return abi.PieceInfo{}, err } + + pieceCids[i] = pinfo + payloadRoundedBytes += pinfo.Size } pieceCID, err := ffi.GenerateUnsealedCID(sector.ProofType, pieceCids) @@ -208,6 +212,15 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi return abi.PieceInfo{}, xerrors.Errorf("generate unsealed CID: %w", err) } + if payloadRoundedBytes < pieceSize.Padded() { + paddedCid, err := commpffi.ZeroPadPieceCommitment(pieceCID, payloadRoundedBytes.Unpadded(), pieceSize) + if err != nil { + return abi.PieceInfo{}, xerrors.Errorf("failed to pad data: %w", err) + } + + pieceCID = paddedCid + } + // validate that the pieceCID was properly formed if _, err := commcid.CIDToPieceCommitmentV1(pieceCID); err != nil { return abi.PieceInfo{}, err From 48feb52cbf0f42c5e86020f4090e90f2a61d0e16 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 27 Apr 2021 14:15:25 -0700 Subject: [PATCH 2/5] add a test for adding padded pieces --- .../sector-storage/ffiwrapper/sealer_test.go | 43 +++++++++++++++++++ go.mod | 2 +- go.sum | 2 + 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_test.go b/extern/sector-storage/ffiwrapper/sealer_test.go index 2efcfc6a0..39e37f0cc 100644 --- a/extern/sector-storage/ffiwrapper/sealer_test.go +++ b/extern/sector-storage/ffiwrapper/sealer_test.go @@ -709,3 +709,46 @@ func BenchmarkAddPiece512M(b *testing.B) { fmt.Println(c) } } + +func TestAddPiece512MPadded(t *testing.T) { + sz := abi.PaddedPieceSize(512 << 20).Unpadded() + + cdir, err := ioutil.TempDir("", "sbtest-c-") + if err != nil { + t.Fatal(err) + } + miner := abi.ActorID(123) + + sp := &basicfs.Provider{ + Root: cdir, + } + sb, err := New(sp) + if err != nil { + t.Fatalf("%+v", err) + } + cleanup := func() { + if t.Failed() { + fmt.Printf("not removing %s\n", cdir) + return + } + if err := os.RemoveAll(cdir); err != nil { + t.Error(err) + } + } + t.Cleanup(cleanup) + + r := rand.New(rand.NewSource(0x7e5)) + + c, err := sb.AddPiece(context.TODO(), storage.SectorRef{ + ID: abi.SectorID{ + Miner: miner, + Number: 0, + }, + ProofType: abi.RegisteredSealProof_StackedDrg512MiBV1_1, + }, nil, sz, io.LimitReader(r, int64(sz/4))) + if err != nil { + t.Fatalf("add piece failed: %s", err) + } + + require.Equal(t, "baga6ea4seaqonenxyku4o7hr5xkzbqsceipf6xgli3on54beqbk6k246sbooobq", c.PieceCID.String()) +} diff --git a/go.mod b/go.mod index 86e6da40d..3c9cfdfc0 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 // indirect github.com/filecoin-project/go-bitfield v0.2.4 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 - github.com/filecoin-project/go-commp-utils v0.1.0 + github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427180530-4606b1a6cbdd github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v1.4.3 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a diff --git a/go.sum b/go.sum index 8bcfafdfc..7ce7ede20 100644 --- a/go.sum +++ b/go.sum @@ -262,6 +262,8 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.m github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= github.com/filecoin-project/go-commp-utils v0.1.0 h1:PaDxoXYh1TXnnz5kA/xSObpAQwcJSUs4Szb72nuaNdk= github.com/filecoin-project/go-commp-utils v0.1.0/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= +github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427180530-4606b1a6cbdd h1:cpwbE1z6a+0fp62P0Qv7Z7ZqIy8Jiay0SWR0xcuZ0qs= +github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427180530-4606b1a6cbdd/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo= From ac77c51d5e23f3b5ed318cde4362d82e46d5aaed Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 29 Apr 2021 10:24:16 -0700 Subject: [PATCH 3/5] address nit --- extern/sector-storage/ffiwrapper/sealer_cgo.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index 51f70e132..36fbacb30 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -212,6 +212,11 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi return abi.PieceInfo{}, xerrors.Errorf("generate unsealed CID: %w", err) } + // validate that the pieceCID was properly formed + if _, err := commcid.CIDToPieceCommitmentV1(pieceCID); err != nil { + return abi.PieceInfo{}, err + } + if payloadRoundedBytes < pieceSize.Padded() { paddedCid, err := commpffi.ZeroPadPieceCommitment(pieceCID, payloadRoundedBytes.Unpadded(), pieceSize) if err != nil { @@ -221,11 +226,6 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector storage.SectorRef, existi pieceCID = paddedCid } - // validate that the pieceCID was properly formed - if _, err := commcid.CIDToPieceCommitmentV1(pieceCID); err != nil { - return abi.PieceInfo{}, err - } - return abi.PieceInfo{ Size: pieceSize.Padded(), PieceCID: pieceCID, From a590fb6a12668ec291a14b7d01594bdf2029daab Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 29 Apr 2021 10:28:33 -0700 Subject: [PATCH 4/5] mod tidy for the magik --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 7ce7ede20..3e548b523 100644 --- a/go.sum +++ b/go.sum @@ -260,8 +260,6 @@ github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQj github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= -github.com/filecoin-project/go-commp-utils v0.1.0 h1:PaDxoXYh1TXnnz5kA/xSObpAQwcJSUs4Szb72nuaNdk= -github.com/filecoin-project/go-commp-utils v0.1.0/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427180530-4606b1a6cbdd h1:cpwbE1z6a+0fp62P0Qv7Z7ZqIy8Jiay0SWR0xcuZ0qs= github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427180530-4606b1a6cbdd/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= From 7e4d906f0667283e4697df61eb2fb626a8e2d884 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 29 Apr 2021 10:59:28 -0700 Subject: [PATCH 5/5] update commp utils import --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3c9cfdfc0..9d5accbf1 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 // indirect github.com/filecoin-project/go-bitfield v0.2.4 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 - github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427180530-4606b1a6cbdd + github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 github.com/filecoin-project/go-data-transfer v1.4.3 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a diff --git a/go.sum b/go.sum index 3e548b523..54c5da743 100644 --- a/go.sum +++ b/go.sum @@ -260,8 +260,8 @@ github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQj github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8= github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= -github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427180530-4606b1a6cbdd h1:cpwbE1z6a+0fp62P0Qv7Z7ZqIy8Jiay0SWR0xcuZ0qs= -github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427180530-4606b1a6cbdd/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= +github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7 h1:U9Z+76pHCKBmtdxFV7JFZJj7OVm12I6dEKwtMVbq5p0= +github.com/filecoin-project/go-commp-utils v0.1.1-0.20210427191551-70bf140d31c7/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo=