From a8997dc35c60db2ac6c9b9e74f14e20ab6fa8933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 25 Jun 2020 21:53:51 +0200 Subject: [PATCH 01/16] ffiwrapper: Insert alignment between pieces --- ffiwrapper/sealer_cgo.go | 46 ++++++++++------- ffiwrapper/sealer_test.go | 103 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 17 deletions(-) diff --git a/ffiwrapper/sealer_cgo.go b/ffiwrapper/sealer_cgo.go index 6510f81cc..177ddeae0 100644 --- a/ffiwrapper/sealer_cgo.go +++ b/ffiwrapper/sealer_cgo.go @@ -543,8 +543,35 @@ func GeneratePieceCIDFromFile(proofType abi.RegisteredSealProof, piece io.Reader } func GenerateUnsealedCID(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) { + allPieces := make([]abi.PieceInfo, 0, len(pieces)) var sum abi.PaddedPieceSize + + padTo := func(s abi.PaddedPieceSize, trailing bool) { + // pad remaining space with 0 CommPs + toFill := uint64(-sum % s) + if trailing && sum == 0 { + toFill = uint64(s) + } + + n := bits.OnesCount64(toFill) + for i := 0; i < n; i++ { + next := bits.TrailingZeros64(toFill) + psize := uint64(1) << uint(next) + toFill ^= psize + + padded := abi.PaddedPieceSize(psize) + allPieces = append(allPieces, abi.PieceInfo{ + Size: padded, + PieceCID: zerocomm.ZeroPieceCommitment(padded.Unpadded()), + }) + sum += padded + } + } + for _, p := range pieces { + padTo(p.Size, false) + + allPieces = append(allPieces, p) sum += p.Size } @@ -553,22 +580,7 @@ func GenerateUnsealedCID(proofType abi.RegisteredSealProof, pieces []abi.PieceIn return cid.Undef, err } - { - // pad remaining space with 0 CommPs - toFill := uint64(abi.PaddedPieceSize(ssize) - sum) - n := bits.OnesCount64(toFill) - for i := 0; i < n; i++ { - next := bits.TrailingZeros64(toFill) - psize := uint64(1) << uint(next) - toFill ^= psize + padTo(abi.PaddedPieceSize(ssize), true) - unpadded := abi.PaddedPieceSize(psize).Unpadded() - pieces = append(pieces, abi.PieceInfo{ - Size: unpadded.Padded(), - PieceCID: zerocomm.ZeroPieceCommitment(unpadded), - }) - } - } - - return ffi.GenerateUnsealedCID(proofType, pieces) + return ffi.GenerateUnsealedCID(proofType, allPieces) } diff --git a/ffiwrapper/sealer_test.go b/ffiwrapper/sealer_test.go index 5e6f02cd2..e9628c2dd 100644 --- a/ffiwrapper/sealer_test.go +++ b/ffiwrapper/sealer_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/ipfs/go-cid" "io" "io/ioutil" "math/rand" @@ -484,3 +485,105 @@ func requireFDsClosed(t *testing.T, start int) { log.Infow("open FDs", "start", start, "now", openNow) require.Equal(t, start, openNow, "FDs shouldn't leak") } + +func TestGenerateUnsealedCID(t *testing.T) { + pt := abi.RegisteredSealProof_StackedDrg2KiBV1 + ups := int(abi.PaddedPieceSize(2048).Unpadded()) + + commP := func(b []byte) cid.Cid { + pf, werr, err := ToReadableFile(bytes.NewReader(b), int64(len(b))) + require.NoError(t, err) + + c, err := ffi.GeneratePieceCIDFromFile(pt, pf, abi.UnpaddedPieceSize(len(b))) + require.NoError(t, err) + + require.NoError(t, werr()) + + return c + } + + testCommEq := func(name string, in [][]byte, expect [][]byte) { + t.Run(name, func(t *testing.T) { + upi := make([]abi.PieceInfo, len(in)) + for i, b := range in { + upi[i] = abi.PieceInfo{ + Size: abi.UnpaddedPieceSize(len(b)).Padded(), + PieceCID: commP(b), + } + } + + sectorPi := []abi.PieceInfo{ + { + Size: 2048, + PieceCID: commP(bytes.Join(expect, nil)), + }, + } + + expectCid, err := GenerateUnsealedCID(pt, sectorPi) + require.NoError(t, err) + + actualCid, err := GenerateUnsealedCID(pt, upi) + require.NoError(t, err) + + require.Equal(t, expectCid, actualCid) + }) + } + + barr := func(b byte, den int) []byte { + return bytes.Repeat([]byte{b}, ups/den) + } + + // 0000 + testCommEq("zero", + nil, + [][]byte{barr(0, 1)}, + ) + + // 1111 + testCommEq("one", + [][]byte{barr(1, 1)}, + [][]byte{barr(1, 1)}, + ) + + // 11 00 + testCommEq("one|2", + [][]byte{barr(1, 2)}, + [][]byte{barr(1, 2), barr(0, 2)}, + ) + + // 1 0 00 + testCommEq("one|4", + [][]byte{barr(1, 4)}, + [][]byte{barr(1, 4), barr(0, 4), barr(0, 2)}, + ) + + // 11 2 0 + testCommEq("one|2-two|4", + [][]byte{barr(1, 2), barr(2, 4)}, + [][]byte{barr(1, 2), barr(2, 4), barr(0, 4)}, + ) + + // 1 0 22 + testCommEq("one|4-two|2", + [][]byte{barr(1, 4), barr(2, 2)}, + [][]byte{barr(1, 4), barr(0, 4), barr(2, 2)}, + ) + + // 1 0 22 0000 + testCommEq("one|8-two|4", + [][]byte{barr(1, 8), barr(2, 4)}, + [][]byte{barr(1, 8), barr(0, 8), barr(2, 4), barr(0, 2)}, + ) + + // 11 2 0 0000 + testCommEq("one|4-two|8", + [][]byte{barr(1, 4), barr(2, 8)}, + [][]byte{barr(1, 4), barr(2, 8), barr(0, 8), barr(0, 2)}, + ) + + // 1 0 22 3 0 00 4444 5 0 00 + testCommEq("one|16-two|8-three|16-four|4-five|16", + [][]byte{barr(1, 16), barr(2, 8), barr(3, 16), barr(4, 4), barr(5, 16)}, + [][]byte{barr(1, 16), barr(0, 16), barr(2, 8), barr(3, 16), barr(0, 16), barr(0, 8), barr(4, 4), barr(5, 16), barr(0, 16), barr(0, 8)}, + ) +} From 31d9abfc8cac192758b62c27945e7480fc5fe328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 3 Jul 2020 21:52:31 +0200 Subject: [PATCH 02/16] Implement FinalizeSector keepUnsealed --- ffiwrapper/partialfile.go | 23 ++++++++++++++++ ffiwrapper/sealer_cgo.go | 56 ++++++++++++++++++++++++++++++++++++++- fsutil/dealloc_linux.go | 28 ++++++++++++++++++++ fsutil/dealloc_other.go | 18 +++++++++++++ go.mod | 2 +- go.sum | 2 ++ localworker.go | 6 +++-- 7 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 fsutil/dealloc_linux.go create mode 100644 fsutil/dealloc_other.go diff --git a/ffiwrapper/partialfile.go b/ffiwrapper/partialfile.go index a2c1f1151..8c4fdcc72 100644 --- a/ffiwrapper/partialfile.go +++ b/ffiwrapper/partialfile.go @@ -12,6 +12,7 @@ import ( rlepluslazy "github.com/filecoin-project/go-bitfield/rle" "github.com/filecoin-project/specs-actors/actors/abi" + "github.com/filecoin-project/sector-storage/fsutil" "github.com/filecoin-project/sector-storage/storiface" ) @@ -218,6 +219,28 @@ func (pf *partialFile) MarkAllocated(offset storiface.PaddedByteIndex, size abi. return nil } +func (pf *partialFile) Free(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) error { + have, err := pf.allocated.RunIterator() + if err != nil { + return err + } + + if err := fsutil.Deallocate(pf.file, int64(offset), int64(size)); err != nil { + return xerrors.Errorf("deallocating: %w", err) + } + + s, err := rlepluslazy.Subtract(have, pieceRun(offset, size)) + if err != nil { + return err + } + + if err := writeTrailer(int64(pf.maxPiece), pf.file, s); err != nil { + return xerrors.Errorf("writing trailer: %w", err) + } + + return nil +} + func (pf *partialFile) Reader(offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (*os.File, error) { if _, err := pf.file.Seek(int64(offset), io.SeekStart); err != nil { return nil, xerrors.Errorf("seek piece start: %w", err) diff --git a/ffiwrapper/sealer_cgo.go b/ffiwrapper/sealer_cgo.go index 177ddeae0..58d9d8c5b 100644 --- a/ffiwrapper/sealer_cgo.go +++ b/ffiwrapper/sealer_cgo.go @@ -15,6 +15,7 @@ import ( "golang.org/x/xerrors" ffi "github.com/filecoin-project/filecoin-ffi" + rlepluslazy "github.com/filecoin-project/go-bitfield/rle" commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-storage/storage" @@ -502,7 +503,60 @@ func (sb *Sealer) SealCommit2(ctx context.Context, sector abi.SectorID, phase1Ou func (sb *Sealer) FinalizeSector(ctx context.Context, sector abi.SectorID, keepUnsealed []storage.Range) error { if len(keepUnsealed) > 0 { - return xerrors.Errorf("keepUnsealed unsupported") // TODO: impl for fastretrieval copies + maxPieceSize := abi.PaddedPieceSize(sb.ssize) + + sr := pieceRun(0, maxPieceSize) + + for _, s := range keepUnsealed { + si := &rlepluslazy.RunSliceIterator{} + if s.Offset != 0 { + si.Runs = append(si.Runs, rlepluslazy.Run{Val: false, Len: uint64(s.Offset)}) + } + si.Runs = append(si.Runs, rlepluslazy.Run{Val: true, Len: uint64(s.Size)}) + + var err error + sr, err = rlepluslazy.Subtract(sr, si) + if err != nil { + return err + } + } + + + paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, 0, false) + if err != nil { + return xerrors.Errorf("acquiring sector cache path: %w", err) + } + defer done() + + pf, err := openPartialFile(maxPieceSize, paths.Unsealed) + if xerrors.Is(err, os.ErrNotExist) { + return xerrors.Errorf("opening partial file: %w", err) + } + + var at uint64 + for sr.HasNext() { + r, err := sr.NextRun() + if err != nil { + _ = pf.Close() + return err + } + + offset := at + at += r.Len + if !r.Val { + continue + } + + err = pf.Free(storiface.PaddedByteIndex(abi.UnpaddedPieceSize(offset).Padded()), abi.UnpaddedPieceSize(r.Len).Padded()) + if err != nil { + _ = pf.Close() + return xerrors.Errorf("free partial file range: %w", err) + } + } + + if err := pf.Close(); err != nil { + return err + } } paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTCache, 0, false) diff --git a/fsutil/dealloc_linux.go b/fsutil/dealloc_linux.go new file mode 100644 index 000000000..0b20c568d --- /dev/null +++ b/fsutil/dealloc_linux.go @@ -0,0 +1,28 @@ +package fsutil + +import ( + "os" + "syscall" + + logging "github.com/ipfs/go-log/v2" +) + +var log = logging.Logger("fsutil") + +const FallocFlPunchHole = 0x02 // linux/falloc.h + +func Deallocate(file *os.File, offset int64, length int64) error { + if length == 0 { + return nil + } + + err := syscall.Fallocate(int(file.Fd()), FallocFlPunchHole, offset, length) + if errno, ok := err.(syscall.Errno); ok { + if errno == syscall.EOPNOTSUPP || errno == syscall.ENOSYS { + log.Warnf("could not deallocate space, ignoring: %v", errno) + err = nil // log and ignore + } + } + + return err +} diff --git a/fsutil/dealloc_other.go b/fsutil/dealloc_other.go new file mode 100644 index 000000000..721116af1 --- /dev/null +++ b/fsutil/dealloc_other.go @@ -0,0 +1,18 @@ +// +build !linux + +package fsutil + +import ( + "os" + + logging "github.com/ipfs/go-log/v2" +) + +var log = logging.Logger("fsutil") + + +func Deallocate(file *os.File, offset int64, length int64) error { + log.Warnf("deallocating space not supported") + + return err +} diff --git a/go.mod b/go.mod index 9e51c0445..83424841f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e github.com/elastic/go-sysinfo v1.3.0 github.com/filecoin-project/filecoin-ffi v0.0.0-20200326153646-e899cc1dd072 - github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e + github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1 github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 github.com/filecoin-project/go-paramfetch v0.0.1 github.com/filecoin-project/specs-actors v0.6.1 diff --git a/go.sum b/go.sum index 2f97216e3..330b97579 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,8 @@ github.com/filecoin-project/go-bitfield v0.0.1 h1:Xg/JnrqqE77aJVKdbEyR04n9FZQWhw github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e h1:gkG/7G+iKy4He+IiQNeQn+nndFznb/vCoOR8iRQsm60= github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= +github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1 h1:xuHlrdznafh7ul5t4xEncnA4qgpQvJZEw+mr98eqHXw= +github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 h1:yvQJCW9mmi9zy+51xA01Ea2X7/dL7r8eKDPuGUjRmbo= diff --git a/localworker.go b/localworker.go index a1d82209a..a6042826a 100644 --- a/localworker.go +++ b/localworker.go @@ -171,8 +171,10 @@ func (l *LocalWorker) FinalizeSector(ctx context.Context, sector abi.SectorID, k return xerrors.Errorf("finalizing sector: %w", err) } - if err := l.storage.Remove(ctx, sector, stores.FTUnsealed, true); err != nil { - return xerrors.Errorf("removing unsealed data: %w", err) + if len(keepUnsealed) == 0 { + if err := l.storage.Remove(ctx, sector, stores.FTUnsealed, true); err != nil { + return xerrors.Errorf("removing unsealed data: %w", err) + } } return nil From 636bf90f842d7532adda8bf6ea45ffcd1d350ff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 3 Jul 2020 22:23:36 +0200 Subject: [PATCH 03/16] Don't error in ReleaseUnsealed --- manager.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manager.go b/manager.go index caea09cd0..767e87cf9 100644 --- a/manager.go +++ b/manager.go @@ -441,7 +441,8 @@ func (m *Manager) FinalizeSector(ctx context.Context, sector abi.SectorID, keepU } func (m *Manager) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error { - return xerrors.Errorf("implement me") + log.Warnw("ReleaseUnsealed todo") + return nil } func (m *Manager) Remove(ctx context.Context, sector abi.SectorID) error { From 0fd142153a2b844e17676e26d29c084fd1c5708f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 3 Jul 2020 22:24:47 +0200 Subject: [PATCH 04/16] mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 330b97579..508d985d7 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,6 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200424220931-6263827e49f2/ github.com/filecoin-project/go-bitfield v0.0.0-20200416002808-b3ee67ec9060/go.mod h1:iodsLxOFZnqKtjj2zkgqzoGNrv6vUqj69AT/J8DKXEw= github.com/filecoin-project/go-bitfield v0.0.1 h1:Xg/JnrqqE77aJVKdbEyR04n9FZQWhwrN+buDgQCVpZU= github.com/filecoin-project/go-bitfield v0.0.1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= -github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e h1:gkG/7G+iKy4He+IiQNeQn+nndFznb/vCoOR8iRQsm60= -github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1 h1:xuHlrdznafh7ul5t4xEncnA4qgpQvJZEw+mr98eqHXw= github.com/filecoin-project/go-bitfield v0.0.4-0.20200703174658-f4a5758051a1/go.mod h1:Ry9/iUlWSyjPUzlAvdnfy4Gtvrq4kWmWDztCU1yEgJY= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= From c5a96fdd08d3fd32ca8b19425adaa903cd9b344e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 6 Jul 2020 16:13:42 +0200 Subject: [PATCH 05/16] Change PathType to string --- faults.go | 6 ++++-- ffiwrapper/sealer_cgo.go | 20 ++++++++++---------- ffiwrapper/sealer_test.go | 2 +- ffiwrapper/verifier_cgo.go | 2 +- manager.go | 4 ++-- stores/filetype.go | 2 +- stores/http_handler.go | 2 +- stores/interface.go | 6 +++--- stores/local.go | 4 ++-- 9 files changed, 25 insertions(+), 23 deletions(-) diff --git a/faults.go b/faults.go index 11c1c3df2..0eebc42f0 100644 --- a/faults.go +++ b/faults.go @@ -43,9 +43,11 @@ func (m *Manager) CheckProvable(ctx context.Context, spt abi.RegisteredSealProof return nil } - lp, _, err := m.localStore.AcquireSector(ctx, sector, spt, stores.FTSealed|stores.FTCache, stores.FTNone, false, stores.AcquireMove) + lp, _, err := m.localStore.AcquireSector(ctx, sector, spt, stores.FTSealed|stores.FTCache, stores.FTNone, stores.PathStorage, stores.AcquireMove) if err != nil { - return xerrors.Errorf("acquire sector in checkProvable: %w", err) + log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err) + bad = append(bad, sector) + return nil } if lp.Sealed == "" || lp.Cache == "" { diff --git a/ffiwrapper/sealer_cgo.go b/ffiwrapper/sealer_cgo.go index 177ddeae0..0ff8c8f2f 100644 --- a/ffiwrapper/sealer_cgo.go +++ b/ffiwrapper/sealer_cgo.go @@ -81,7 +81,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector abi.SectorID, existingPie var stagedPath stores.SectorPaths if len(existingPieceSizes) == 0 { - stagedPath, done, err = sb.sectors.AcquireSector(ctx, sector, 0, stores.FTUnsealed, true) + stagedPath, done, err = sb.sectors.AcquireSector(ctx, sector, 0, stores.FTUnsealed, stores.PathSealing) if err != nil { return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) } @@ -91,7 +91,7 @@ func (sb *Sealer) AddPiece(ctx context.Context, sector abi.SectorID, existingPie return abi.PieceInfo{}, xerrors.Errorf("creating unsealed sector file: %w", err) } } else { - stagedPath, done, err = sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, 0, true) + stagedPath, done, err = sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, 0, stores.PathSealing) if err != nil { return abi.PieceInfo{}, xerrors.Errorf("acquire unsealed sector: %w", err) } @@ -198,12 +198,12 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector abi.SectorID, offset s maxPieceSize := abi.PaddedPieceSize(sb.ssize) // try finding existing - unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTNone, false) + unsealedPath, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTNone, stores.PathStorage) var pf *partialFile switch { case xerrors.Is(err, storiface.ErrSectorNotFound): - unsealedPath, done, err = sb.sectors.AcquireSector(ctx, sector, stores.FTNone, stores.FTUnsealed, false) + unsealedPath, done, err = sb.sectors.AcquireSector(ctx, sector, stores.FTNone, stores.FTUnsealed, stores.PathStorage) if err != nil { return xerrors.Errorf("acquire unsealed sector path (allocate): %w", err) } @@ -240,7 +240,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector abi.SectorID, offset s return nil } - srcPaths, srcDone, err := sb.sectors.AcquireSector(ctx, sector, stores.FTCache|stores.FTSealed, stores.FTNone, false) + srcPaths, srcDone, err := sb.sectors.AcquireSector(ctx, sector, stores.FTCache|stores.FTSealed, stores.FTNone, stores.PathStorage) if err != nil { return xerrors.Errorf("acquire sealed sector paths: %w", err) } @@ -358,7 +358,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector abi.SectorID, offset s } func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector abi.SectorID, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) error { - path, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTNone, false) + path, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTNone, stores.PathStorage) if err != nil { return xerrors.Errorf("acquire unsealed sector path: %w", err) } @@ -395,7 +395,7 @@ func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector abi.Se } func (sb *Sealer) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { - paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTSealed|stores.FTCache, true) + paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, stores.FTSealed|stores.FTCache, stores.PathSealing) if err != nil { return nil, xerrors.Errorf("acquiring sector paths: %w", err) } @@ -452,7 +452,7 @@ func (sb *Sealer) SealPreCommit1(ctx context.Context, sector abi.SectorID, ticke } func (sb *Sealer) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase1Out storage.PreCommit1Out) (storage.SectorCids, error) { - paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTSealed|stores.FTCache, 0, true) + paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTSealed|stores.FTCache, 0, stores.PathSealing) if err != nil { return storage.SectorCids{}, xerrors.Errorf("acquiring sector paths: %w", err) } @@ -470,7 +470,7 @@ func (sb *Sealer) SealPreCommit2(ctx context.Context, sector abi.SectorID, phase } func (sb *Sealer) SealCommit1(ctx context.Context, sector abi.SectorID, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storage.Commit1Out, error) { - paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTSealed|stores.FTCache, 0, true) + paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTSealed|stores.FTCache, 0, stores.PathSealing) if err != nil { return nil, xerrors.Errorf("acquire sector paths: %w", err) } @@ -505,7 +505,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector abi.SectorID, keepU return xerrors.Errorf("keepUnsealed unsupported") // TODO: impl for fastretrieval copies } - paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTCache, 0, false) + paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTCache, 0, stores.PathStorage) if err != nil { return xerrors.Errorf("acquiring sector cache path: %w", err) } diff --git a/ffiwrapper/sealer_test.go b/ffiwrapper/sealer_test.go index 5b9c3d1ac..0b5018d84 100644 --- a/ffiwrapper/sealer_test.go +++ b/ffiwrapper/sealer_test.go @@ -121,7 +121,7 @@ func (s *seal) unseal(t *testing.T, sb *Sealer, sp *basicfs.Provider, si abi.Sec t.Fatal("read wrong bytes") } - p, sd, err := sp.AcquireSector(context.TODO(), si, stores.FTUnsealed, stores.FTNone, false) + p, sd, err := sp.AcquireSector(context.TODO(), si, stores.FTUnsealed, stores.FTNone, stores.PathStorage) if err != nil { t.Fatal(err) } diff --git a/ffiwrapper/verifier_cgo.go b/ffiwrapper/verifier_cgo.go index e3e8dd886..60d56dddc 100644 --- a/ffiwrapper/verifier_cgo.go +++ b/ffiwrapper/verifier_cgo.go @@ -62,7 +62,7 @@ func (sb *Sealer) pubSectorToPriv(ctx context.Context, mid abi.ActorID, sectorIn sid := abi.SectorID{Miner: mid, Number: s.SectorNumber} - paths, d, err := sb.sectors.AcquireSector(ctx, sid, stores.FTCache|stores.FTSealed, 0, false) + paths, d, err := sb.sectors.AcquireSector(ctx, sid, stores.FTCache|stores.FTSealed, 0, stores.PathStorage) if err != nil { log.Warnw("failed to acquire sector, skipping", "sector", sid, "error", err) skipped = append(skipped, sid) diff --git a/manager.go b/manager.go index caea09cd0..6c1b93ced 100644 --- a/manager.go +++ b/manager.go @@ -218,12 +218,12 @@ func (m *Manager) ReadPiece(ctx context.Context, sink io.Writer, sector abi.Sect // TODO: Optimization: don't send unseal to a worker if the requested range is already unsealed unsealFetch := func(ctx context.Context, worker Worker) error { - if err := worker.Fetch(ctx, sector, stores.FTSealed|stores.FTCache, true, stores.AcquireCopy); err != nil { + if err := worker.Fetch(ctx, sector, stores.FTSealed|stores.FTCache, stores.PathSealing, stores.AcquireCopy); err != nil { return xerrors.Errorf("copy sealed/cache sector data: %w", err) } if len(best) > 0 { - if err := worker.Fetch(ctx, sector, stores.FTUnsealed, true, stores.AcquireMove); err != nil { + if err := worker.Fetch(ctx, sector, stores.FTUnsealed, stores.PathSealing, stores.AcquireMove); err != nil { return xerrors.Errorf("copy unsealed sector data: %w", err) } } diff --git a/stores/filetype.go b/stores/filetype.go index c31dfefb2..60c47d1f7 100644 --- a/stores/filetype.go +++ b/stores/filetype.go @@ -22,7 +22,7 @@ const ( var FSOverheadSeal = map[SectorFileType]int{ // 10x overheads FTUnsealed: 10, FTSealed: 10, - FTCache: 70, // TODO: confirm for 32G + FTCache: 141, // 11 layers + D(2x ssize) + C + R } var FsOverheadFinalized = map[SectorFileType]int{ diff --git a/stores/http_handler.go b/stores/http_handler.go index 60f8a41c5..93fb94637 100644 --- a/stores/http_handler.go +++ b/stores/http_handler.go @@ -72,7 +72,7 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ // The caller has a lock on this sector already, no need to get one here // passing 0 spt because we don't allocate anything - paths, _, err := handler.Local.AcquireSector(r.Context(), id, 0, ft, FTNone, false, AcquireMove) + paths, _, err := handler.Local.AcquireSector(r.Context(), id, 0, ft, FTNone, PathStorage, AcquireMove) if err != nil { log.Error("%+v", err) w.WriteHeader(500) diff --git a/stores/interface.go b/stores/interface.go index 54aaec90c..b61980125 100644 --- a/stores/interface.go +++ b/stores/interface.go @@ -9,11 +9,11 @@ import ( "github.com/filecoin-project/specs-actors/actors/abi" ) -type PathType bool +type PathType string const ( - PathStorage = false - PathSealing = true + PathStorage = "storage" + PathSealing = "sealing" ) type AcquireMode string diff --git a/stores/local.go b/stores/local.go index 26b7ccb75..ac63ae0dd 100644 --- a/stores/local.go +++ b/stores/local.go @@ -398,12 +398,12 @@ func (st *Local) removeSector(ctx context.Context, sid abi.SectorID, typ SectorF } func (st *Local) MoveStorage(ctx context.Context, s abi.SectorID, spt abi.RegisteredSealProof, types SectorFileType) error { - dest, destIds, err := st.AcquireSector(ctx, s, spt, FTNone, types, false, AcquireMove) + dest, destIds, err := st.AcquireSector(ctx, s, spt, FTNone, types, PathStorage, AcquireMove) if err != nil { return xerrors.Errorf("acquire dest storage: %w", err) } - src, srcIds, err := st.AcquireSector(ctx, s, spt, types, FTNone, false, AcquireMove) + src, srcIds, err := st.AcquireSector(ctx, s, spt, types, FTNone, PathStorage, AcquireMove) if err != nil { return xerrors.Errorf("acquire src storage: %w", err) } From 8099621cd0b2a73f96751431ddd861e27498fae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 6 Jul 2020 18:36:44 +0200 Subject: [PATCH 06/16] stores: Allow reserving local storage --- localworker.go | 8 ++++ stores/filetype.go | 12 +++--- stores/http_handler.go | 2 + stores/index.go | 2 +- stores/interface.go | 11 +++--- stores/local.go | 83 ++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 104 insertions(+), 14 deletions(-) diff --git a/localworker.go b/localworker.go index a1d82209a..d03ace359 100644 --- a/localworker.go +++ b/localworker.go @@ -61,14 +61,22 @@ type localWorkerPathProvider struct { } func (l *localWorkerPathProvider) AcquireSector(ctx context.Context, sector abi.SectorID, existing stores.SectorFileType, allocate stores.SectorFileType, sealing stores.PathType) (stores.SectorPaths, func(), error) { + paths, storageIDs, err := l.w.storage.AcquireSector(ctx, sector, l.w.scfg.SealProofType, existing, allocate, sealing, l.op) if err != nil { return stores.SectorPaths{}, nil, err } + releaseStorage, err := l.w.localStore.Reserve(ctx, sector, l.w.scfg.SealProofType, allocate, storageIDs, stores.FSOverheadSeal) + if err != nil { + return stores.SectorPaths{}, nil, xerrors.Errorf("reserving storage space: %w", err) + } + log.Debugf("acquired sector %d (e:%d; a:%d): %v", sector, existing, allocate, paths) return paths, func() { + releaseStorage() + for _, fileType := range pathTypes { if fileType&allocate == 0 { continue diff --git a/stores/filetype.go b/stores/filetype.go index 60c47d1f7..650b92f71 100644 --- a/stores/filetype.go +++ b/stores/filetype.go @@ -19,15 +19,17 @@ const ( FTNone SectorFileType = 0 ) +const FSOverheadDen = 10 + var FSOverheadSeal = map[SectorFileType]int{ // 10x overheads - FTUnsealed: 10, - FTSealed: 10, + FTUnsealed: FSOverheadDen, + FTSealed: FSOverheadDen, FTCache: 141, // 11 layers + D(2x ssize) + C + R } var FsOverheadFinalized = map[SectorFileType]int{ - FTUnsealed: 10, - FTSealed: 10, + FTUnsealed: FSOverheadDen, + FTSealed: FSOverheadDen, FTCache: 2, } @@ -67,7 +69,7 @@ func (t SectorFileType) SealSpaceUse(spt abi.RegisteredSealProof) (uint64, error return 0, xerrors.Errorf("no seal overhead info for %s", pathType) } - need += uint64(oh) * uint64(ssize) / 10 + need += uint64(oh) * uint64(ssize) / FSOverheadDen } return need, nil diff --git a/stores/http_handler.go b/stores/http_handler.go index 93fb94637..4f0556138 100644 --- a/stores/http_handler.go +++ b/stores/http_handler.go @@ -79,6 +79,8 @@ func (handler *FetchHandler) remoteGetSector(w http.ResponseWriter, r *http.Requ return } + // TODO: reserve local storage here + path := PathByType(paths, ft) if path == "" { log.Error("acquired path was empty") diff --git a/stores/index.go b/stores/index.go index 049e2dc20..e48ae02bb 100644 --- a/stores/index.go +++ b/stores/index.go @@ -361,7 +361,7 @@ func (i *Index) StorageBestAlloc(ctx context.Context, allocate SectorFileType, s continue } - if spaceReq > p.fsi.Available { + if spaceReq > uint64(p.fsi.Available) { log.Debugf("not allocating on %s, out of space (available: %d, need: %d)", p.info.ID, p.fsi.Available, spaceReq) continue } diff --git a/stores/interface.go b/stores/interface.go index b61980125..6fd4a7ad7 100644 --- a/stores/interface.go +++ b/stores/interface.go @@ -44,13 +44,14 @@ func Stat(path string) (FsStat, error) { } return FsStat{ - Capacity: stat.Blocks * uint64(stat.Bsize), - Available: stat.Bavail * uint64(stat.Bsize), + Capacity: int64(stat.Blocks) * stat.Bsize, + Available: int64(stat.Bavail) * stat.Bsize, }, nil } type FsStat struct { - Capacity uint64 - Available uint64 // Available to use for sector storage - Used uint64 + Capacity int64 + Available int64 // Available to use for sector storage + Used int64 + Reserved int64 } diff --git a/stores/local.go b/stores/local.go index ac63ae0dd..a21909d69 100644 --- a/stores/local.go +++ b/stores/local.go @@ -67,6 +67,25 @@ type Local struct { type path struct { local string // absolute local path + + reserved int64 + reservations map[abi.SectorID]SectorFileType +} + +type statFn func(path string) (FsStat, error) +func (p *path) stat(st statFn) (FsStat, error) { + stat, err := st(p.local) + if err != nil { + return FsStat{}, err + } + + stat.Reserved = p.reserved + stat.Available -= p.reserved + if stat.Available < 0 { + stat.Available = 0 + } + + return stat, err } func NewLocal(ctx context.Context, ls LocalStorage, index SectorIndex, urls []string) (*Local, error) { @@ -98,9 +117,12 @@ func (st *Local) OpenPath(ctx context.Context, p string) error { out := &path{ local: p, + + reserved: 0, + reservations: map[abi.SectorID]SectorFileType{}, } - fst, err := st.localStorage.Stat(p) + fst, err := out.stat(st.localStorage.Stat) if err != nil { return err } @@ -179,7 +201,7 @@ func (st *Local) reportHealth(ctx context.Context) { toReport := map[ID]HealthReport{} for id, p := range st.paths { - stat, err := st.localStorage.Stat(p.local) + stat, err := p.stat(st.localStorage.Stat) toReport[id] = HealthReport{ Stat: stat, @@ -197,6 +219,61 @@ func (st *Local) reportHealth(ctx context.Context) { } } +func (st *Local) Reserve(ctx context.Context, sid abi.SectorID, spt abi.RegisteredSealProof, ft SectorFileType, storageIDs SectorPaths, overheadTab map[SectorFileType]int) (func(), error) { + ssize, err := spt.SectorSize() + if err != nil { + return nil, xerrors.Errorf("getting sector size: %w", err) + } + + st.localLk.Lock() + + done := func(){} + deferredDone := func() { done() } + defer func() { + st.localLk.Unlock() + deferredDone() + }() + + for _, fileType := range PathTypes { + if fileType&ft == 0 { + continue + } + + id := ID(PathByType(storageIDs, fileType)) + + p, ok := st.paths[id] + if !ok { + return nil, errPathNotFound + } + + stat, err := p.stat(st.localStorage.Stat) + if err != nil { + return nil, err + } + + overhead := int64(overheadTab[fileType]) * int64(ssize) / FSOverheadDen + + if stat.Available < overhead { + return nil, xerrors.Errorf("can't reserve %d bytes in '%s' (id:%s), only %d available", overhead, p.local, id, stat.Available) + } + + p.reserved += overhead + + prevDone := done + done = func() { + prevDone() + + st.localLk.Lock() + defer st.localLk.Unlock() + + p.reserved -= overhead + } + } + + deferredDone = func() {} + return done, nil +} + func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, spt abi.RegisteredSealProof, existing SectorFileType, allocate SectorFileType, pathType PathType, op AcquireMode) (SectorPaths, SectorPaths, error) { if existing|allocate != existing^allocate { return SectorPaths{}, SectorPaths{}, xerrors.New("can't both find and allocate a sector") @@ -463,7 +540,7 @@ func (st *Local) FsStat(ctx context.Context, id ID) (FsStat, error) { return FsStat{}, errPathNotFound } - return st.localStorage.Stat(p.local) + return p.stat(st.localStorage.Stat) } var _ Store = &Local{} From 7279a80dfafdce363883434002431d98e8cdfeee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 6 Jul 2020 18:56:46 +0200 Subject: [PATCH 07/16] localstorage: don't double count reserved storage --- stores/local.go | 48 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/stores/local.go b/stores/local.go index a21909d69..4e91748f8 100644 --- a/stores/local.go +++ b/stores/local.go @@ -49,6 +49,7 @@ type LocalStorage interface { SetStorage(func(*StorageConfig)) error Stat(path string) (FsStat, error) + DiskUsage(path string) (int64, error) // returns real disk usage for a file/directory } const MetaFile = "sectorstore.json" @@ -72,15 +73,36 @@ type path struct { reservations map[abi.SectorID]SectorFileType } -type statFn func(path string) (FsStat, error) -func (p *path) stat(st statFn) (FsStat, error) { - stat, err := st(p.local) +func (p *path) stat(ls LocalStorage) (FsStat, error) { + stat, err := ls.Stat(p.local) if err != nil { return FsStat{}, err } stat.Reserved = p.reserved - stat.Available -= p.reserved + + for id, ft := range p.reservations { + for _, fileType := range PathTypes { + if fileType&ft == 0 { + continue + } + + used, err := ls.DiskUsage(p.sectorPath(id, fileType)) + if err != nil { + log.Errorf("getting disk usage of '%s': %+v", p.sectorPath(id, fileType), err) + continue + } + + stat.Reserved -= used + } + } + + if stat.Reserved < 0 { + log.Warnf("negative reserved storage: p.reserved=%d, reserved: %d", p.reserved, stat.Reserved) + stat.Reserved = 0 + } + + stat.Available -= stat.Reserved if stat.Available < 0 { stat.Available = 0 } @@ -88,6 +110,10 @@ func (p *path) stat(st statFn) (FsStat, error) { return stat, err } +func (p *path) sectorPath(sid abi.SectorID, fileType SectorFileType) string { + return filepath.Join(p.local, fileType.String(), SectorName(sid)) +} + func NewLocal(ctx context.Context, ls LocalStorage, index SectorIndex, urls []string) (*Local, error) { l := &Local{ localStorage: ls, @@ -122,7 +148,7 @@ func (st *Local) OpenPath(ctx context.Context, p string) error { reservations: map[abi.SectorID]SectorFileType{}, } - fst, err := out.stat(st.localStorage.Stat) + fst, err := out.stat(st.localStorage) if err != nil { return err } @@ -201,7 +227,7 @@ func (st *Local) reportHealth(ctx context.Context) { toReport := map[ID]HealthReport{} for id, p := range st.paths { - stat, err := p.stat(st.localStorage.Stat) + stat, err := p.stat(st.localStorage) toReport[id] = HealthReport{ Stat: stat, @@ -246,7 +272,7 @@ func (st *Local) Reserve(ctx context.Context, sid abi.SectorID, spt abi.Register return nil, errPathNotFound } - stat, err := p.stat(st.localStorage.Stat) + stat, err := p.stat(st.localStorage) if err != nil { return nil, err } @@ -306,7 +332,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, spt abi.Re continue } - spath := filepath.Join(p.local, fileType.String(), SectorName(sid)) + spath := p.sectorPath(sid, fileType) SetPathByType(&out, fileType, spath) SetPathByType(&storageIDs, fileType, string(info.ID)) @@ -348,7 +374,7 @@ func (st *Local) AcquireSector(ctx context.Context, sid abi.SectorID, spt abi.Re // TODO: Check free space - best = filepath.Join(p.local, fileType.String(), SectorName(sid)) + best = p.sectorPath(sid, fileType) bestID = si.ID } @@ -464,7 +490,7 @@ func (st *Local) removeSector(ctx context.Context, sid abi.SectorID, typ SectorF return xerrors.Errorf("dropping sector from index: %w", err) } - spath := filepath.Join(p.local, typ.String(), SectorName(sid)) + spath := p.sectorPath(sid, typ) log.Infof("remove %s", spath) if err := os.RemoveAll(spath); err != nil { @@ -540,7 +566,7 @@ func (st *Local) FsStat(ctx context.Context, id ID) (FsStat, error) { return FsStat{}, errPathNotFound } - return p.stat(st.localStorage.Stat) + return p.stat(st.localStorage) } var _ Store = &Local{} From 63c62c49cecb5aa8dec5c4f51368e1676382d86c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 6 Jul 2020 19:19:13 +0200 Subject: [PATCH 08/16] Fix tests --- manager_test.go | 4 ++++ stores/local_test.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/manager_test.go b/manager_test.go index ae318b487..b9198a2b3 100644 --- a/manager_test.go +++ b/manager_test.go @@ -24,6 +24,10 @@ import ( type testStorage stores.StorageConfig +func (t testStorage) DiskUsage(path string) (int64, error) { + return 1, nil // close enough +} + func newTestStorage(t *testing.T) *testStorage { tp, err := ioutil.TempDir(os.TempDir(), "sector-storage-test-") require.NoError(t, err) diff --git a/stores/local_test.go b/stores/local_test.go index 8e654d725..e748d061b 100644 --- a/stores/local_test.go +++ b/stores/local_test.go @@ -19,6 +19,10 @@ type TestingLocalStorage struct { c StorageConfig } +func (t *TestingLocalStorage) DiskUsage(path string) (int64, error) { + return 1, nil +} + func (t *TestingLocalStorage) GetStorage() (StorageConfig, error) { return t.c, nil } From 63ba9bd01836bcf2ed10242f3562bd17c38e2438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 6 Jul 2020 19:19:24 +0200 Subject: [PATCH 09/16] gofmt --- stores/local.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stores/local.go b/stores/local.go index 4e91748f8..cf52540ad 100644 --- a/stores/local.go +++ b/stores/local.go @@ -69,7 +69,7 @@ type Local struct { type path struct { local string // absolute local path - reserved int64 + reserved int64 reservations map[abi.SectorID]SectorFileType } @@ -144,7 +144,7 @@ func (st *Local) OpenPath(ctx context.Context, p string) error { out := &path{ local: p, - reserved: 0, + reserved: 0, reservations: map[abi.SectorID]SectorFileType{}, } @@ -253,7 +253,7 @@ func (st *Local) Reserve(ctx context.Context, sid abi.SectorID, spt abi.Register st.localLk.Lock() - done := func(){} + done := func() {} deferredDone := func() { done() } defer func() { st.localLk.Unlock() From 0bc41d562dd3ba0a824e825dd9ea79939742ca11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 8 Jul 2020 16:58:09 +0200 Subject: [PATCH 10/16] Move statfs to fsutil --- ffiwrapper/sealer_cgo.go | 3 +-- fsutil/dealloc_other.go | 1 - fsutil/statfs.go | 7 +++++++ fsutil/statfs_unix.go | 19 +++++++++++++++++++ fsutil/statfs_windows.go | 28 ++++++++++++++++++++++++++++ manager.go | 3 ++- manager_test.go | 5 +++-- stores/index.go | 9 +++++---- stores/interface.go | 26 ++------------------------ stores/local.go | 11 ++++++----- stores/local_test.go | 6 +++--- stores/remote.go | 25 +++++++++++++------------ 12 files changed, 89 insertions(+), 54 deletions(-) create mode 100644 fsutil/statfs.go create mode 100644 fsutil/statfs_unix.go create mode 100644 fsutil/statfs_windows.go diff --git a/ffiwrapper/sealer_cgo.go b/ffiwrapper/sealer_cgo.go index d3abe1063..c766f5555 100644 --- a/ffiwrapper/sealer_cgo.go +++ b/ffiwrapper/sealer_cgo.go @@ -521,8 +521,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector abi.SectorID, keepU } } - - paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, 0, false) + paths, done, err := sb.sectors.AcquireSector(ctx, sector, stores.FTUnsealed, 0, stores.PathStorage) if err != nil { return xerrors.Errorf("acquiring sector cache path: %w", err) } diff --git a/fsutil/dealloc_other.go b/fsutil/dealloc_other.go index 721116af1..3ae8973ff 100644 --- a/fsutil/dealloc_other.go +++ b/fsutil/dealloc_other.go @@ -10,7 +10,6 @@ import ( var log = logging.Logger("fsutil") - func Deallocate(file *os.File, offset int64, length int64) error { log.Warnf("deallocating space not supported") diff --git a/fsutil/statfs.go b/fsutil/statfs.go new file mode 100644 index 000000000..2a00ccb9a --- /dev/null +++ b/fsutil/statfs.go @@ -0,0 +1,7 @@ +package fsutil + +type FsStat struct { + Capacity int64 + Available int64 // Available to use for sector storage + Reserved int64 +} diff --git a/fsutil/statfs_unix.go b/fsutil/statfs_unix.go new file mode 100644 index 000000000..3e69d5a8e --- /dev/null +++ b/fsutil/statfs_unix.go @@ -0,0 +1,19 @@ +package fsutil + +import ( + "syscall" + + "golang.org/x/xerrors" +) + +func Statfs(path string) (FsStat, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + return FsStat{}, xerrors.Errorf("statfs: %w", err) + } + + return FsStat{ + Capacity: int64(stat.Blocks) * stat.Bsize, + Available: int64(stat.Bavail) * stat.Bsize, + }, nil +} diff --git a/fsutil/statfs_windows.go b/fsutil/statfs_windows.go new file mode 100644 index 000000000..d78565182 --- /dev/null +++ b/fsutil/statfs_windows.go @@ -0,0 +1,28 @@ +package fsutil + +import ( + "syscall" + "unsafe" +) + +func Statfs(volumePath string) (FsStat, error) { + // From https://github.com/ricochet2200/go-disk-usage/blob/master/du/diskusage_windows.go + + h := syscall.MustLoadDLL("kernel32.dll") + c := h.MustFindProc("GetDiskFreeSpaceExW") + + var freeBytes int64 + var totalBytes int64 + var availBytes int64 + + c.Call( + uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(volumePath))), + uintptr(unsafe.Pointer(&freeBytes)), + uintptr(unsafe.Pointer(&totalBytes)), + uintptr(unsafe.Pointer(&availBytes))) + + return FsStat{ + Capacity: totalBytes, + Available: availBytes, + }, nil +} diff --git a/manager.go b/manager.go index a7053c102..0c18645ac 100644 --- a/manager.go +++ b/manager.go @@ -3,6 +3,7 @@ package sectorstorage import ( "context" "errors" + "github.com/filecoin-project/sector-storage/fsutil" "io" "net/http" @@ -491,7 +492,7 @@ func (m *Manager) StorageLocal(ctx context.Context) (map[stores.ID]string, error return out, nil } -func (m *Manager) FsStat(ctx context.Context, id stores.ID) (stores.FsStat, error) { +func (m *Manager) FsStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error) { return m.storage.FsStat(ctx, id) } diff --git a/manager_test.go b/manager_test.go index b9198a2b3..8539f8918 100644 --- a/manager_test.go +++ b/manager_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/filecoin-project/sector-storage/fsutil" "github.com/filecoin-project/sector-storage/sealtasks" logging "github.com/ipfs/go-log" "io/ioutil" @@ -69,8 +70,8 @@ func (t *testStorage) SetStorage(f func(*stores.StorageConfig)) error { return nil } -func (t *testStorage) Stat(path string) (stores.FsStat, error) { - return stores.Stat(path) +func (t *testStorage) Stat(path string) (fsutil.FsStat, error) { + return fsutil.Statfs(path) } var _ stores.LocalStorage = &testStorage{} diff --git a/stores/index.go b/stores/index.go index e48ae02bb..c85dc125e 100644 --- a/stores/index.go +++ b/stores/index.go @@ -2,6 +2,7 @@ package stores import ( "context" + "github.com/filecoin-project/sector-storage/fsutil" "net/url" gopath "path" "sort" @@ -34,7 +35,7 @@ type StorageInfo struct { } type HealthReport struct { - Stat FsStat + Stat fsutil.FsStat Err error } @@ -50,7 +51,7 @@ type SectorStorageInfo struct { } type SectorIndex interface { // part of storage-miner api - StorageAttach(context.Context, StorageInfo, FsStat) error + StorageAttach(context.Context, StorageInfo, fsutil.FsStat) error StorageInfo(context.Context, ID) (StorageInfo, error) StorageReportHealth(context.Context, ID, HealthReport) error @@ -77,7 +78,7 @@ type declMeta struct { type storageEntry struct { info *StorageInfo - fsi FsStat + fsi fsutil.FsStat lastHeartbeat time.Time heartbeatErr error @@ -130,7 +131,7 @@ func (i *Index) StorageList(ctx context.Context) (map[ID][]Decl, error) { return out, nil } -func (i *Index) StorageAttach(ctx context.Context, si StorageInfo, st FsStat) error { +func (i *Index) StorageAttach(ctx context.Context, si StorageInfo, st fsutil.FsStat) error { i.lk.Lock() defer i.lk.Unlock() diff --git a/stores/interface.go b/stores/interface.go index 6fd4a7ad7..836705f40 100644 --- a/stores/interface.go +++ b/stores/interface.go @@ -2,10 +2,7 @@ package stores import ( "context" - "syscall" - - "golang.org/x/xerrors" - + "github.com/filecoin-project/sector-storage/fsutil" "github.com/filecoin-project/specs-actors/actors/abi" ) @@ -34,24 +31,5 @@ type Store interface { // move sectors into storage MoveStorage(ctx context.Context, s abi.SectorID, spt abi.RegisteredSealProof, types SectorFileType) error - FsStat(ctx context.Context, id ID) (FsStat, error) -} - -func Stat(path string) (FsStat, error) { - var stat syscall.Statfs_t - if err := syscall.Statfs(path, &stat); err != nil { - return FsStat{}, xerrors.Errorf("statfs: %w", err) - } - - return FsStat{ - Capacity: int64(stat.Blocks) * stat.Bsize, - Available: int64(stat.Bavail) * stat.Bsize, - }, nil -} - -type FsStat struct { - Capacity int64 - Available int64 // Available to use for sector storage - Used int64 - Reserved int64 + FsStat(ctx context.Context, id ID) (fsutil.FsStat, error) } diff --git a/stores/local.go b/stores/local.go index cf52540ad..cbc9dbae1 100644 --- a/stores/local.go +++ b/stores/local.go @@ -3,6 +3,7 @@ package stores import ( "context" "encoding/json" + "github.com/filecoin-project/sector-storage/fsutil" "io/ioutil" "math/bits" "math/rand" @@ -48,7 +49,7 @@ type LocalStorage interface { GetStorage() (StorageConfig, error) SetStorage(func(*StorageConfig)) error - Stat(path string) (FsStat, error) + Stat(path string) (fsutil.FsStat, error) DiskUsage(path string) (int64, error) // returns real disk usage for a file/directory } @@ -73,10 +74,10 @@ type path struct { reservations map[abi.SectorID]SectorFileType } -func (p *path) stat(ls LocalStorage) (FsStat, error) { +func (p *path) stat(ls LocalStorage) (fsutil.FsStat, error) { stat, err := ls.Stat(p.local) if err != nil { - return FsStat{}, err + return fsutil.FsStat{}, err } stat.Reserved = p.reserved @@ -557,13 +558,13 @@ func (st *Local) MoveStorage(ctx context.Context, s abi.SectorID, spt abi.Regist var errPathNotFound = xerrors.Errorf("fsstat: path not found") -func (st *Local) FsStat(ctx context.Context, id ID) (FsStat, error) { +func (st *Local) FsStat(ctx context.Context, id ID) (fsutil.FsStat, error) { st.localLk.RLock() defer st.localLk.RUnlock() p, ok := st.paths[id] if !ok { - return FsStat{}, errPathNotFound + return fsutil.FsStat{}, errPathNotFound } return p.stat(st.localStorage) diff --git a/stores/local_test.go b/stores/local_test.go index e748d061b..56ac7c020 100644 --- a/stores/local_test.go +++ b/stores/local_test.go @@ -3,6 +3,7 @@ package stores import ( "context" "encoding/json" + "github.com/filecoin-project/sector-storage/fsutil" "github.com/google/uuid" "io/ioutil" "os" @@ -32,11 +33,10 @@ func (t *TestingLocalStorage) SetStorage(f func(*StorageConfig)) error { return nil } -func (t *TestingLocalStorage) Stat(path string) (FsStat, error) { - return FsStat{ +func (t *TestingLocalStorage) Stat(path string) (fsutil.FsStat, error) { + return fsutil.FsStat{ Capacity: pathSize, Available: pathSize, - Used: 0, }, nil } diff --git a/stores/remote.go b/stores/remote.go index 30fe3abf9..c78f026f4 100644 --- a/stores/remote.go +++ b/stores/remote.go @@ -3,6 +3,7 @@ package stores import ( "context" "encoding/json" + "github.com/filecoin-project/sector-storage/fsutil" "io/ioutil" "math/bits" "mime" @@ -270,7 +271,7 @@ func (r *Remote) deleteFromRemote(ctx context.Context, url string) error { return nil } -func (r *Remote) FsStat(ctx context.Context, id ID) (FsStat, error) { +func (r *Remote) FsStat(ctx context.Context, id ID) (fsutil.FsStat, error) { st, err := r.local.FsStat(ctx, id) switch err { case nil: @@ -278,53 +279,53 @@ func (r *Remote) FsStat(ctx context.Context, id ID) (FsStat, error) { case errPathNotFound: break default: - return FsStat{}, xerrors.Errorf("local stat: %w", err) + return fsutil.FsStat{}, xerrors.Errorf("local stat: %w", err) } si, err := r.index.StorageInfo(ctx, id) if err != nil { - return FsStat{}, xerrors.Errorf("getting remote storage info: %w", err) + return fsutil.FsStat{}, xerrors.Errorf("getting remote storage info: %w", err) } if len(si.URLs) == 0 { - return FsStat{}, xerrors.Errorf("no known URLs for remote storage %s", id) + return fsutil.FsStat{}, xerrors.Errorf("no known URLs for remote storage %s", id) } rl, err := url.Parse(si.URLs[0]) if err != nil { - return FsStat{}, xerrors.Errorf("failed to parse url: %w", err) + return fsutil.FsStat{}, xerrors.Errorf("failed to parse url: %w", err) } rl.Path = gopath.Join(rl.Path, "stat", string(id)) req, err := http.NewRequest("GET", rl.String(), nil) if err != nil { - return FsStat{}, xerrors.Errorf("request: %w", err) + return fsutil.FsStat{}, xerrors.Errorf("request: %w", err) } req.Header = r.auth req = req.WithContext(ctx) resp, err := http.DefaultClient.Do(req) if err != nil { - return FsStat{}, xerrors.Errorf("do request: %w", err) + return fsutil.FsStat{}, xerrors.Errorf("do request: %w", err) } switch resp.StatusCode { case 200: break case 404: - return FsStat{}, errPathNotFound + return fsutil.FsStat{}, errPathNotFound case 500: b, err := ioutil.ReadAll(resp.Body) if err != nil { - return FsStat{}, xerrors.Errorf("fsstat: got http 500, then failed to read the error: %w", err) + return fsutil.FsStat{}, xerrors.Errorf("fsstat: got http 500, then failed to read the error: %w", err) } - return FsStat{}, xerrors.Errorf("fsstat: got http 500: %s", string(b)) + return fsutil.FsStat{}, xerrors.Errorf("fsstat: got http 500: %s", string(b)) } - var out FsStat + var out fsutil.FsStat if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return FsStat{}, xerrors.Errorf("decoding fsstat: %w", err) + return fsutil.FsStat{}, xerrors.Errorf("decoding fsstat: %w", err) } defer resp.Body.Close() From c0a242a1eb664dbbc32e6454c26a4890eb95d5dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 8 Jul 2020 17:09:35 +0200 Subject: [PATCH 11/16] fsutil: FileSize util --- fsutil/filesize_unix.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 fsutil/filesize_unix.go diff --git a/fsutil/filesize_unix.go b/fsutil/filesize_unix.go new file mode 100644 index 000000000..e45ccca17 --- /dev/null +++ b/fsutil/filesize_unix.go @@ -0,0 +1,25 @@ +package fsutil + +import ( + "syscall" + + "golang.org/x/xerrors" +) + +type SizeInfo struct { + OnDisk int64 +} + +// FileSize returns bytes used by a file on disk +func FileSize(path string) (SizeInfo, error) { + var stat syscall.Stat_t + if err := syscall.Stat(path, &stat); err != nil { + return SizeInfo{}, xerrors.Errorf("stat: %w", err) + } + + // NOTE: stat.Blocks is in 512B blocks, NOT in stat.Blksize + // See https://www.gnu.org/software/libc/manual/html_node/Attribute-Meanings.html + return SizeInfo{ + stat.Blocks * 512, + }, nil +} \ No newline at end of file From 56570a22005f77a5eb744109a3bd845c3a5def0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 8 Jul 2020 17:39:58 +0200 Subject: [PATCH 12/16] mock: Implemet ReleaseUnsealed correctly --- mock/mock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mock/mock.go b/mock/mock.go index cbc3a1f99..7c9ed57f0 100644 --- a/mock/mock.go +++ b/mock/mock.go @@ -320,7 +320,7 @@ func (mgr *SectorMgr) FinalizeSector(context.Context, abi.SectorID, []storage.Ra } func (mgr *SectorMgr) ReleaseUnsealed(ctx context.Context, sector abi.SectorID, safeToFree []storage.Range) error { - panic("implement me") + return nil } func (mgr *SectorMgr) Remove(ctx context.Context, sector abi.SectorID) error { From 9af64c9b217e6b118ec29a669be7fb455bfe54e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 8 Jul 2020 19:51:26 +0200 Subject: [PATCH 13/16] ffiwrapper: Fix UnsealPiece --- ffiwrapper/sealer_cgo.go | 5 ++++- fsutil/filesize_unix.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ffiwrapper/sealer_cgo.go b/ffiwrapper/sealer_cgo.go index c766f5555..88218921c 100644 --- a/ffiwrapper/sealer_cgo.go +++ b/ffiwrapper/sealer_cgo.go @@ -254,7 +254,10 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector abi.SectorID, offset s defer sealed.Close() var at, nextat abi.PaddedPieceSize - for { + first := true + for first || toUnseal.HasNext() { + first = false + piece, err := toUnseal.NextRun() if err != nil { return xerrors.Errorf("getting next range to unseal: %w", err) diff --git a/fsutil/filesize_unix.go b/fsutil/filesize_unix.go index e45ccca17..d596e4be7 100644 --- a/fsutil/filesize_unix.go +++ b/fsutil/filesize_unix.go @@ -22,4 +22,4 @@ func FileSize(path string) (SizeInfo, error) { return SizeInfo{ stat.Blocks * 512, }, nil -} \ No newline at end of file +} From d1a18c15e6655b63d2ca171d6ae6e3e71d76278a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 9 Jul 2020 19:29:05 +0200 Subject: [PATCH 14/16] Fix build on osx --- fsutil/dealloc_other.go | 2 +- fsutil/filesize_unix.go | 2 +- fsutil/statfs_unix.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fsutil/dealloc_other.go b/fsutil/dealloc_other.go index 3ae8973ff..4f8347951 100644 --- a/fsutil/dealloc_other.go +++ b/fsutil/dealloc_other.go @@ -13,5 +13,5 @@ var log = logging.Logger("fsutil") func Deallocate(file *os.File, offset int64, length int64) error { log.Warnf("deallocating space not supported") - return err + return nil } diff --git a/fsutil/filesize_unix.go b/fsutil/filesize_unix.go index d596e4be7..41b62daf6 100644 --- a/fsutil/filesize_unix.go +++ b/fsutil/filesize_unix.go @@ -20,6 +20,6 @@ func FileSize(path string) (SizeInfo, error) { // NOTE: stat.Blocks is in 512B blocks, NOT in stat.Blksize // See https://www.gnu.org/software/libc/manual/html_node/Attribute-Meanings.html return SizeInfo{ - stat.Blocks * 512, + int64(stat.Blocks) * 512, }, nil } diff --git a/fsutil/statfs_unix.go b/fsutil/statfs_unix.go index 3e69d5a8e..7fcb8af37 100644 --- a/fsutil/statfs_unix.go +++ b/fsutil/statfs_unix.go @@ -13,7 +13,7 @@ func Statfs(path string) (FsStat, error) { } return FsStat{ - Capacity: int64(stat.Blocks) * stat.Bsize, - Available: int64(stat.Bavail) * stat.Bsize, + Capacity: int64(stat.Blocks) * int64(stat.Bsize), + Available: int64(stat.Bavail) * int64(stat.Bsize), }, nil } From 1d67dcfa3c156dc04ca09c8a4f8efe70522f72ef Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sat, 11 Jul 2020 21:30:16 -0400 Subject: [PATCH 15/16] extract GetRequiredPadding --- ffiwrapper/sealer_cgo.go | 78 +++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/ffiwrapper/sealer_cgo.go b/ffiwrapper/sealer_cgo.go index 88218921c..5c6e40ef9 100644 --- a/ffiwrapper/sealer_cgo.go +++ b/ffiwrapper/sealer_cgo.go @@ -598,45 +598,65 @@ func GeneratePieceCIDFromFile(proofType abi.RegisteredSealProof, piece io.Reader return pieceCID, werr() } -func GenerateUnsealedCID(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) { - allPieces := make([]abi.PieceInfo, 0, len(pieces)) +func GetRequiredPadding(oldLength abi.PaddedPieceSize, newPieceLength abi.PaddedPieceSize) ([]abi.PaddedPieceSize, abi.PaddedPieceSize) { + + padPieces := make([]abi.PaddedPieceSize, 0) + + toFill := uint64(-oldLength % newPieceLength) + + n := bits.OnesCount64(toFill) var sum abi.PaddedPieceSize + for i := 0; i < n; i++ { + next := bits.TrailingZeros64(toFill) + psize := uint64(1) << uint(next) + toFill ^= psize - padTo := func(s abi.PaddedPieceSize, trailing bool) { - // pad remaining space with 0 CommPs - toFill := uint64(-sum % s) - if trailing && sum == 0 { - toFill = uint64(s) - } - - n := bits.OnesCount64(toFill) - for i := 0; i < n; i++ { - next := bits.TrailingZeros64(toFill) - psize := uint64(1) << uint(next) - toFill ^= psize - - padded := abi.PaddedPieceSize(psize) - allPieces = append(allPieces, abi.PieceInfo{ - Size: padded, - PieceCID: zerocomm.ZeroPieceCommitment(padded.Unpadded()), - }) - sum += padded - } + padded := abi.PaddedPieceSize(psize) + padPieces = append(padPieces, padded) + sum += padded } - for _, p := range pieces { - padTo(p.Size, false) - - allPieces = append(allPieces, p) - sum += p.Size - } + return padPieces, sum +} +func GenerateUnsealedCID(proofType abi.RegisteredSealProof, pieces []abi.PieceInfo) (cid.Cid, error) { ssize, err := proofType.SectorSize() if err != nil { return cid.Undef, err } - padTo(abi.PaddedPieceSize(ssize), true) + pssize := abi.PaddedPieceSize(ssize) + allPieces := make([]abi.PieceInfo, 0, len(pieces)) + if len(pieces) == 0 { + allPieces = append(allPieces, abi.PieceInfo{ + Size: pssize, + PieceCID: zerocomm.ZeroPieceCommitment(pssize.Unpadded()), + }) + } else { + var sum abi.PaddedPieceSize + + padTo := func(pads []abi.PaddedPieceSize) { + for _, p := range pads { + allPieces = append(allPieces, abi.PieceInfo{ + Size: p, + PieceCID: zerocomm.ZeroPieceCommitment(p.Unpadded()), + }) + + sum += p + } + } + + for _, p := range pieces { + ps, _ := GetRequiredPadding(sum, p.Size) + padTo(ps) + + allPieces = append(allPieces, p) + sum += p.Size + } + + ps, _ := GetRequiredPadding(sum, pssize) + padTo(ps) + } return ffi.GenerateUnsealedCID(proofType, allPieces) } From 4f8015b58a9da612f1a49479d9fa299459da787d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 16 Jul 2020 18:18:22 +0200 Subject: [PATCH 16/16] Correctly turn randomness into fr32 values --- ffiwrapper/verifier_cgo.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ffiwrapper/verifier_cgo.go b/ffiwrapper/verifier_cgo.go index 60d56dddc..1fecf9598 100644 --- a/ffiwrapper/verifier_cgo.go +++ b/ffiwrapper/verifier_cgo.go @@ -15,7 +15,7 @@ import ( ) func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []abi.SectorInfo, randomness abi.PoStRandomness) ([]abi.PoStProof, error) { - randomness[31] = 0 // TODO: Not correct, fixme + randomness[31] &= 0x3f privsectors, skipped, done, err := sb.pubSectorToPriv(ctx, minerID, sectorInfo, nil, abi.RegisteredSealProof.RegisteredWinningPoStProof) // TODO: FAULTS? if err != nil { return nil, err @@ -29,7 +29,7 @@ func (sb *Sealer) GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, } func (sb *Sealer) GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []abi.SectorInfo, randomness abi.PoStRandomness) ([]abi.PoStProof, []abi.SectorID, error) { - randomness[31] = 0 // TODO: Not correct, fixme + randomness[31] &= 0x3f privsectors, skipped, done, err := sb.pubSectorToPriv(ctx, minerID, sectorInfo, nil, abi.RegisteredSealProof.RegisteredWindowPoStProof) if err != nil { return nil, nil, xerrors.Errorf("gathering sector info: %w", err) @@ -98,7 +98,7 @@ func (proofVerifier) VerifySeal(info abi.SealVerifyInfo) (bool, error) { } func (proofVerifier) VerifyWinningPoSt(ctx context.Context, info abi.WinningPoStVerifyInfo) (bool, error) { - info.Randomness[31] = 0 // TODO: Not correct, fixme + info.Randomness[31] &= 0x3f _, span := trace.StartSpan(ctx, "VerifyWinningPoSt") defer span.End() @@ -106,7 +106,7 @@ func (proofVerifier) VerifyWinningPoSt(ctx context.Context, info abi.WinningPoSt } func (proofVerifier) VerifyWindowPoSt(ctx context.Context, info abi.WindowPoStVerifyInfo) (bool, error) { - info.Randomness[31] = 0 // TODO: Not correct, fixme + info.Randomness[31] &= 0x3f _, span := trace.StartSpan(ctx, "VerifyWindowPoSt") defer span.End() @@ -114,6 +114,6 @@ func (proofVerifier) VerifyWindowPoSt(ctx context.Context, info abi.WindowPoStVe } func (proofVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proofType abi.RegisteredPoStProof, minerID abi.ActorID, randomness abi.PoStRandomness, eligibleSectorCount uint64) ([]uint64, error) { - randomness[31] = 0 // TODO: Not correct, fixme + randomness[31] &= 0x3f return ffi.GenerateWinningPoStSectorChallenge(proofType, minerID, randomness, eligibleSectorCount) }