From de07fe8aafe379693f8ba0b1b96a73abed0375fd Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 7 Apr 2020 14:44:33 -0700 Subject: [PATCH 1/7] persist deal schedule to piece data for pre-commit sector expiry --- cbor_gen.go | 379 ++++++++++++++++++++++++++++++++++++++------------ checks.go | 34 +++-- fsm_events.go | 13 +- garbage.go | 20 ++- gen/main.go | 4 +- sealing.go | 17 +-- states.go | 16 +-- types.go | 56 +++++--- types_test.go | 20 ++- 9 files changed, 403 insertions(+), 156 deletions(-) diff --git a/cbor_gen.go b/cbor_gen.go index 08ba98546..37a75a662 100644 --- a/cbor_gen.go +++ b/cbor_gen.go @@ -13,12 +13,125 @@ import ( var _ = xerrors.Errorf -func (t *Piece) MarshalCBOR(w io.Writer) error { +func (t *PieceWithOptionalDealInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{163}); err != nil { + if _, err := w.Write([]byte{162}); err != nil { + return err + } + + // t.Piece (abi.PieceInfo) (struct) + if len("Piece") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Piece\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Piece")))); err != nil { + return err + } + if _, err := w.Write([]byte("Piece")); err != nil { + return err + } + + if err := t.Piece.MarshalCBOR(w); err != nil { + return err + } + + // t.DealInfo (sealing.DealInfo) (struct) + if len("DealInfo") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealInfo\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("DealInfo")))); err != nil { + return err + } + if _, err := w.Write([]byte("DealInfo")); err != nil { + return err + } + + if err := t.DealInfo.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *PieceWithOptionalDealInfo) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("PieceWithOptionalDealInfo: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Piece (abi.PieceInfo) (struct) + case "Piece": + + { + + if err := t.Piece.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.Piece: %w", err) + } + + } + // t.DealInfo (sealing.DealInfo) (struct) + case "DealInfo": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.DealInfo = new(DealInfo) + if err := t.DealInfo.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.DealInfo pointer: %w", err) + } + } + + } + + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) + } + } + + return nil +} +func (t *DealInfo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{162}); err != nil { return err } @@ -34,52 +147,29 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { return err } - if t.DealID == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(*t.DealID))); err != nil { - return err - } - } - - // t.Size (abi.UnpaddedPieceSize) (uint64) - if len("Size") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Size\" was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Size")))); err != nil { - return err - } - if _, err := w.Write([]byte("Size")); err != nil { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { return err } - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { + // t.DealSchedule (sealing.DealSchedule) (struct) + if len("DealSchedule") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DealSchedule\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("DealSchedule")))); err != nil { + return err + } + if _, err := w.Write([]byte("DealSchedule")); err != nil { return err } - // t.CommP (cid.Cid) (struct) - if len("CommP") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"CommP\" was too long") - } - - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("CommP")))); err != nil { + if err := t.DealSchedule.MarshalCBOR(w); err != nil { return err } - if _, err := w.Write([]byte("CommP")); err != nil { - return err - } - - if err := cbg.WriteCid(w, t.CommP); err != nil { - return xerrors.Errorf("failed to write cid field t.CommP: %w", err) - } - return nil } -func (t *Piece) UnmarshalCBOR(r io.Reader) error { +func (t *DealInfo) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) @@ -91,7 +181,7 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { } if extra > cbg.MaxLength { - return fmt.Errorf("Piece: map struct too large (%d)", extra) + return fmt.Errorf("DealInfo: map struct too large (%d)", extra) } var name string @@ -112,33 +202,6 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { // t.DealID (abi.DealID) (uint64) case "DealID": - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - typed := abi.DealID(extra) - t.DealID = &typed - } - - } - // t.Size (abi.UnpaddedPieceSize) (uint64) - case "Size": - { maj, extra, err = cbg.CborReadHeader(br) @@ -148,21 +211,163 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { if maj != cbg.MajUnsignedInt { return fmt.Errorf("wrong type for uint64 field") } - t.Size = abi.UnpaddedPieceSize(extra) + t.DealID = abi.DealID(extra) } - // t.CommP (cid.Cid) (struct) - case "CommP": + // t.DealSchedule (sealing.DealSchedule) (struct) + case "DealSchedule": { - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.CommP: %w", err) + if err := t.DealSchedule.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err) } - t.CommP = c + } + default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) + } + } + + return nil +} +func (t *DealSchedule) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{162}); err != nil { + return err + } + + // t.StartEpoch (abi.ChainEpoch) (int64) + if len("StartEpoch") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"StartEpoch\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("StartEpoch")))); err != nil { + return err + } + if _, err := w.Write([]byte("StartEpoch")); err != nil { + return err + } + + if t.StartEpoch >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.StartEpoch))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.StartEpoch)-1)); err != nil { + return err + } + } + + // t.EndEpoch (abi.ChainEpoch) (int64) + if len("EndEpoch") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"EndEpoch\" was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("EndEpoch")))); err != nil { + return err + } + if _, err := w.Write([]byte("EndEpoch")); err != nil { + return err + } + + if t.EndEpoch >= 0 { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.EndEpoch))); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajNegativeInt, uint64(-t.EndEpoch)-1)); err != nil { + return err + } + } + return nil +} + +func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("DealSchedule: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.StartEpoch (abi.ChainEpoch) (int64) + case "StartEpoch": + { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.StartEpoch = abi.ChainEpoch(extraI) + } + // t.EndEpoch (abi.ChainEpoch) (int64) + case "EndEpoch": + { + maj, extra, err := cbg.CborReadHeader(br) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.EndEpoch = abi.ChainEpoch(extraI) } default: @@ -258,26 +463,26 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } } - // t.Pieces ([]sealing.Piece) (slice) - if len("Pieces") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"Pieces\" was too long") + // t.PiecesWithOptionalDealInfo ([]sealing.PieceWithOptionalDealInfo) (slice) + if len("PiecesWithOptionalDealInfo") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PiecesWithOptionalDealInfo\" was too long") } - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Pieces")))); err != nil { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PiecesWithOptionalDealInfo")))); err != nil { return err } - if _, err := w.Write([]byte("Pieces")); err != nil { + if _, err := w.Write([]byte("PiecesWithOptionalDealInfo")); err != nil { return err } - if len(t.Pieces) > cbg.MaxLength { - return xerrors.Errorf("Slice value in field t.Pieces was too long") + if len(t.PiecesWithOptionalDealInfo) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.PiecesWithOptionalDealInfo was too long") } - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Pieces)))); err != nil { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.PiecesWithOptionalDealInfo)))); err != nil { return err } - for _, v := range t.Pieces { + for _, v := range t.PiecesWithOptionalDealInfo { if err := v.MarshalCBOR(w); err != nil { return err } @@ -692,8 +897,8 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { t.SectorType = abi.RegisteredProof(extraI) } - // t.Pieces ([]sealing.Piece) (slice) - case "Pieces": + // t.PiecesWithOptionalDealInfo ([]sealing.PieceWithOptionalDealInfo) (slice) + case "PiecesWithOptionalDealInfo": maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -701,23 +906,23 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { } if extra > cbg.MaxLength { - return fmt.Errorf("t.Pieces: array too large (%d)", extra) + return fmt.Errorf("t.PiecesWithOptionalDealInfo: array too large (%d)", extra) } if maj != cbg.MajArray { return fmt.Errorf("expected cbor array") } if extra > 0 { - t.Pieces = make([]Piece, extra) + t.PiecesWithOptionalDealInfo = make([]PieceWithOptionalDealInfo, extra) } for i := 0; i < int(extra); i++ { - var v Piece + var v PieceWithOptionalDealInfo if err := v.UnmarshalCBOR(br); err != nil { return err } - t.Pieces[i] = v + t.PiecesWithOptionalDealInfo[i] = v } // t.TicketValue (abi.SealRandomness) (slice) diff --git a/checks.go b/checks.go index 3c18f9f34..fc41e395c 100644 --- a/checks.go +++ b/checks.go @@ -27,40 +27,38 @@ type ErrExpiredTicket struct{ error } type ErrBadSeed struct{ error } type ErrInvalidProof struct{ error } -// checkPieces validates that: -// - Each piece han a corresponding on chain deal -// - Piece commitments match with on chain deals -// - Piece sizes match -// - Deals aren't expired func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error { tok, height, err := api.ChainHead(ctx) if err != nil { return &ErrApi{xerrors.Errorf("getting chain head: %w", err)} } - for i, piece := range si.Pieces { - if piece.DealID == nil { - exp := zerocomm.ZeroPieceCommitment(piece.Size) - if piece.CommP != exp { - return &ErrInvalidPiece{xerrors.Errorf("deal %d piece %d had non-zero CommP %+v", piece.DealID, i, piece.CommP)} + for i, pdi := range si.PiecesWithOptionalDealInfo { + // if no deal is associated with the piece, ensure that we added it as + // filler (i.e. ensure that it has a zero PieceCID) + if pdi.DealInfo == nil { + exp := zerocomm.ZeroPieceCommitment(pdi.Piece.Size.Unpadded()) + if !pdi.Piece.PieceCID.Equals(exp) { + return &ErrInvalidPiece{xerrors.Errorf("sector %d piece %d had non-zero PieceCID %+v", si.SectorNumber, i, pdi.Piece.PieceCID)} } continue } - proposal, _, err := api.StateMarketStorageDeal(ctx, *piece.DealID, tok) + + proposal, _, err := api.StateMarketStorageDeal(ctx, pdi.DealInfo.DealID, tok) if err != nil { - return &ErrApi{xerrors.Errorf("getting deal %d for piece %d: %w", piece.DealID, i, err)} + return &ErrApi{xerrors.Errorf("getting deal %d for piece %d: %w", pdi.DealInfo.DealID, i, err)} } - if proposal.PieceCID != piece.CommP { - return &ErrInvalidDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers deal %d with wrong CommP: %x != %x", i, len(si.Pieces), si.SectorNumber, piece.DealID, piece.CommP, proposal.PieceCID)} + if proposal.PieceCID != pdi.Piece.PieceCID { + return &ErrInvalidDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers deal %d with wrong PieceCID: %x != %x", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.PieceCID, proposal.PieceCID)} } - if piece.Size != proposal.PieceSize.Unpadded() { - return &ErrInvalidDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers deal %d with different size: %d != %d", i, len(si.Pieces), si.SectorNumber, piece.DealID, piece.Size, proposal.PieceSize)} + if pdi.Piece.Size != proposal.PieceSize { + return &ErrInvalidDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers deal %d with different size: %d != %d", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.Size, proposal.PieceSize)} } if height >= proposal.StartEpoch { - return &ErrExpiredDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers expired deal %d - should start at %d, head %d", i, len(si.Pieces), si.SectorNumber, piece.DealID, proposal.StartEpoch, height)} + return &ErrExpiredDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers expired deal %d - should start at %d, head %d", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, proposal.StartEpoch, height)} } } @@ -75,7 +73,7 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, a return &ErrApi{xerrors.Errorf("getting chain head: %w", err)} } - commD, err := api.StateComputeDataCommitment(ctx, maddr, si.SectorType, si.deals(), tok) + commD, err := api.StateComputeDataCommitment(ctx, maddr, si.SectorType, si.dealIDs(), tok) if err != nil { return &ErrApi{xerrors.Errorf("calling StateComputeDataCommitment: %w", err)} } diff --git a/fsm_events.go b/fsm_events.go index 7316d16a1..d7d79c42d 100644 --- a/fsm_events.go +++ b/fsm_events.go @@ -50,19 +50,24 @@ func (evt SectorForceState) applyGlobal(state *SectorInfo) bool { type SectorStart struct { ID abi.SectorNumber SectorType abi.RegisteredProof - Pieces []Piece + Pieces []PieceWithOptionalDealInfo } func (evt SectorStart) apply(state *SectorInfo) { state.SectorNumber = evt.ID - state.Pieces = evt.Pieces + state.PiecesWithOptionalDealInfo = evt.Pieces state.SectorType = evt.SectorType } -type SectorPacked struct{ Pieces []Piece } +type SectorPacked struct{ FillerPieces []abi.PieceInfo } func (evt SectorPacked) apply(state *SectorInfo) { - state.Pieces = append(state.Pieces, evt.Pieces...) + for idx := range evt.FillerPieces { + state.PiecesWithOptionalDealInfo = append(state.PiecesWithOptionalDealInfo, PieceWithOptionalDealInfo{ + Piece: evt.FillerPieces[idx], + DealInfo: nil, // filler pieces don't have deals associated with them + }) + } } type SectorPackingFailed struct{ error } diff --git a/garbage.go b/garbage.go index e65d9419a..638f7500d 100644 --- a/garbage.go +++ b/garbage.go @@ -16,14 +16,14 @@ func (m *Sealing) pledgeReader(size abi.UnpaddedPieceSize) io.Reader { return io.LimitReader(&nr.Reader{}, int64(size)) } -func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorID, existingPieceSizes []abi.UnpaddedPieceSize, sizes ...abi.UnpaddedPieceSize) ([]Piece, error) { +func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorID, existingPieceSizes []abi.UnpaddedPieceSize, sizes ...abi.UnpaddedPieceSize) ([]abi.PieceInfo, error) { if len(sizes) == 0 { return nil, nil } log.Infof("Pledge %d, contains %+v", sectorID, existingPieceSizes) - out := make([]Piece, len(sizes)) + out := make([]abi.PieceInfo, len(sizes)) for i, size := range sizes { ppi, err := m.sealer.AddPiece(ctx, sectorID, existingPieceSizes, size, m.pledgeReader(size)) if err != nil { @@ -32,9 +32,9 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorID, exist existingPieceSizes = append(existingPieceSizes, size) - out[i] = Piece{ - Size: ppi.Size.Unpadded(), - CommP: ppi.PieceCID, + out[i] = abi.PieceInfo{ + Size: ppi.Size, + PieceCID: ppi.PieceCID, } } @@ -72,7 +72,15 @@ func (m *Sealing) PledgeSector() error { return } - if err := m.newSector(sid, rt, pieces); err != nil { + pdis := make([]PieceWithOptionalDealInfo, len(pieces)) + for idx := range pdis { + pdis[idx] = PieceWithOptionalDealInfo{ + Piece: pieces[idx], + DealInfo: nil, + } + } + + if err := m.newSector(sid, rt, pdis); err != nil { log.Errorf("%+v", err) return } diff --git a/gen/main.go b/gen/main.go index 7b9ade11f..e9a68425a 100644 --- a/gen/main.go +++ b/gen/main.go @@ -11,7 +11,9 @@ import ( func main() { err := gen.WriteMapEncodersToFile("./cbor_gen.go", "sealing", - sealing.Piece{}, + sealing.PieceWithOptionalDealInfo{}, + sealing.DealInfo{}, + sealing.DealSchedule{}, sealing.SectorInfo{}, sealing.Log{}, ) diff --git a/sealing.go b/sealing.go index f649e62ab..aee3321b5 100644 --- a/sealing.go +++ b/sealing.go @@ -101,8 +101,8 @@ func (m *Sealing) AllocatePiece(size abi.UnpaddedPieceSize) (sectorID abi.Sector return sid, 0, nil } -func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, sectorID abi.SectorNumber, dealID abi.DealID) error { - log.Infof("Seal piece for deal %d", dealID) +func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, sectorID abi.SectorNumber, pdi PieceWithDealInfo) error { + log.Infof("Seal piece for deal %d", pdi.DealInfo.DealID) ppi, err := m.sealer.AddPiece(ctx, m.minerSector(sectorID), []abi.UnpaddedPieceSize{}, size, r) if err != nil { @@ -114,17 +114,18 @@ func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r i return xerrors.Errorf("bad sector size: %w", err) } - return m.newSector(sectorID, rt, []Piece{ + return m.newSector(sectorID, rt, []PieceWithOptionalDealInfo{ { - DealID: &dealID, - - Size: ppi.Size.Unpadded(), - CommP: ppi.PieceCID, + Piece: ppi, + DealInfo: &pdi.DealInfo, }, }) } -func (m *Sealing) newSector(sid abi.SectorNumber, rt abi.RegisteredProof, pieces []Piece) error { +// newSector accepts a slice of pieces which will have a deal associated with +// them (in the event of a storage deal) or no deal (in the event of sealing +// garbage data) +func (m *Sealing) newSector(sid abi.SectorNumber, rt abi.RegisteredProof, pieces []PieceWithOptionalDealInfo) error { log.Infof("Start sealing %d", sid) return m.sectors.Send(uint64(sid), SectorStart{ ID: sid, diff --git a/states.go b/states.go index 222a365a0..13946f3b1 100644 --- a/states.go +++ b/states.go @@ -19,8 +19,8 @@ func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) err log.Infow("performing filling up rest of the sector...", "sector", sector.SectorNumber) var allocated abi.UnpaddedPieceSize - for _, piece := range sector.Pieces { - allocated += piece.Size + for _, piece := range sector.PiecesWithOptionalDealInfo { + allocated += piece.Piece.Size.Unpadded() } ubytes := abi.PaddedPieceSize(m.sealer.SectorSize()).Unpadded() @@ -38,12 +38,12 @@ func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) err log.Warnf("Creating %d filler pieces for sector %d", len(fillerSizes), sector.SectorNumber) } - pieces, err := m.pledgeSector(ctx.Context(), m.minerSector(sector.SectorNumber), sector.existingPieces(), fillerSizes...) + fillerPieces, err := m.pledgeSector(ctx.Context(), m.minerSector(sector.SectorNumber), sector.existingPieceSizes(), fillerSizes...) if err != nil { return xerrors.Errorf("filling up the sector (%v): %w", fillerSizes, err) } - return ctx.Send(SectorPacked{Pieces: pieces}) + return ctx.Send(SectorPacked{FillerPieces: fillerPieces}) } func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) error { @@ -59,9 +59,9 @@ func (m *Sealing) handlePreCommit1(ctx statemachine.Context, sector SectorInfo) log.Errorf("handlePreCommit1: api error, not proceeding: %+v", err) return nil case *ErrInvalidDeals: - return ctx.Send(SectorPackingFailed{xerrors.Errorf("invalid deals in sector: %w", err)}) + return ctx.Send(SectorPackingFailed{xerrors.Errorf("invalid dealIDs in sector: %w", err)}) case *ErrExpiredDeals: // Probably not much we can do here, maybe re-pack the sector? - return ctx.Send(SectorPackingFailed{xerrors.Errorf("expired deals in sector: %w", err)}) + return ctx.Send(SectorPackingFailed{xerrors.Errorf("expired dealIDs in sector: %w", err)}) default: return xerrors.Errorf("checkPieces sanity check error: %w", err) } @@ -113,13 +113,13 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf } params := &miner.SectorPreCommitInfo{ - Expiration: 10000000, // TODO: implement + Expiration: 10000000, // TODO: implement expiration SectorNumber: sector.SectorNumber, RegisteredProof: sector.SectorType, SealedCID: *sector.CommR, SealRandEpoch: sector.TicketEpoch, - DealIDs: sector.deals(), + DealIDs: sector.dealIDs(), } enc := new(bytes.Buffer) diff --git a/types.go b/types.go index 7846ed1fe..a6d6a2b5f 100644 --- a/types.go +++ b/types.go @@ -11,11 +11,30 @@ import ( "github.com/filecoin-project/specs-storage/storage" ) -type Piece struct { - DealID *abi.DealID +// PieceWithOptionalDealInfo is a tuple of piece and deal info +type PieceWithDealInfo struct { + Piece abi.PieceInfo + DealInfo DealInfo +} - Size abi.UnpaddedPieceSize - CommP cid.Cid +// PieceWithOptionalDealInfo is a tuple of piece info and optional deal +type PieceWithOptionalDealInfo struct { + Piece abi.PieceInfo + DealInfo *DealInfo // nil for pieces which do not yet appear in self-deals +} + +// DealInfo is a tuple of deal identity and its schedule +type DealInfo struct { + DealID abi.DealID + DealSchedule DealSchedule +} + +// DealSchedule communicates the time interval of a storage deal. The deal must +// appear in a sealed (proven) sector no later than StartEpoch, otherwise it +// is invalid. +type DealSchedule struct { + StartEpoch abi.ChainEpoch + EndEpoch abi.ChainEpoch } type Log struct { @@ -36,8 +55,7 @@ type SectorInfo struct { SectorType abi.RegisteredProof // Packing - - Pieces []Piece + PiecesWithOptionalDealInfo []PieceWithOptionalDealInfo // PreCommit1 TicketValue abi.SealRandomness @@ -69,31 +87,31 @@ type SectorInfo struct { } func (t *SectorInfo) pieceInfos() []abi.PieceInfo { - out := make([]abi.PieceInfo, len(t.Pieces)) - for i, piece := range t.Pieces { + out := make([]abi.PieceInfo, len(t.PiecesWithOptionalDealInfo)) + for i, pdi := range t.PiecesWithOptionalDealInfo { out[i] = abi.PieceInfo{ - Size: piece.Size.Padded(), - PieceCID: piece.CommP, + Size: pdi.Piece.Size, + PieceCID: pdi.Piece.PieceCID, } } return out } -func (t *SectorInfo) deals() []abi.DealID { - out := make([]abi.DealID, 0, len(t.Pieces)) - for _, piece := range t.Pieces { - if piece.DealID == nil { +func (t *SectorInfo) dealIDs() []abi.DealID { + out := make([]abi.DealID, 0, len(t.PiecesWithOptionalDealInfo)) + for _, pdi := range t.PiecesWithOptionalDealInfo { + if pdi.DealInfo == nil { continue } - out = append(out, *piece.DealID) + out = append(out, pdi.DealInfo.DealID) } return out } -func (t *SectorInfo) existingPieces() []abi.UnpaddedPieceSize { - out := make([]abi.UnpaddedPieceSize, len(t.Pieces)) - for i, piece := range t.Pieces { - out[i] = piece.Size +func (t *SectorInfo) existingPieceSizes() []abi.UnpaddedPieceSize { + out := make([]abi.UnpaddedPieceSize, len(t.PiecesWithOptionalDealInfo)) + for i, pdi := range t.PiecesWithOptionalDealInfo { + out[i] = pdi.Piece.Size.Unpadded() } return out } diff --git a/types_test.go b/types_test.go index 7dc8ddcca..379c59564 100644 --- a/types_test.go +++ b/types_test.go @@ -14,16 +14,26 @@ import ( func TestSectorInfoSelialization(t *testing.T) { d := abi.DealID(1234) + dealInfo := DealInfo{ + DealID: d, + DealSchedule: DealSchedule{ + StartEpoch: 0, + EndEpoch: 100, + }, + } + dummyCid := builtin.AccountActorCodeID si := &SectorInfo{ State: "stateful", SectorNumber: 234, Nonce: 345, - Pieces: []Piece{{ - DealID: &d, - Size: 5, - CommP: dummyCid, + PiecesWithOptionalDealInfo: []PieceWithOptionalDealInfo{{ + Piece: abi.PieceInfo{ + Size: 5, + PieceCID: dummyCid, + }, + DealInfo: &dealInfo, }}, CommD: &dummyCid, CommR: nil, @@ -52,7 +62,7 @@ func TestSectorInfoSelialization(t *testing.T) { assert.Equal(t, si.Nonce, si2.Nonce) assert.Equal(t, si.SectorNumber, si2.SectorNumber) - assert.Equal(t, si.Pieces, si2.Pieces) + assert.Equal(t, si.PiecesWithOptionalDealInfo, si2.PiecesWithOptionalDealInfo) assert.Equal(t, si.CommD, si2.CommD) assert.Equal(t, si.TicketValue, si2.TicketValue) assert.Equal(t, si.TicketEpoch, si2.TicketEpoch) From 1f1e291cd12431af9fe8e65183937557b15c0dc1 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 7 Apr 2020 15:26:07 -0700 Subject: [PATCH 2/7] use pre-commit policy to set pre-commit sector expiry epoch --- go.mod | 1 + precommit_policy.go | 76 ++++++++++++++++++++++ precommit_policy_test.go | 134 +++++++++++++++++++++++++++++++++++++++ sealing.go | 5 +- states.go | 7 +- 5 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 precommit_policy.go create mode 100644 precommit_policy_test.go diff --git a/go.mod b/go.mod index 5ccc6b09a..b8250346b 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 + github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5 github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6 github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663 // indirect github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9 diff --git a/precommit_policy.go b/precommit_policy.go new file mode 100644 index 000000000..58ddcb492 --- /dev/null +++ b/precommit_policy.go @@ -0,0 +1,76 @@ +package sealing + +import ( + "context" + + "github.com/filecoin-project/specs-actors/actors/abi" +) + +type PreCommitPolicy interface { + Expiration(ctx context.Context, pdis ...PieceWithOptionalDealInfo) (abi.ChainEpoch, error) +} + +type Chain interface { + ChainHead(ctx context.Context) (TipSetToken, abi.ChainEpoch, error) +} + +// BasicPreCommitPolicy satisfies PreCommitPolicy. It has two modes: +// +// Mode 1: The sector contains a non-zero quantity of pieces with deal info +// Mode 2: The sector contains no pieces with deal info +// +// The BasicPreCommitPolicy#Expiration method is given a slice of the pieces +// which the miner has encoded into the sector, and from that slice picks either +// the first or second mode. +// +// If we're in Mode 1: The pre-commit expiration epoch will be the maximum +// deal end epoch of a piece in the sector. +// +// If we're in Mode 2: The pre-commit expiration epoch will be set to the +// current epoch + the provided default duration. +type BasicPreCommitPolicy struct { + api Chain + + duration abi.ChainEpoch +} + +// NewBasicPreCommitPolicy produces a BasicPreCommitPolicy +func NewBasicPreCommitPolicy(api Chain, duration abi.ChainEpoch) BasicPreCommitPolicy { + return BasicPreCommitPolicy{ + api: api, + duration: duration, + } +} + +// Expiration produces the pre-commit sector expiration epoch for an encoded +// replica containing the provided enumeration of pieces and deals. +func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, pdis ...PieceWithOptionalDealInfo) (abi.ChainEpoch, error) { + _, epoch, err := p.api.ChainHead(ctx) + if err != nil { + return 0, nil + } + + var end *abi.ChainEpoch + + for _, p := range pdis { + if p.DealInfo == nil { + continue + } + + if p.DealInfo.DealSchedule.EndEpoch < epoch { + log.Warnf("piece schedule %+v ended before current epoch %d", p, epoch) + continue + } + + if end == nil || *end < p.DealInfo.DealSchedule.EndEpoch { + end = &p.DealInfo.DealSchedule.EndEpoch + } + } + + if end == nil { + tmp := epoch + p.duration + end = &tmp + } + + return *end, nil +} diff --git a/precommit_policy_test.go b/precommit_policy_test.go new file mode 100644 index 000000000..d28f71825 --- /dev/null +++ b/precommit_policy_test.go @@ -0,0 +1,134 @@ +package sealing_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + commcid "github.com/filecoin-project/go-fil-commcid" + "github.com/filecoin-project/specs-actors/actors/abi" + + sealing "github.com/filecoin-project/storage-fsm" +) + +type fakeChain struct { + h abi.ChainEpoch +} + +func (f *fakeChain) ChainHead(ctx context.Context) (sealing.TipSetToken, abi.ChainEpoch, error) { + return []byte{1, 2, 3}, f.h, nil +} + +func TestBasicPolicyEmptySector(t *testing.T) { + policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ + h: abi.ChainEpoch(55), + }, 10) + + exp, err := policy.Expiration(context.Background()) + require.NoError(t, err) + + assert.Equal(t, 65, int(exp)) +} + +func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { + policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ + h: abi.ChainEpoch(55), + }, 100) + + pieces := []sealing.PieceWithOptionalDealInfo{ + { + Piece: abi.PieceInfo{ + Size: abi.PaddedPieceSize(1024), + PieceCID: commcid.ReplicaCommitmentV1ToCID([]byte{1, 2, 3}), + }, + DealInfo: &sealing.DealInfo{ + DealID: abi.DealID(42), + DealSchedule: sealing.DealSchedule{ + StartEpoch: abi.ChainEpoch(70), + EndEpoch: abi.ChainEpoch(75), + }, + }, + }, + { + Piece: abi.PieceInfo{ + Size: abi.PaddedPieceSize(1024), + PieceCID: commcid.ReplicaCommitmentV1ToCID([]byte{1, 2, 3}), + }, + DealInfo: &sealing.DealInfo{ + DealID: abi.DealID(43), + DealSchedule: sealing.DealSchedule{ + StartEpoch: abi.ChainEpoch(80), + EndEpoch: abi.ChainEpoch(100), + }, + }, + }, + } + + exp, err := policy.Expiration(context.Background(), pieces...) + require.NoError(t, err) + + assert.Equal(t, 100, int(exp)) +} + +func TestBasicPolicyIgnoresExistingScheduleIfExpired(t *testing.T) { + policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ + h: abi.ChainEpoch(55), + }, 100) + + pieces := []sealing.PieceWithOptionalDealInfo{ + { + Piece: abi.PieceInfo{ + Size: abi.PaddedPieceSize(1024), + PieceCID: commcid.ReplicaCommitmentV1ToCID([]byte{1, 2, 3}), + }, + DealInfo: &sealing.DealInfo{ + DealID: abi.DealID(44), + DealSchedule: sealing.DealSchedule{ + StartEpoch: abi.ChainEpoch(1), + EndEpoch: abi.ChainEpoch(10), + }, + }, + }, + } + + exp, err := policy.Expiration(context.Background(), pieces...) + require.NoError(t, err) + + assert.Equal(t, 155, int(exp)) +} + +func TestMissingDealIsIgnored(t *testing.T) { + policy := sealing.NewBasicPreCommitPolicy(&fakeChain{ + h: abi.ChainEpoch(55), + }, 100) + + pieces := []sealing.PieceWithOptionalDealInfo{ + { + Piece: abi.PieceInfo{ + Size: abi.PaddedPieceSize(1024), + PieceCID: commcid.ReplicaCommitmentV1ToCID([]byte{1, 2, 3}), + }, + DealInfo: &sealing.DealInfo{ + DealID: abi.DealID(44), + DealSchedule: sealing.DealSchedule{ + StartEpoch: abi.ChainEpoch(1), + EndEpoch: abi.ChainEpoch(10), + }, + }, + }, + { + Piece: abi.PieceInfo{ + Size: abi.PaddedPieceSize(1024), + PieceCID: commcid.ReplicaCommitmentV1ToCID([]byte{1, 2, 3}), + }, + DealInfo: nil, + }, + } + + exp, err := policy.Expiration(context.Background(), pieces...) + require.NoError(t, err) + + assert.Equal(t, 155, int(exp)) +} diff --git a/sealing.go b/sealing.go index aee3321b5..99fafa549 100644 --- a/sealing.go +++ b/sealing.go @@ -49,9 +49,11 @@ type Sealing struct { sc SectorIDCounter verif ffiwrapper.Verifier tktFn TicketFn + + pcp PreCommitPolicy } -func New(api SealingAPI, events Events, maddr address.Address, worker address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, tktFn TicketFn) *Sealing { +func New(api SealingAPI, events Events, maddr address.Address, worker address.Address, ds datastore.Batching, sealer sectorstorage.SectorManager, sc SectorIDCounter, verif ffiwrapper.Verifier, tktFn TicketFn, pcp PreCommitPolicy) *Sealing { s := &Sealing{ api: api, events: events, @@ -62,6 +64,7 @@ func New(api SealingAPI, events Events, maddr address.Address, worker address.Ad sc: sc, verif: verif, tktFn: tktFn, + pcp: pcp, } s.sectors = statemachine.New(namespace.Wrap(ds, datastore.NewKey(SectorStorePrefix)), s, SectorInfo{}) diff --git a/states.go b/states.go index 13946f3b1..d6b057606 100644 --- a/states.go +++ b/states.go @@ -112,8 +112,13 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf } } + expiration, err := m.pcp.Expiration(ctx.Context(), sector.PiecesWithOptionalDealInfo...) + if err != nil { + return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("handlePreCommitting: failed to compute pre-commit expiry: %w", err)}) + } + params := &miner.SectorPreCommitInfo{ - Expiration: 10000000, // TODO: implement expiration + Expiration: expiration, SectorNumber: sector.SectorNumber, RegisteredProof: sector.SectorType, From c9dd3b32382285d5913a2e7c57ef430ab07e34a5 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 7 Apr 2020 15:30:26 -0700 Subject: [PATCH 3/7] fix typo ("of" instead of "or") --- checks.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/checks.go b/checks.go index fc41e395c..8e0e10881 100644 --- a/checks.go +++ b/checks.go @@ -50,15 +50,15 @@ func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error { } if proposal.PieceCID != pdi.Piece.PieceCID { - return &ErrInvalidDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers deal %d with wrong PieceCID: %x != %x", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.PieceCID, proposal.PieceCID)} + return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with wrong PieceCID: %x != %x", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.PieceCID, proposal.PieceCID)} } if pdi.Piece.Size != proposal.PieceSize { - return &ErrInvalidDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers deal %d with different size: %d != %d", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.Size, proposal.PieceSize)} + return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with different size: %d != %d", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.Size, proposal.PieceSize)} } if height >= proposal.StartEpoch { - return &ErrExpiredDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers expired deal %d - should start at %d, head %d", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, proposal.StartEpoch, height)} + return &ErrExpiredDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers expired deal %d - should start at %d, head %d", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, proposal.StartEpoch, height)} } } From 5f3990d1bcd0f919807c571b97c4d8f7d9dd78c6 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 7 Apr 2020 15:33:24 -0700 Subject: [PATCH 4/7] remove unused, intermediate literal --- garbage.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/garbage.go b/garbage.go index 638f7500d..797989fad 100644 --- a/garbage.go +++ b/garbage.go @@ -32,10 +32,7 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorID, exist existingPieceSizes = append(existingPieceSizes, size) - out[i] = abi.PieceInfo{ - Size: ppi.Size, - PieceCID: ppi.PieceCID, - } + out[i] = ppi } return out, nil From 0cd4e0413f1bcf37ed0867a9bbd9185475e4ee36 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 8 Apr 2020 07:52:20 -0700 Subject: [PATCH 5/7] renaming PieceWithOptionalDealInfo -> Piece, drop PieceWithDealInfo --- cbor_gen.go | 36 ++++++++++++++++++------------------ checks.go | 8 ++++---- fsm_events.go | 6 +++--- garbage.go | 4 ++-- gen/main.go | 2 +- precommit_policy.go | 4 ++-- precommit_policy_test.go | 6 +++--- sealing.go | 11 ++++++----- states.go | 4 ++-- types.go | 22 +++++++++++----------- types_test.go | 4 ++-- 11 files changed, 54 insertions(+), 53 deletions(-) diff --git a/cbor_gen.go b/cbor_gen.go index 37a75a662..b2bd03971 100644 --- a/cbor_gen.go +++ b/cbor_gen.go @@ -13,7 +13,7 @@ import ( var _ = xerrors.Errorf -func (t *PieceWithOptionalDealInfo) MarshalCBOR(w io.Writer) error { +func (t *Piece) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -56,7 +56,7 @@ func (t *PieceWithOptionalDealInfo) MarshalCBOR(w io.Writer) error { return nil } -func (t *PieceWithOptionalDealInfo) UnmarshalCBOR(r io.Reader) error { +func (t *Piece) UnmarshalCBOR(r io.Reader) error { br := cbg.GetPeeker(r) maj, extra, err := cbg.CborReadHeader(br) @@ -68,7 +68,7 @@ func (t *PieceWithOptionalDealInfo) UnmarshalCBOR(r io.Reader) error { } if extra > cbg.MaxLength { - return fmt.Errorf("PieceWithOptionalDealInfo: map struct too large (%d)", extra) + return fmt.Errorf("Piece: map struct too large (%d)", extra) } var name string @@ -463,26 +463,26 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } } - // t.PiecesWithOptionalDealInfo ([]sealing.PieceWithOptionalDealInfo) (slice) - if len("PiecesWithOptionalDealInfo") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"PiecesWithOptionalDealInfo\" was too long") + // t.Pieces ([]sealing.Piece) (slice) + if len("Pieces") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Pieces\" was too long") } - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("PiecesWithOptionalDealInfo")))); err != nil { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Pieces")))); err != nil { return err } - if _, err := w.Write([]byte("PiecesWithOptionalDealInfo")); err != nil { + if _, err := w.Write([]byte("Pieces")); err != nil { return err } - if len(t.PiecesWithOptionalDealInfo) > cbg.MaxLength { - return xerrors.Errorf("Slice value in field t.PiecesWithOptionalDealInfo was too long") + if len(t.Pieces) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Pieces was too long") } - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.PiecesWithOptionalDealInfo)))); err != nil { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Pieces)))); err != nil { return err } - for _, v := range t.PiecesWithOptionalDealInfo { + for _, v := range t.Pieces { if err := v.MarshalCBOR(w); err != nil { return err } @@ -897,8 +897,8 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { t.SectorType = abi.RegisteredProof(extraI) } - // t.PiecesWithOptionalDealInfo ([]sealing.PieceWithOptionalDealInfo) (slice) - case "PiecesWithOptionalDealInfo": + // t.Pieces ([]sealing.Piece) (slice) + case "Pieces": maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -906,23 +906,23 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { } if extra > cbg.MaxLength { - return fmt.Errorf("t.PiecesWithOptionalDealInfo: array too large (%d)", extra) + return fmt.Errorf("t.Pieces: array too large (%d)", extra) } if maj != cbg.MajArray { return fmt.Errorf("expected cbor array") } if extra > 0 { - t.PiecesWithOptionalDealInfo = make([]PieceWithOptionalDealInfo, extra) + t.Pieces = make([]Piece, extra) } for i := 0; i < int(extra); i++ { - var v PieceWithOptionalDealInfo + var v Piece if err := v.UnmarshalCBOR(br); err != nil { return err } - t.PiecesWithOptionalDealInfo[i] = v + t.Pieces[i] = v } // t.TicketValue (abi.SealRandomness) (slice) diff --git a/checks.go b/checks.go index 8e0e10881..bdd6b5761 100644 --- a/checks.go +++ b/checks.go @@ -33,7 +33,7 @@ func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error { return &ErrApi{xerrors.Errorf("getting chain head: %w", err)} } - for i, pdi := range si.PiecesWithOptionalDealInfo { + for i, pdi := range si.Pieces { // if no deal is associated with the piece, ensure that we added it as // filler (i.e. ensure that it has a zero PieceCID) if pdi.DealInfo == nil { @@ -50,15 +50,15 @@ func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error { } if proposal.PieceCID != pdi.Piece.PieceCID { - return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with wrong PieceCID: %x != %x", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.PieceCID, proposal.PieceCID)} + return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with wrong PieceCID: %x != %x", i, len(si.Pieces), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.PieceCID, proposal.PieceCID)} } if pdi.Piece.Size != proposal.PieceSize { - return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with different size: %d != %d", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.Size, proposal.PieceSize)} + return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with different size: %d != %d", i, len(si.Pieces), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.Size, proposal.PieceSize)} } if height >= proposal.StartEpoch { - return &ErrExpiredDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers expired deal %d - should start at %d, head %d", i, len(si.PiecesWithOptionalDealInfo), si.SectorNumber, pdi.DealInfo.DealID, proposal.StartEpoch, height)} + return &ErrExpiredDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers expired deal %d - should start at %d, head %d", i, len(si.Pieces), si.SectorNumber, pdi.DealInfo.DealID, proposal.StartEpoch, height)} } } diff --git a/fsm_events.go b/fsm_events.go index d7d79c42d..a0f6c6368 100644 --- a/fsm_events.go +++ b/fsm_events.go @@ -50,12 +50,12 @@ func (evt SectorForceState) applyGlobal(state *SectorInfo) bool { type SectorStart struct { ID abi.SectorNumber SectorType abi.RegisteredProof - Pieces []PieceWithOptionalDealInfo + Pieces []Piece } func (evt SectorStart) apply(state *SectorInfo) { state.SectorNumber = evt.ID - state.PiecesWithOptionalDealInfo = evt.Pieces + state.Pieces = evt.Pieces state.SectorType = evt.SectorType } @@ -63,7 +63,7 @@ type SectorPacked struct{ FillerPieces []abi.PieceInfo } func (evt SectorPacked) apply(state *SectorInfo) { for idx := range evt.FillerPieces { - state.PiecesWithOptionalDealInfo = append(state.PiecesWithOptionalDealInfo, PieceWithOptionalDealInfo{ + state.Pieces = append(state.Pieces, Piece{ Piece: evt.FillerPieces[idx], DealInfo: nil, // filler pieces don't have deals associated with them }) diff --git a/garbage.go b/garbage.go index 797989fad..694f6aea1 100644 --- a/garbage.go +++ b/garbage.go @@ -69,9 +69,9 @@ func (m *Sealing) PledgeSector() error { return } - pdis := make([]PieceWithOptionalDealInfo, len(pieces)) + pdis := make([]Piece, len(pieces)) for idx := range pdis { - pdis[idx] = PieceWithOptionalDealInfo{ + pdis[idx] = Piece{ Piece: pieces[idx], DealInfo: nil, } diff --git a/gen/main.go b/gen/main.go index e9a68425a..0d5f7507b 100644 --- a/gen/main.go +++ b/gen/main.go @@ -11,7 +11,7 @@ import ( func main() { err := gen.WriteMapEncodersToFile("./cbor_gen.go", "sealing", - sealing.PieceWithOptionalDealInfo{}, + sealing.Piece{}, sealing.DealInfo{}, sealing.DealSchedule{}, sealing.SectorInfo{}, diff --git a/precommit_policy.go b/precommit_policy.go index 58ddcb492..c0c855f73 100644 --- a/precommit_policy.go +++ b/precommit_policy.go @@ -7,7 +7,7 @@ import ( ) type PreCommitPolicy interface { - Expiration(ctx context.Context, pdis ...PieceWithOptionalDealInfo) (abi.ChainEpoch, error) + Expiration(ctx context.Context, pdis ...Piece) (abi.ChainEpoch, error) } type Chain interface { @@ -44,7 +44,7 @@ func NewBasicPreCommitPolicy(api Chain, duration abi.ChainEpoch) BasicPreCommitP // Expiration produces the pre-commit sector expiration epoch for an encoded // replica containing the provided enumeration of pieces and deals. -func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, pdis ...PieceWithOptionalDealInfo) (abi.ChainEpoch, error) { +func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, pdis ...Piece) (abi.ChainEpoch, error) { _, epoch, err := p.api.ChainHead(ctx) if err != nil { return 0, nil diff --git a/precommit_policy_test.go b/precommit_policy_test.go index d28f71825..762e34685 100644 --- a/precommit_policy_test.go +++ b/precommit_policy_test.go @@ -37,7 +37,7 @@ func TestBasicPolicyMostConstrictiveSchedule(t *testing.T) { h: abi.ChainEpoch(55), }, 100) - pieces := []sealing.PieceWithOptionalDealInfo{ + pieces := []sealing.Piece{ { Piece: abi.PieceInfo{ Size: abi.PaddedPieceSize(1024), @@ -77,7 +77,7 @@ func TestBasicPolicyIgnoresExistingScheduleIfExpired(t *testing.T) { h: abi.ChainEpoch(55), }, 100) - pieces := []sealing.PieceWithOptionalDealInfo{ + pieces := []sealing.Piece{ { Piece: abi.PieceInfo{ Size: abi.PaddedPieceSize(1024), @@ -104,7 +104,7 @@ func TestMissingDealIsIgnored(t *testing.T) { h: abi.ChainEpoch(55), }, 100) - pieces := []sealing.PieceWithOptionalDealInfo{ + pieces := []sealing.Piece{ { Piece: abi.PieceInfo{ Size: abi.PaddedPieceSize(1024), diff --git a/sealing.go b/sealing.go index 99fafa549..837ba279d 100644 --- a/sealing.go +++ b/sealing.go @@ -35,6 +35,7 @@ type SealingAPI interface { SendMsg(ctx context.Context, from, to address.Address, method abi.MethodNum, value, gasPrice big.Int, gasLimit int64, params []byte) (cid.Cid, error) ChainHead(ctx context.Context) (TipSetToken, abi.ChainEpoch, error) ChainGetRandomness(ctx context.Context, tok TipSetToken, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) } type Sealing struct { @@ -104,8 +105,8 @@ func (m *Sealing) AllocatePiece(size abi.UnpaddedPieceSize) (sectorID abi.Sector return sid, 0, nil } -func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, sectorID abi.SectorNumber, pdi PieceWithDealInfo) error { - log.Infof("Seal piece for deal %d", pdi.DealInfo.DealID) +func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r io.Reader, sectorID abi.SectorNumber, d DealInfo) error { + log.Infof("Seal piece for deal %d", d.DealID) ppi, err := m.sealer.AddPiece(ctx, m.minerSector(sectorID), []abi.UnpaddedPieceSize{}, size, r) if err != nil { @@ -117,10 +118,10 @@ func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r i return xerrors.Errorf("bad sector size: %w", err) } - return m.newSector(sectorID, rt, []PieceWithOptionalDealInfo{ + return m.newSector(sectorID, rt, []Piece{ { Piece: ppi, - DealInfo: &pdi.DealInfo, + DealInfo: &d, }, }) } @@ -128,7 +129,7 @@ func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r i // newSector accepts a slice of pieces which will have a deal associated with // them (in the event of a storage deal) or no deal (in the event of sealing // garbage data) -func (m *Sealing) newSector(sid abi.SectorNumber, rt abi.RegisteredProof, pieces []PieceWithOptionalDealInfo) error { +func (m *Sealing) newSector(sid abi.SectorNumber, rt abi.RegisteredProof, pieces []Piece) error { log.Infof("Start sealing %d", sid) return m.sectors.Send(uint64(sid), SectorStart{ ID: sid, diff --git a/states.go b/states.go index d6b057606..aca00f4b3 100644 --- a/states.go +++ b/states.go @@ -19,7 +19,7 @@ func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) err log.Infow("performing filling up rest of the sector...", "sector", sector.SectorNumber) var allocated abi.UnpaddedPieceSize - for _, piece := range sector.PiecesWithOptionalDealInfo { + for _, piece := range sector.Pieces { allocated += piece.Piece.Size.Unpadded() } @@ -112,7 +112,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf } } - expiration, err := m.pcp.Expiration(ctx.Context(), sector.PiecesWithOptionalDealInfo...) + expiration, err := m.pcp.Expiration(ctx.Context(), sector.Pieces...) if err != nil { return ctx.Send(SectorSealPreCommitFailed{xerrors.Errorf("handlePreCommitting: failed to compute pre-commit expiry: %w", err)}) } diff --git a/types.go b/types.go index a6d6a2b5f..5d8019721 100644 --- a/types.go +++ b/types.go @@ -11,16 +11,16 @@ import ( "github.com/filecoin-project/specs-storage/storage" ) -// PieceWithOptionalDealInfo is a tuple of piece and deal info +// Piece is a tuple of piece and deal info type PieceWithDealInfo struct { Piece abi.PieceInfo DealInfo DealInfo } -// PieceWithOptionalDealInfo is a tuple of piece info and optional deal -type PieceWithOptionalDealInfo struct { +// Piece is a tuple of piece info and optional deal +type Piece struct { Piece abi.PieceInfo - DealInfo *DealInfo // nil for pieces which do not yet appear in self-deals + DealInfo *DealInfo // nil for pieces which do not appear in deals (e.g. filler pieces) } // DealInfo is a tuple of deal identity and its schedule @@ -55,7 +55,7 @@ type SectorInfo struct { SectorType abi.RegisteredProof // Packing - PiecesWithOptionalDealInfo []PieceWithOptionalDealInfo + Pieces []Piece // PreCommit1 TicketValue abi.SealRandomness @@ -87,8 +87,8 @@ type SectorInfo struct { } func (t *SectorInfo) pieceInfos() []abi.PieceInfo { - out := make([]abi.PieceInfo, len(t.PiecesWithOptionalDealInfo)) - for i, pdi := range t.PiecesWithOptionalDealInfo { + out := make([]abi.PieceInfo, len(t.Pieces)) + for i, pdi := range t.Pieces { out[i] = abi.PieceInfo{ Size: pdi.Piece.Size, PieceCID: pdi.Piece.PieceCID, @@ -98,8 +98,8 @@ func (t *SectorInfo) pieceInfos() []abi.PieceInfo { } func (t *SectorInfo) dealIDs() []abi.DealID { - out := make([]abi.DealID, 0, len(t.PiecesWithOptionalDealInfo)) - for _, pdi := range t.PiecesWithOptionalDealInfo { + out := make([]abi.DealID, 0, len(t.Pieces)) + for _, pdi := range t.Pieces { if pdi.DealInfo == nil { continue } @@ -109,8 +109,8 @@ func (t *SectorInfo) dealIDs() []abi.DealID { } func (t *SectorInfo) existingPieceSizes() []abi.UnpaddedPieceSize { - out := make([]abi.UnpaddedPieceSize, len(t.PiecesWithOptionalDealInfo)) - for i, pdi := range t.PiecesWithOptionalDealInfo { + out := make([]abi.UnpaddedPieceSize, len(t.Pieces)) + for i, pdi := range t.Pieces { out[i] = pdi.Piece.Size.Unpadded() } return out diff --git a/types_test.go b/types_test.go index 379c59564..9bb1df8cf 100644 --- a/types_test.go +++ b/types_test.go @@ -28,7 +28,7 @@ func TestSectorInfoSelialization(t *testing.T) { State: "stateful", SectorNumber: 234, Nonce: 345, - PiecesWithOptionalDealInfo: []PieceWithOptionalDealInfo{{ + Pieces: []Piece{{ Piece: abi.PieceInfo{ Size: 5, PieceCID: dummyCid, @@ -62,7 +62,7 @@ func TestSectorInfoSelialization(t *testing.T) { assert.Equal(t, si.Nonce, si2.Nonce) assert.Equal(t, si.SectorNumber, si2.SectorNumber) - assert.Equal(t, si.PiecesWithOptionalDealInfo, si2.PiecesWithOptionalDealInfo) + assert.Equal(t, si.Pieces, si2.Pieces) assert.Equal(t, si.CommD, si2.CommD) assert.Equal(t, si.TicketValue, si2.TicketValue) assert.Equal(t, si.TicketEpoch, si2.TicketEpoch) From 1dfca2ff76ebf0b884853e272631a16e4e3e715b Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 8 Apr 2020 07:52:47 -0700 Subject: [PATCH 6/7] drop unused method brought in via rebase --- sealing.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sealing.go b/sealing.go index 837ba279d..e9dd2c6fd 100644 --- a/sealing.go +++ b/sealing.go @@ -35,7 +35,6 @@ type SealingAPI interface { SendMsg(ctx context.Context, from, to address.Address, method abi.MethodNum, value, gasPrice big.Int, gasLimit int64, params []byte) (cid.Cid, error) ChainHead(ctx context.Context) (TipSetToken, abi.ChainEpoch, error) ChainGetRandomness(ctx context.Context, tok TipSetToken, personalization crypto.DomainSeparationTag, randEpoch abi.ChainEpoch, entropy []byte) (abi.Randomness, error) - ChainReadObj(context.Context, cid.Cid) ([]byte, error) } type Sealing struct { From f2649af5a19e6e3e0ce853d267ac207072a67182 Mon Sep 17 00:00:00 2001 From: laser Date: Wed, 8 Apr 2020 07:56:35 -0700 Subject: [PATCH 7/7] rename pdi ("peace with deal info") -> p ("piece") --- checks.go | 24 ++++++++++++------------ garbage.go | 8 ++++---- precommit_policy.go | 6 +++--- types.go | 17 +++++++---------- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/checks.go b/checks.go index bdd6b5761..e5b7ed8af 100644 --- a/checks.go +++ b/checks.go @@ -33,32 +33,32 @@ func checkPieces(ctx context.Context, si SectorInfo, api SealingAPI) error { return &ErrApi{xerrors.Errorf("getting chain head: %w", err)} } - for i, pdi := range si.Pieces { + for i, p := range si.Pieces { // if no deal is associated with the piece, ensure that we added it as // filler (i.e. ensure that it has a zero PieceCID) - if pdi.DealInfo == nil { - exp := zerocomm.ZeroPieceCommitment(pdi.Piece.Size.Unpadded()) - if !pdi.Piece.PieceCID.Equals(exp) { - return &ErrInvalidPiece{xerrors.Errorf("sector %d piece %d had non-zero PieceCID %+v", si.SectorNumber, i, pdi.Piece.PieceCID)} + if p.DealInfo == nil { + exp := zerocomm.ZeroPieceCommitment(p.Piece.Size.Unpadded()) + if !p.Piece.PieceCID.Equals(exp) { + return &ErrInvalidPiece{xerrors.Errorf("sector %d piece %d had non-zero PieceCID %+v", si.SectorNumber, i, p.Piece.PieceCID)} } continue } - proposal, _, err := api.StateMarketStorageDeal(ctx, pdi.DealInfo.DealID, tok) + proposal, _, err := api.StateMarketStorageDeal(ctx, p.DealInfo.DealID, tok) if err != nil { - return &ErrApi{xerrors.Errorf("getting deal %d for piece %d: %w", pdi.DealInfo.DealID, i, err)} + return &ErrApi{xerrors.Errorf("getting deal %d for piece %d: %w", p.DealInfo.DealID, i, err)} } - if proposal.PieceCID != pdi.Piece.PieceCID { - return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with wrong PieceCID: %x != %x", i, len(si.Pieces), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.PieceCID, proposal.PieceCID)} + if proposal.PieceCID != p.Piece.PieceCID { + return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with wrong PieceCID: %x != %x", i, len(si.Pieces), si.SectorNumber, p.DealInfo.DealID, p.Piece.PieceCID, proposal.PieceCID)} } - if pdi.Piece.Size != proposal.PieceSize { - return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with different size: %d != %d", i, len(si.Pieces), si.SectorNumber, pdi.DealInfo.DealID, pdi.Piece.Size, proposal.PieceSize)} + if p.Piece.Size != proposal.PieceSize { + return &ErrInvalidDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers deal %d with different size: %d != %d", i, len(si.Pieces), si.SectorNumber, p.DealInfo.DealID, p.Piece.Size, proposal.PieceSize)} } if height >= proposal.StartEpoch { - return &ErrExpiredDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers expired deal %d - should start at %d, head %d", i, len(si.Pieces), si.SectorNumber, pdi.DealInfo.DealID, proposal.StartEpoch, height)} + return &ErrExpiredDeals{xerrors.Errorf("piece %d (of %d) of sector %d refers expired deal %d - should start at %d, head %d", i, len(si.Pieces), si.SectorNumber, p.DealInfo.DealID, proposal.StartEpoch, height)} } } diff --git a/garbage.go b/garbage.go index 694f6aea1..57697ec7b 100644 --- a/garbage.go +++ b/garbage.go @@ -69,15 +69,15 @@ func (m *Sealing) PledgeSector() error { return } - pdis := make([]Piece, len(pieces)) - for idx := range pdis { - pdis[idx] = Piece{ + ps := make([]Piece, len(pieces)) + for idx := range ps { + ps[idx] = Piece{ Piece: pieces[idx], DealInfo: nil, } } - if err := m.newSector(sid, rt, pdis); err != nil { + if err := m.newSector(sid, rt, ps); err != nil { log.Errorf("%+v", err) return } diff --git a/precommit_policy.go b/precommit_policy.go index c0c855f73..6477a791a 100644 --- a/precommit_policy.go +++ b/precommit_policy.go @@ -7,7 +7,7 @@ import ( ) type PreCommitPolicy interface { - Expiration(ctx context.Context, pdis ...Piece) (abi.ChainEpoch, error) + Expiration(ctx context.Context, ps ...Piece) (abi.ChainEpoch, error) } type Chain interface { @@ -44,7 +44,7 @@ func NewBasicPreCommitPolicy(api Chain, duration abi.ChainEpoch) BasicPreCommitP // Expiration produces the pre-commit sector expiration epoch for an encoded // replica containing the provided enumeration of pieces and deals. -func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, pdis ...Piece) (abi.ChainEpoch, error) { +func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, ps ...Piece) (abi.ChainEpoch, error) { _, epoch, err := p.api.ChainHead(ctx) if err != nil { return 0, nil @@ -52,7 +52,7 @@ func (p *BasicPreCommitPolicy) Expiration(ctx context.Context, pdis ...Piece) (a var end *abi.ChainEpoch - for _, p := range pdis { + for _, p := range ps { if p.DealInfo == nil { continue } diff --git a/types.go b/types.go index 5d8019721..332016d15 100644 --- a/types.go +++ b/types.go @@ -88,30 +88,27 @@ type SectorInfo struct { func (t *SectorInfo) pieceInfos() []abi.PieceInfo { out := make([]abi.PieceInfo, len(t.Pieces)) - for i, pdi := range t.Pieces { - out[i] = abi.PieceInfo{ - Size: pdi.Piece.Size, - PieceCID: pdi.Piece.PieceCID, - } + for i, p := range t.Pieces { + out[i] = p.Piece } return out } func (t *SectorInfo) dealIDs() []abi.DealID { out := make([]abi.DealID, 0, len(t.Pieces)) - for _, pdi := range t.Pieces { - if pdi.DealInfo == nil { + for _, p := range t.Pieces { + if p.DealInfo == nil { continue } - out = append(out, pdi.DealInfo.DealID) + out = append(out, p.DealInfo.DealID) } return out } func (t *SectorInfo) existingPieceSizes() []abi.UnpaddedPieceSize { out := make([]abi.UnpaddedPieceSize, len(t.Pieces)) - for i, pdi := range t.Pieces { - out[i] = pdi.Piece.Size.Unpadded() + for i, p := range t.Pieces { + out[i] = p.Piece.Size.Unpadded() } return out }