diff --git a/.github/workflows/testground-on-push.yml b/.github/workflows/testground-on-push.yml index 2a3c8af1d..8e749bfae 100644 --- a/.github/workflows/testground-on-push.yml +++ b/.github/workflows/testground-on-push.yml @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: testground run - uses: coryschwartz/testground-github-action@v1.1 + uses: testground/testground-github-action@v1 with: backend_addr: ${{ matrix.backend_addr }} backend_proto: ${{ matrix.backend_proto }} diff --git a/api/api_storage.go b/api/api_storage.go index 5785ede1f..9de54ff3e 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -227,6 +227,9 @@ type StorageMiner interface { // DagstoreGC runs garbage collection on the DAG store. DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin + // DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID + DagstoreRegisterShard(ctx context.Context, key string) error //perm:admin + // IndexerAnnounceDeal informs indexer nodes that a new deal was received, // so they can download its index IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin diff --git a/api/cbor_gen.go b/api/cbor_gen.go index eb4c34f8a..62e523df1 100644 --- a/api/cbor_gen.go +++ b/api/cbor_gen.go @@ -26,25 +26,26 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{163}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { return err } - scratch := make([]byte, 9) - // t.Channel (address.Address) (struct) if len("Channel") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Channel\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Channel"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Channel"))); err != nil { return err } if _, err := io.WriteString(w, string("Channel")); err != nil { return err } - if err := t.Channel.MarshalCBOR(w); err != nil { + if err := t.Channel.MarshalCBOR(cw); err != nil { return err } @@ -53,14 +54,14 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"WaitSentinel\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil { return err } if _, err := io.WriteString(w, string("WaitSentinel")); err != nil { return err } - if err := cbg.WriteCidBuf(scratch, w, t.WaitSentinel); err != nil { + if err := cbg.WriteCid(cw, t.WaitSentinel); err != nil { return xerrors.Errorf("failed to write cid field t.WaitSentinel: %w", err) } @@ -69,7 +70,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Vouchers\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Vouchers"))); err != nil { return err } if _, err := io.WriteString(w, string("Vouchers")); err != nil { @@ -80,27 +81,32 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Vouchers was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Vouchers))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Vouchers))); err != nil { return err } for _, v := range t.Vouchers { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } return nil } -func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { +func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) (err error) { *t = PaymentInfo{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -115,7 +121,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -129,7 +135,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { { - if err := t.Channel.UnmarshalCBOR(br); err != nil { + if err := t.Channel.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Channel: %w", err) } @@ -139,7 +145,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.WaitSentinel: %w", err) } @@ -150,7 +156,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { // t.Vouchers ([]*paych.SignedVoucher) (slice) case "Vouchers": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -170,7 +176,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v paych.SignedVoucher - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -190,25 +196,26 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{163}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { return err } - scratch := make([]byte, 9) - // t.SectorID (abi.SectorNumber) (uint64) if len("SectorID") > cbg.MaxLength { return xerrors.Errorf("Value in field \"SectorID\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("SectorID"))); err != nil { return err } if _, err := io.WriteString(w, string("SectorID")); err != nil { return err } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SectorID)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.SectorID)); err != nil { return err } @@ -217,14 +224,14 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Offset\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Offset"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Offset"))); err != nil { return err } if _, err := io.WriteString(w, string("Offset")); err != nil { return err } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Offset)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Offset)); err != nil { return err } @@ -233,30 +240,35 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Size\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Size"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Size"))); err != nil { return err } if _, err := io.WriteString(w, string("Size")); err != nil { return err } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Size)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Size)); err != nil { return err } return nil } -func (t *SealedRef) UnmarshalCBOR(r io.Reader) error { +func (t *SealedRef) UnmarshalCBOR(r io.Reader) (err error) { *t = SealedRef{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -271,7 +283,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -285,7 +297,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -300,7 +312,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -315,7 +327,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -339,18 +351,19 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{161}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { return err } - scratch := make([]byte, 9) - // t.Refs ([]api.SealedRef) (slice) if len("Refs") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Refs\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Refs"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Refs"))); err != nil { return err } if _, err := io.WriteString(w, string("Refs")); err != nil { @@ -361,27 +374,32 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Refs was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Refs))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Refs))); err != nil { return err } for _, v := range t.Refs { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } return nil } -func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error { +func (t *SealedRefs) UnmarshalCBOR(r io.Reader) (err error) { *t = SealedRefs{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -396,7 +414,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -408,7 +426,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error { // t.Refs ([]api.SealedRef) (slice) case "Refs": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -428,7 +446,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v SealedRef - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -448,18 +466,19 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{162}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { return err } - scratch := make([]byte, 9) - // t.Value (abi.SealRandomness) (slice) if len("Value") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Value\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Value"))); err != nil { return err } if _, err := io.WriteString(w, string("Value")); err != nil { @@ -470,11 +489,11 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.Value was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Value))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Value))); err != nil { return err } - if _, err := w.Write(t.Value[:]); err != nil { + if _, err := cw.Write(t.Value[:]); err != nil { return err } @@ -483,7 +502,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Epoch\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Epoch"))); err != nil { return err } if _, err := io.WriteString(w, string("Epoch")); err != nil { @@ -491,27 +510,32 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error { } if t.Epoch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil { return err } } return nil } -func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { +func (t *SealTicket) UnmarshalCBOR(r io.Reader) (err error) { *t = SealTicket{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -526,7 +550,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -538,7 +562,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { // t.Value (abi.SealRandomness) (slice) case "Value": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -554,13 +578,13 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { t.Value = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.Value[:]); err != nil { + if _, err := io.ReadFull(cr, t.Value[:]); err != nil { return err } // t.Epoch (abi.ChainEpoch) (int64) case "Epoch": { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -597,18 +621,19 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{162}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { return err } - scratch := make([]byte, 9) - // t.Value (abi.InteractiveSealRandomness) (slice) if len("Value") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Value\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Value"))); err != nil { return err } if _, err := io.WriteString(w, string("Value")); err != nil { @@ -619,11 +644,11 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.Value was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Value))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Value))); err != nil { return err } - if _, err := w.Write(t.Value[:]); err != nil { + if _, err := cw.Write(t.Value[:]); err != nil { return err } @@ -632,7 +657,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Epoch\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Epoch"))); err != nil { return err } if _, err := io.WriteString(w, string("Epoch")); err != nil { @@ -640,27 +665,32 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error { } if t.Epoch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil { return err } } return nil } -func (t *SealSeed) UnmarshalCBOR(r io.Reader) error { +func (t *SealSeed) UnmarshalCBOR(r io.Reader) (err error) { *t = SealSeed{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -675,7 +705,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -687,7 +717,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error { // t.Value (abi.InteractiveSealRandomness) (slice) case "Value": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -703,13 +733,13 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error { t.Value = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.Value[:]); err != nil { + if _, err := io.ReadFull(cr, t.Value[:]); err != nil { return err } // t.Epoch (abi.ChainEpoch) (int64) case "Epoch": { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -746,18 +776,19 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{165}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{165}); err != nil { return err } - scratch := make([]byte, 9) - // t.PublishCid (cid.Cid) (struct) if len("PublishCid") > cbg.MaxLength { return xerrors.Errorf("Value in field \"PublishCid\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PublishCid"))); err != nil { return err } if _, err := io.WriteString(w, string("PublishCid")); err != nil { @@ -765,11 +796,11 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error { } if t.PublishCid == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil { + if err := cbg.WriteCid(cw, *t.PublishCid); err != nil { return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err) } } @@ -779,14 +810,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"DealID\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealID"))); err != nil { return err } if _, err := io.WriteString(w, string("DealID")); err != nil { return err } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { return err } @@ -795,14 +826,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"DealProposal\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealProposal"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealProposal"))); err != nil { return err } if _, err := io.WriteString(w, string("DealProposal")); err != nil { return err } - if err := t.DealProposal.MarshalCBOR(w); err != nil { + if err := t.DealProposal.MarshalCBOR(cw); err != nil { return err } @@ -811,14 +842,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"DealSchedule\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealSchedule"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealSchedule"))); err != nil { return err } if _, err := io.WriteString(w, string("DealSchedule")); err != nil { return err } - if err := t.DealSchedule.MarshalCBOR(w); err != nil { + if err := t.DealSchedule.MarshalCBOR(cw); err != nil { return err } @@ -827,7 +858,7 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil { return err } if _, err := io.WriteString(w, string("KeepUnsealed")); err != nil { @@ -840,16 +871,21 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error { return nil } -func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error { +func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) (err error) { *t = PieceDealInfo{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -864,7 +900,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -878,16 +914,16 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err) } @@ -901,7 +937,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -916,16 +952,16 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.DealProposal = new(market.DealProposal) - if err := t.DealProposal.UnmarshalCBOR(br); err != nil { + if err := t.DealProposal.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.DealProposal pointer: %w", err) } } @@ -936,7 +972,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error { { - if err := t.DealSchedule.UnmarshalCBOR(br); err != nil { + if err := t.DealSchedule.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err) } @@ -944,7 +980,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error { // t.KeepUnsealed (bool) (bool) case "KeepUnsealed": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -973,18 +1009,19 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{162}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { return err } - scratch := make([]byte, 9) - // t.StartEpoch (abi.ChainEpoch) (int64) if len("StartEpoch") > cbg.MaxLength { return xerrors.Errorf("Value in field \"StartEpoch\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartEpoch"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("StartEpoch"))); err != nil { return err } if _, err := io.WriteString(w, string("StartEpoch")); err != nil { @@ -992,11 +1029,11 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error { } if t.StartEpoch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil { return err } } @@ -1006,7 +1043,7 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"EndEpoch\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("EndEpoch"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("EndEpoch"))); err != nil { return err } if _, err := io.WriteString(w, string("EndEpoch")); err != nil { @@ -1014,27 +1051,32 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error { } if t.EndEpoch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil { return err } } return nil } -func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error { +func (t *DealSchedule) UnmarshalCBOR(r io.Reader) (err error) { *t = DealSchedule{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -1049,7 +1091,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -1061,7 +1103,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error { // t.StartEpoch (abi.ChainEpoch) (int64) case "StartEpoch": { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -1087,7 +1129,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error { // t.EndEpoch (abi.ChainEpoch) (int64) case "EndEpoch": { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 7542ac367..a40d665ce 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -663,6 +663,8 @@ type StorageMinerStruct struct { DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"` + DagstoreRegisterShard func(p0 context.Context, p1 string) error `perm:"admin"` + DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"` DealsConsiderOfflineStorageDeals func(p0 context.Context) (bool, error) `perm:"admin"` @@ -3990,6 +3992,17 @@ func (s *StorageMinerStub) DagstoreRecoverShard(p0 context.Context, p1 string) e return ErrNotSupported } +func (s *StorageMinerStruct) DagstoreRegisterShard(p0 context.Context, p1 string) error { + if s.Internal.DagstoreRegisterShard == nil { + return ErrNotSupported + } + return s.Internal.DagstoreRegisterShard(p0, p1) +} + +func (s *StorageMinerStub) DagstoreRegisterShard(p0 context.Context, p1 string) error { + return ErrNotSupported +} + func (s *StorageMinerStruct) DealsConsiderOfflineRetrievalDeals(p0 context.Context) (bool, error) { if s.Internal.DealsConsiderOfflineRetrievalDeals == nil { return false, ErrNotSupported diff --git a/build/genesis/butterflynet.car b/build/genesis/butterflynet.car index c79eab38e..6ef5243be 100644 Binary files a/build/genesis/butterflynet.car and b/build/genesis/butterflynet.car differ diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 92731a5f6..63589e07a 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index c77d8624f..f9e8ea67e 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index c61688870..cfceac96f 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/build/version.go b/build/version.go index ff1b55fa7..24da6355e 100644 --- a/build/version.go +++ b/build/version.go @@ -37,7 +37,7 @@ func BuildTypeString() string { } // BuildVersion is the local build version -const BuildVersion = "1.15.3-dev" +const BuildVersion = "1.15.4-dev" func UserVersion() string { if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" { diff --git a/chain/exchange/cbor_gen.go b/chain/exchange/cbor_gen.go index 7a8597fd0..9c7f68ab8 100644 --- a/chain/exchange/cbor_gen.go +++ b/chain/exchange/cbor_gen.go @@ -26,51 +26,57 @@ func (t *Request) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufRequest); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufRequest); err != nil { return err } - scratch := make([]byte, 9) - // t.Head ([]cid.Cid) (slice) if len(t.Head) > cbg.MaxLength { return xerrors.Errorf("Slice value in field t.Head was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Head))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Head))); err != nil { return err } for _, v := range t.Head { - if err := cbg.WriteCidBuf(scratch, w, v); err != nil { + if err := cbg.WriteCid(w, v); err != nil { return xerrors.Errorf("failed writing cid field t.Head: %w", err) } } // t.Length (uint64) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Length)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Length)); err != nil { return err } // t.Options (uint64) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Options)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Options)); err != nil { return err } return nil } -func (t *Request) UnmarshalCBOR(r io.Reader) error { +func (t *Request) UnmarshalCBOR(r io.Reader) (err error) { *t = Request{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -81,7 +87,7 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error { // t.Head ([]cid.Cid) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -100,7 +106,7 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("reading cid field t.Head failed: %w", err) } @@ -111,7 +117,7 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -125,7 +131,7 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -145,15 +151,16 @@ func (t *Response) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufResponse); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufResponse); err != nil { return err } - scratch := make([]byte, 9) - // t.Status (exchange.status) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Status)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Status)); err != nil { return err } @@ -162,7 +169,7 @@ func (t *Response) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.ErrorMessage was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.ErrorMessage))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.ErrorMessage))); err != nil { return err } if _, err := io.WriteString(w, string(t.ErrorMessage)); err != nil { @@ -174,27 +181,32 @@ func (t *Response) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Chain was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Chain))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Chain))); err != nil { return err } for _, v := range t.Chain { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } return nil } -func (t *Response) UnmarshalCBOR(r io.Reader) error { +func (t *Response) UnmarshalCBOR(r io.Reader) (err error) { *t = Response{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -207,7 +219,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -220,7 +232,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { // t.ErrorMessage (string) (string) { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -229,7 +241,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { } // t.Chain ([]*exchange.BSTipSet) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -249,7 +261,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v BSTipSet - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -266,22 +278,23 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufCompactedMessages); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufCompactedMessages); err != nil { return err } - scratch := make([]byte, 9) - // t.Bls ([]*types.Message) (slice) if len(t.Bls) > cbg.MaxLength { return xerrors.Errorf("Slice value in field t.Bls was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Bls))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Bls))); err != nil { return err } for _, v := range t.Bls { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } @@ -291,7 +304,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.BlsIncludes was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.BlsIncludes))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.BlsIncludes))); err != nil { return err } for _, v := range t.BlsIncludes { @@ -299,11 +312,11 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field v was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil { return err } for _, v := range v { - if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil { + if err := cw.CborWriteHeader(cbg.MajUnsignedInt, uint64(v)); err != nil { return err } } @@ -314,11 +327,11 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Secpk was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Secpk))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Secpk))); err != nil { return err } for _, v := range t.Secpk { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } @@ -328,7 +341,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.SecpkIncludes was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.SecpkIncludes))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.SecpkIncludes))); err != nil { return err } for _, v := range t.SecpkIncludes { @@ -336,11 +349,11 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field v was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil { return err } for _, v := range v { - if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil { + if err := cw.CborWriteHeader(cbg.MajUnsignedInt, uint64(v)); err != nil { return err } } @@ -348,16 +361,21 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error { return nil } -func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error { +func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) { *t = CompactedMessages{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -368,7 +386,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error { // t.Bls ([]*types.Message) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -388,7 +406,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v types.Message - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -397,7 +415,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error { // t.BlsIncludes ([][]uint64) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -420,7 +438,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error { var extra uint64 var err error - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -439,7 +457,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error { for j := 0; j < int(extra); j++ { - maj, val, err := cbg.CborReadHeaderBuf(br, scratch) + maj, val, err := cr.ReadHeader() if err != nil { return xerrors.Errorf("failed to read uint64 for t.BlsIncludes[i] slice: %w", err) } @@ -456,7 +474,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error { // t.Secpk ([]*types.SignedMessage) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -476,7 +494,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v types.SignedMessage - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -485,7 +503,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error { // t.SecpkIncludes ([][]uint64) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -508,7 +526,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error { var extra uint64 var err error - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -527,7 +545,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error { for j := 0; j < int(extra); j++ { - maj, val, err := cbg.CborReadHeaderBuf(br, scratch) + maj, val, err := cr.ReadHeader() if err != nil { return xerrors.Errorf("failed to read uint64 for t.SecpkIncludes[i] slice: %w", err) } @@ -552,43 +570,49 @@ func (t *BSTipSet) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufBSTipSet); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufBSTipSet); err != nil { return err } - scratch := make([]byte, 9) - // t.Blocks ([]*types.BlockHeader) (slice) if len(t.Blocks) > cbg.MaxLength { return xerrors.Errorf("Slice value in field t.Blocks was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Blocks))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Blocks))); err != nil { return err } for _, v := range t.Blocks { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } // t.Messages (exchange.CompactedMessages) (struct) - if err := t.Messages.MarshalCBOR(w); err != nil { + if err := t.Messages.MarshalCBOR(cw); err != nil { return err } return nil } -func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { +func (t *BSTipSet) UnmarshalCBOR(r io.Reader) (err error) { *t = BSTipSet{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -599,7 +623,7 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { // t.Blocks ([]*types.BlockHeader) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -619,7 +643,7 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v types.BlockHeader - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -630,16 +654,16 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.Messages = new(CompactedMessages) - if err := t.Messages.UnmarshalCBOR(br); err != nil { + if err := t.Messages.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Messages pointer: %w", err) } } diff --git a/chain/market/cbor_gen.go b/chain/market/cbor_gen.go index 1c13e9ddc..9c9ef1a94 100644 --- a/chain/market/cbor_gen.go +++ b/chain/market/cbor_gen.go @@ -25,30 +25,31 @@ func (t *FundedAddressState) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufFundedAddressState); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufFundedAddressState); err != nil { return err } - scratch := make([]byte, 9) - // t.Addr (address.Address) (struct) - if err := t.Addr.MarshalCBOR(w); err != nil { + if err := t.Addr.MarshalCBOR(cw); err != nil { return err } // t.AmtReserved (big.Int) (struct) - if err := t.AmtReserved.MarshalCBOR(w); err != nil { + if err := t.AmtReserved.MarshalCBOR(cw); err != nil { return err } // t.MsgCid (cid.Cid) (struct) if t.MsgCid == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.MsgCid); err != nil { + if err := cbg.WriteCid(cw, *t.MsgCid); err != nil { return xerrors.Errorf("failed to write cid field t.MsgCid: %w", err) } } @@ -56,16 +57,21 @@ func (t *FundedAddressState) MarshalCBOR(w io.Writer) error { return nil } -func (t *FundedAddressState) UnmarshalCBOR(r io.Reader) error { +func (t *FundedAddressState) UnmarshalCBOR(r io.Reader) (err error) { *t = FundedAddressState{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -78,7 +84,7 @@ func (t *FundedAddressState) UnmarshalCBOR(r io.Reader) error { { - if err := t.Addr.UnmarshalCBOR(br); err != nil { + if err := t.Addr.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Addr: %w", err) } @@ -87,7 +93,7 @@ func (t *FundedAddressState) UnmarshalCBOR(r io.Reader) error { { - if err := t.AmtReserved.UnmarshalCBOR(br); err != nil { + if err := t.AmtReserved.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.AmtReserved: %w", err) } @@ -96,16 +102,16 @@ func (t *FundedAddressState) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.MsgCid: %w", err) } diff --git a/chain/store/snapshot.go b/chain/store/snapshot.go index b9630bcbd..07d086e1f 100644 --- a/chain/store/snapshot.go +++ b/chain/store/snapshot.go @@ -8,6 +8,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipld/go-car" carutil "github.com/ipld/go-car/util" + mh "github.com/multiformats/go-multihash" cbg "github.com/whyrusleeping/cbor-gen" "golang.org/x/xerrors" @@ -142,7 +143,18 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe for _, c := range out { if seen.Visit(c) { - if c.Prefix().Codec != cid.DagCBOR { + prefix := c.Prefix() + + // Don't include identity CIDs. + if prefix.MhType == mh.IDENTITY { + continue + } + + // We only include raw and dagcbor, for now. + // Raw for "code" CIDs. + switch prefix.Codec { + case cid.Raw, cid.DagCBOR: + default: continue } diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index b54c18c07..544728a3c 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -29,24 +29,25 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufBlockHeader); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufBlockHeader); err != nil { return err } - scratch := make([]byte, 9) - // t.Miner (address.Address) (struct) - if err := t.Miner.MarshalCBOR(w); err != nil { + if err := t.Miner.MarshalCBOR(cw); err != nil { return err } // t.Ticket (types.Ticket) (struct) - if err := t.Ticket.MarshalCBOR(w); err != nil { + if err := t.Ticket.MarshalCBOR(cw); err != nil { return err } // t.ElectionProof (types.ElectionProof) (struct) - if err := t.ElectionProof.MarshalCBOR(w); err != nil { + if err := t.ElectionProof.MarshalCBOR(cw); err != nil { return err } @@ -55,11 +56,11 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.BeaconEntries was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.BeaconEntries))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.BeaconEntries))); err != nil { return err } for _, v := range t.BeaconEntries { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } @@ -69,11 +70,11 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.WinPoStProof was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.WinPoStProof))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.WinPoStProof))); err != nil { return err } for _, v := range t.WinPoStProof { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } @@ -83,88 +84,93 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Parents was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Parents))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Parents))); err != nil { return err } for _, v := range t.Parents { - if err := cbg.WriteCidBuf(scratch, w, v); err != nil { + if err := cbg.WriteCid(w, v); err != nil { return xerrors.Errorf("failed writing cid field t.Parents: %w", err) } } // t.ParentWeight (big.Int) (struct) - if err := t.ParentWeight.MarshalCBOR(w); err != nil { + if err := t.ParentWeight.MarshalCBOR(cw); err != nil { return err } // t.Height (abi.ChainEpoch) (int64) if t.Height >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Height)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Height)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Height-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Height-1)); err != nil { return err } } // t.ParentStateRoot (cid.Cid) (struct) - if err := cbg.WriteCidBuf(scratch, w, t.ParentStateRoot); err != nil { + if err := cbg.WriteCid(cw, t.ParentStateRoot); err != nil { return xerrors.Errorf("failed to write cid field t.ParentStateRoot: %w", err) } // t.ParentMessageReceipts (cid.Cid) (struct) - if err := cbg.WriteCidBuf(scratch, w, t.ParentMessageReceipts); err != nil { + if err := cbg.WriteCid(cw, t.ParentMessageReceipts); err != nil { return xerrors.Errorf("failed to write cid field t.ParentMessageReceipts: %w", err) } // t.Messages (cid.Cid) (struct) - if err := cbg.WriteCidBuf(scratch, w, t.Messages); err != nil { + if err := cbg.WriteCid(cw, t.Messages); err != nil { return xerrors.Errorf("failed to write cid field t.Messages: %w", err) } // t.BLSAggregate (crypto.Signature) (struct) - if err := t.BLSAggregate.MarshalCBOR(w); err != nil { + if err := t.BLSAggregate.MarshalCBOR(cw); err != nil { return err } // t.Timestamp (uint64) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Timestamp)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Timestamp)); err != nil { return err } // t.BlockSig (crypto.Signature) (struct) - if err := t.BlockSig.MarshalCBOR(w); err != nil { + if err := t.BlockSig.MarshalCBOR(cw); err != nil { return err } // t.ForkSignaling (uint64) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.ForkSignaling)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ForkSignaling)); err != nil { return err } // t.ParentBaseFee (big.Int) (struct) - if err := t.ParentBaseFee.MarshalCBOR(w); err != nil { + if err := t.ParentBaseFee.MarshalCBOR(cw); err != nil { return err } return nil } -func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { +func (t *BlockHeader) UnmarshalCBOR(r io.Reader) (err error) { *t = BlockHeader{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -177,7 +183,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - if err := t.Miner.UnmarshalCBOR(br); err != nil { + if err := t.Miner.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Miner: %w", err) } @@ -186,16 +192,16 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.Ticket = new(Ticket) - if err := t.Ticket.UnmarshalCBOR(br); err != nil { + if err := t.Ticket.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Ticket pointer: %w", err) } } @@ -205,16 +211,16 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.ElectionProof = new(ElectionProof) - if err := t.ElectionProof.UnmarshalCBOR(br); err != nil { + if err := t.ElectionProof.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.ElectionProof pointer: %w", err) } } @@ -222,7 +228,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { } // t.BeaconEntries ([]types.BeaconEntry) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -242,7 +248,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v BeaconEntry - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -251,7 +257,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { // t.WinPoStProof ([]proof.PoStProof) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -271,7 +277,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v proof.PoStProof - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -280,7 +286,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { // t.Parents ([]cid.Cid) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -299,7 +305,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("reading cid field t.Parents failed: %w", err) } @@ -310,14 +316,14 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - if err := t.ParentWeight.UnmarshalCBOR(br); err != nil { + if err := t.ParentWeight.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.ParentWeight: %w", err) } } // t.Height (abi.ChainEpoch) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -344,7 +350,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.ParentStateRoot: %w", err) } @@ -356,7 +362,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.ParentMessageReceipts: %w", err) } @@ -368,7 +374,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.Messages: %w", err) } @@ -380,16 +386,16 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.BLSAggregate = new(crypto.Signature) - if err := t.BLSAggregate.UnmarshalCBOR(br); err != nil { + if err := t.BLSAggregate.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.BLSAggregate pointer: %w", err) } } @@ -399,7 +405,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -413,16 +419,16 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.BlockSig = new(crypto.Signature) - if err := t.BlockSig.UnmarshalCBOR(br); err != nil { + if err := t.BlockSig.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.BlockSig pointer: %w", err) } } @@ -432,7 +438,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -446,7 +452,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { { - if err := t.ParentBaseFee.UnmarshalCBOR(br); err != nil { + if err := t.ParentBaseFee.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.ParentBaseFee: %w", err) } @@ -461,37 +467,43 @@ func (t *Ticket) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufTicket); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufTicket); err != nil { return err } - scratch := make([]byte, 9) - // t.VRFProof ([]uint8) (slice) if len(t.VRFProof) > cbg.ByteArrayMaxLen { return xerrors.Errorf("Byte array in field t.VRFProof was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.VRFProof))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.VRFProof))); err != nil { return err } - if _, err := w.Write(t.VRFProof[:]); err != nil { + if _, err := cw.Write(t.VRFProof[:]); err != nil { return err } return nil } -func (t *Ticket) UnmarshalCBOR(r io.Reader) error { +func (t *Ticket) UnmarshalCBOR(r io.Reader) (err error) { *t = Ticket{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -502,7 +514,7 @@ func (t *Ticket) UnmarshalCBOR(r io.Reader) error { // t.VRFProof ([]uint8) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -518,7 +530,7 @@ func (t *Ticket) UnmarshalCBOR(r io.Reader) error { t.VRFProof = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.VRFProof[:]); err != nil { + if _, err := io.ReadFull(cr, t.VRFProof[:]); err != nil { return err } return nil @@ -531,19 +543,20 @@ func (t *ElectionProof) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufElectionProof); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufElectionProof); err != nil { return err } - scratch := make([]byte, 9) - // t.WinCount (int64) (int64) if t.WinCount >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.WinCount)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.WinCount)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.WinCount-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.WinCount-1)); err != nil { return err } } @@ -553,26 +566,31 @@ func (t *ElectionProof) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.VRFProof was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.VRFProof))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.VRFProof))); err != nil { return err } - if _, err := w.Write(t.VRFProof[:]); err != nil { + if _, err := cw.Write(t.VRFProof[:]); err != nil { return err } return nil } -func (t *ElectionProof) UnmarshalCBOR(r io.Reader) error { +func (t *ElectionProof) UnmarshalCBOR(r io.Reader) (err error) { *t = ElectionProof{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -583,7 +601,7 @@ func (t *ElectionProof) UnmarshalCBOR(r io.Reader) error { // t.WinCount (int64) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -608,7 +626,7 @@ func (t *ElectionProof) UnmarshalCBOR(r io.Reader) error { } // t.VRFProof ([]uint8) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -624,7 +642,7 @@ func (t *ElectionProof) UnmarshalCBOR(r io.Reader) error { t.VRFProof = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.VRFProof[:]); err != nil { + if _, err := io.ReadFull(cr, t.VRFProof[:]); err != nil { return err } return nil @@ -637,63 +655,64 @@ func (t *Message) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufMessage); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufMessage); err != nil { return err } - scratch := make([]byte, 9) - // t.Version (uint64) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Version)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Version)); err != nil { return err } // t.To (address.Address) (struct) - if err := t.To.MarshalCBOR(w); err != nil { + if err := t.To.MarshalCBOR(cw); err != nil { return err } // t.From (address.Address) (struct) - if err := t.From.MarshalCBOR(w); err != nil { + if err := t.From.MarshalCBOR(cw); err != nil { return err } // t.Nonce (uint64) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Nonce)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Nonce)); err != nil { return err } // t.Value (big.Int) (struct) - if err := t.Value.MarshalCBOR(w); err != nil { + if err := t.Value.MarshalCBOR(cw); err != nil { return err } // t.GasLimit (int64) (int64) if t.GasLimit >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.GasLimit)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.GasLimit)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.GasLimit-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.GasLimit-1)); err != nil { return err } } // t.GasFeeCap (big.Int) (struct) - if err := t.GasFeeCap.MarshalCBOR(w); err != nil { + if err := t.GasFeeCap.MarshalCBOR(cw); err != nil { return err } // t.GasPremium (big.Int) (struct) - if err := t.GasPremium.MarshalCBOR(w); err != nil { + if err := t.GasPremium.MarshalCBOR(cw); err != nil { return err } // t.Method (abi.MethodNum) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Method)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Method)); err != nil { return err } @@ -702,26 +721,31 @@ func (t *Message) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.Params was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Params))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Params))); err != nil { return err } - if _, err := w.Write(t.Params[:]); err != nil { + if _, err := cw.Write(t.Params[:]); err != nil { return err } return nil } -func (t *Message) UnmarshalCBOR(r io.Reader) error { +func (t *Message) UnmarshalCBOR(r io.Reader) (err error) { *t = Message{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -734,7 +758,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -748,7 +772,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { { - if err := t.To.UnmarshalCBOR(br); err != nil { + if err := t.To.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.To: %w", err) } @@ -757,7 +781,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { { - if err := t.From.UnmarshalCBOR(br); err != nil { + if err := t.From.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.From: %w", err) } @@ -766,7 +790,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -780,14 +804,14 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { { - if err := t.Value.UnmarshalCBOR(br); err != nil { + if err := t.Value.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Value: %w", err) } } // t.GasLimit (int64) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -814,7 +838,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { { - if err := t.GasFeeCap.UnmarshalCBOR(br); err != nil { + if err := t.GasFeeCap.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.GasFeeCap: %w", err) } @@ -823,7 +847,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { { - if err := t.GasPremium.UnmarshalCBOR(br); err != nil { + if err := t.GasPremium.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.GasPremium: %w", err) } @@ -832,7 +856,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -844,7 +868,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } // t.Params ([]uint8) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -860,7 +884,7 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { t.Params = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.Params[:]); err != nil { + if _, err := io.ReadFull(cr, t.Params[:]); err != nil { return err } return nil @@ -873,32 +897,40 @@ func (t *SignedMessage) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufSignedMessage); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufSignedMessage); err != nil { return err } // t.Message (types.Message) (struct) - if err := t.Message.MarshalCBOR(w); err != nil { + if err := t.Message.MarshalCBOR(cw); err != nil { return err } // t.Signature (crypto.Signature) (struct) - if err := t.Signature.MarshalCBOR(w); err != nil { + if err := t.Signature.MarshalCBOR(cw); err != nil { return err } return nil } -func (t *SignedMessage) UnmarshalCBOR(r io.Reader) error { +func (t *SignedMessage) UnmarshalCBOR(r io.Reader) (err error) { *t = SignedMessage{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -911,7 +943,7 @@ func (t *SignedMessage) UnmarshalCBOR(r io.Reader) error { { - if err := t.Message.UnmarshalCBOR(br); err != nil { + if err := t.Message.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Message: %w", err) } @@ -920,7 +952,7 @@ func (t *SignedMessage) UnmarshalCBOR(r io.Reader) error { { - if err := t.Signature.UnmarshalCBOR(br); err != nil { + if err := t.Signature.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Signature: %w", err) } @@ -935,37 +967,43 @@ func (t *MsgMeta) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufMsgMeta); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufMsgMeta); err != nil { return err } - scratch := make([]byte, 9) - // t.BlsMessages (cid.Cid) (struct) - if err := cbg.WriteCidBuf(scratch, w, t.BlsMessages); err != nil { + if err := cbg.WriteCid(cw, t.BlsMessages); err != nil { return xerrors.Errorf("failed to write cid field t.BlsMessages: %w", err) } // t.SecpkMessages (cid.Cid) (struct) - if err := cbg.WriteCidBuf(scratch, w, t.SecpkMessages); err != nil { + if err := cbg.WriteCid(cw, t.SecpkMessages); err != nil { return xerrors.Errorf("failed to write cid field t.SecpkMessages: %w", err) } return nil } -func (t *MsgMeta) UnmarshalCBOR(r io.Reader) error { +func (t *MsgMeta) UnmarshalCBOR(r io.Reader) (err error) { *t = MsgMeta{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -978,7 +1016,7 @@ func (t *MsgMeta) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.BlsMessages: %w", err) } @@ -990,7 +1028,7 @@ func (t *MsgMeta) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.SecpkMessages: %w", err) } @@ -1008,47 +1046,53 @@ func (t *Actor) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufActor); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufActor); err != nil { return err } - scratch := make([]byte, 9) - // t.Code (cid.Cid) (struct) - if err := cbg.WriteCidBuf(scratch, w, t.Code); err != nil { + if err := cbg.WriteCid(cw, t.Code); err != nil { return xerrors.Errorf("failed to write cid field t.Code: %w", err) } // t.Head (cid.Cid) (struct) - if err := cbg.WriteCidBuf(scratch, w, t.Head); err != nil { + if err := cbg.WriteCid(cw, t.Head); err != nil { return xerrors.Errorf("failed to write cid field t.Head: %w", err) } // t.Nonce (uint64) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Nonce)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Nonce)); err != nil { return err } // t.Balance (big.Int) (struct) - if err := t.Balance.MarshalCBOR(w); err != nil { + if err := t.Balance.MarshalCBOR(cw); err != nil { return err } return nil } -func (t *Actor) UnmarshalCBOR(r io.Reader) error { +func (t *Actor) UnmarshalCBOR(r io.Reader) (err error) { *t = Actor{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -1061,7 +1105,7 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.Code: %w", err) } @@ -1073,7 +1117,7 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.Head: %w", err) } @@ -1085,7 +1129,7 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1099,7 +1143,7 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { { - if err := t.Balance.UnmarshalCBOR(br); err != nil { + if err := t.Balance.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Balance: %w", err) } @@ -1114,19 +1158,20 @@ func (t *MessageReceipt) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufMessageReceipt); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufMessageReceipt); err != nil { return err } - scratch := make([]byte, 9) - // t.ExitCode (exitcode.ExitCode) (int64) if t.ExitCode >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.ExitCode)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ExitCode)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.ExitCode-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.ExitCode-1)); err != nil { return err } } @@ -1136,37 +1181,42 @@ func (t *MessageReceipt) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.Return was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Return))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Return))); err != nil { return err } - if _, err := w.Write(t.Return[:]); err != nil { + if _, err := cw.Write(t.Return[:]); err != nil { return err } // t.GasUsed (int64) (int64) if t.GasUsed >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.GasUsed)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.GasUsed)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.GasUsed-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.GasUsed-1)); err != nil { return err } } return nil } -func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error { +func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) (err error) { *t = MessageReceipt{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -1177,7 +1227,7 @@ func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error { // t.ExitCode (exitcode.ExitCode) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -1202,7 +1252,7 @@ func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error { } // t.Return ([]uint8) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1218,12 +1268,12 @@ func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error { t.Return = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.Return[:]); err != nil { + if _, err := io.ReadFull(cr, t.Return[:]); err != nil { return err } // t.GasUsed (int64) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -1256,14 +1306,15 @@ func (t *BlockMsg) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufBlockMsg); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufBlockMsg); err != nil { return err } - scratch := make([]byte, 9) - // t.Header (types.BlockHeader) (struct) - if err := t.Header.MarshalCBOR(w); err != nil { + if err := t.Header.MarshalCBOR(cw); err != nil { return err } @@ -1272,11 +1323,11 @@ func (t *BlockMsg) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.BlsMessages was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.BlsMessages))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.BlsMessages))); err != nil { return err } for _, v := range t.BlsMessages { - if err := cbg.WriteCidBuf(scratch, w, v); err != nil { + if err := cbg.WriteCid(w, v); err != nil { return xerrors.Errorf("failed writing cid field t.BlsMessages: %w", err) } } @@ -1286,27 +1337,32 @@ func (t *BlockMsg) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.SecpkMessages was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.SecpkMessages))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.SecpkMessages))); err != nil { return err } for _, v := range t.SecpkMessages { - if err := cbg.WriteCidBuf(scratch, w, v); err != nil { + if err := cbg.WriteCid(w, v); err != nil { return xerrors.Errorf("failed writing cid field t.SecpkMessages: %w", err) } } return nil } -func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { +func (t *BlockMsg) UnmarshalCBOR(r io.Reader) (err error) { *t = BlockMsg{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -1319,16 +1375,16 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.Header = new(BlockHeader) - if err := t.Header.UnmarshalCBOR(br); err != nil { + if err := t.Header.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Header pointer: %w", err) } } @@ -1336,7 +1392,7 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { } // t.BlsMessages ([]cid.Cid) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1355,7 +1411,7 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("reading cid field t.BlsMessages failed: %w", err) } @@ -1364,7 +1420,7 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { // t.SecpkMessages ([]cid.Cid) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1383,7 +1439,7 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("reading cid field t.SecpkMessages failed: %w", err) } @@ -1400,22 +1456,23 @@ func (t *ExpTipSet) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufExpTipSet); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufExpTipSet); err != nil { return err } - scratch := make([]byte, 9) - // t.Cids ([]cid.Cid) (slice) if len(t.Cids) > cbg.MaxLength { return xerrors.Errorf("Slice value in field t.Cids was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Cids))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Cids))); err != nil { return err } for _, v := range t.Cids { - if err := cbg.WriteCidBuf(scratch, w, v); err != nil { + if err := cbg.WriteCid(w, v); err != nil { return xerrors.Errorf("failed writing cid field t.Cids: %w", err) } } @@ -1425,38 +1482,43 @@ func (t *ExpTipSet) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Blocks was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Blocks))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Blocks))); err != nil { return err } for _, v := range t.Blocks { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } // t.Height (abi.ChainEpoch) (int64) if t.Height >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Height)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Height)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Height-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Height-1)); err != nil { return err } } return nil } -func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { +func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) (err error) { *t = ExpTipSet{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -1467,7 +1529,7 @@ func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { // t.Cids ([]cid.Cid) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1486,7 +1548,7 @@ func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("reading cid field t.Cids failed: %w", err) } @@ -1495,7 +1557,7 @@ func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { // t.Blocks ([]*types.BlockHeader) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1515,7 +1577,7 @@ func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v BlockHeader - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -1524,7 +1586,7 @@ func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { // t.Height (abi.ChainEpoch) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -1557,15 +1619,16 @@ func (t *BeaconEntry) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufBeaconEntry); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufBeaconEntry); err != nil { return err } - scratch := make([]byte, 9) - // t.Round (uint64) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Round)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Round)); err != nil { return err } @@ -1574,26 +1637,31 @@ func (t *BeaconEntry) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.Data was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Data))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Data))); err != nil { return err } - if _, err := w.Write(t.Data[:]); err != nil { + if _, err := cw.Write(t.Data[:]); err != nil { return err } return nil } -func (t *BeaconEntry) UnmarshalCBOR(r io.Reader) error { +func (t *BeaconEntry) UnmarshalCBOR(r io.Reader) (err error) { *t = BeaconEntry{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -1606,7 +1674,7 @@ func (t *BeaconEntry) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1618,7 +1686,7 @@ func (t *BeaconEntry) UnmarshalCBOR(r io.Reader) error { } // t.Data ([]uint8) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1634,7 +1702,7 @@ func (t *BeaconEntry) UnmarshalCBOR(r io.Reader) error { t.Data = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.Data[:]); err != nil { + if _, err := io.ReadFull(cr, t.Data[:]); err != nil { return err } return nil @@ -1647,43 +1715,49 @@ func (t *StateRoot) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufStateRoot); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufStateRoot); err != nil { return err } - scratch := make([]byte, 9) - // t.Version (types.StateTreeVersion) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Version)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Version)); err != nil { return err } // t.Actors (cid.Cid) (struct) - if err := cbg.WriteCidBuf(scratch, w, t.Actors); err != nil { + if err := cbg.WriteCid(cw, t.Actors); err != nil { return xerrors.Errorf("failed to write cid field t.Actors: %w", err) } // t.Info (cid.Cid) (struct) - if err := cbg.WriteCidBuf(scratch, w, t.Info); err != nil { + if err := cbg.WriteCid(cw, t.Info); err != nil { return xerrors.Errorf("failed to write cid field t.Info: %w", err) } return nil } -func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { +func (t *StateRoot) UnmarshalCBOR(r io.Reader) (err error) { *t = StateRoot{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -1696,7 +1770,7 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1710,7 +1784,7 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.Actors: %w", err) } @@ -1722,7 +1796,7 @@ func (t *StateRoot) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.Info: %w", err) } @@ -1740,23 +1814,30 @@ func (t *StateInfo0) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufStateInfo0); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufStateInfo0); err != nil { return err } - return nil } -func (t *StateInfo0) UnmarshalCBOR(r io.Reader) error { +func (t *StateInfo0) UnmarshalCBOR(r io.Reader) (err error) { *t = StateInfo0{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } diff --git a/chain/vm/cbor_gen.go b/chain/vm/cbor_gen.go index 76d8e9d48..0e1677c95 100644 --- a/chain/vm/cbor_gen.go +++ b/chain/vm/cbor_gen.go @@ -26,19 +26,20 @@ func (t *FvmExecutionTrace) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufFvmExecutionTrace); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufFvmExecutionTrace); err != nil { return err } - scratch := make([]byte, 9) - // t.Msg (types.Message) (struct) - if err := t.Msg.MarshalCBOR(w); err != nil { + if err := t.Msg.MarshalCBOR(cw); err != nil { return err } // t.MsgRct (types.MessageReceipt) (struct) - if err := t.MsgRct.MarshalCBOR(w); err != nil { + if err := t.MsgRct.MarshalCBOR(cw); err != nil { return err } @@ -47,7 +48,7 @@ func (t *FvmExecutionTrace) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.Error was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Error))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Error))); err != nil { return err } if _, err := io.WriteString(w, string(t.Error)); err != nil { @@ -59,27 +60,32 @@ func (t *FvmExecutionTrace) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Subcalls was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Subcalls))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Subcalls))); err != nil { return err } for _, v := range t.Subcalls { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } return nil } -func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error { +func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) (err error) { *t = FvmExecutionTrace{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -92,16 +98,16 @@ func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.Msg = new(types.Message) - if err := t.Msg.UnmarshalCBOR(br); err != nil { + if err := t.Msg.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Msg pointer: %w", err) } } @@ -111,16 +117,16 @@ func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.MsgRct = new(types.MessageReceipt) - if err := t.MsgRct.UnmarshalCBOR(br); err != nil { + if err := t.MsgRct.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.MsgRct pointer: %w", err) } } @@ -129,7 +135,7 @@ func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error { // t.Error (string) (string) { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -138,7 +144,7 @@ func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error { } // t.Subcalls ([]vm.FvmExecutionTrace) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -158,7 +164,7 @@ func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v FvmExecutionTrace - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } diff --git a/cmd/lotus-miner/dagstore.go b/cmd/lotus-miner/dagstore.go index b35dcbe50..37d048825 100644 --- a/cmd/lotus-miner/dagstore.go +++ b/cmd/lotus-miner/dagstore.go @@ -19,6 +19,7 @@ var dagstoreCmd = &cli.Command{ Usage: "Manage the dagstore on the markets subsystem", Subcommands: []*cli.Command{ dagstoreListShardsCmd, + dagstoreRegisterShardCmd, dagstoreInitializeShardCmd, dagstoreRecoverShardCmd, dagstoreInitializeAllCmd, @@ -59,6 +60,45 @@ var dagstoreListShardsCmd = &cli.Command{ }, } +var dagstoreRegisterShardCmd = &cli.Command{ + Name: "register-shard", + ArgsUsage: "[key]", + Usage: "Register a shard", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "color", + Usage: "use color in display output", + DefaultText: "depends on output being a TTY", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.IsSet("color") { + color.NoColor = !cctx.Bool("color") + } + + if cctx.NArg() != 1 { + return fmt.Errorf("must provide a single shard key") + } + + marketsAPI, closer, err := lcli.GetMarketsAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := lcli.ReqContext(cctx) + + shardKey := cctx.Args().First() + err = marketsAPI.DagstoreRegisterShard(ctx, shardKey) + if err != nil { + return err + } + + fmt.Println("Registered shard " + shardKey) + return nil + }, +} + var dagstoreInitializeShardCmd = &cli.Command{ Name: "initialize-shard", ArgsUsage: "[key]", diff --git a/cmd/lotus-miner/precommits-info.go b/cmd/lotus-miner/precommits-info.go new file mode 100644 index 000000000..e700859e9 --- /dev/null +++ b/cmd/lotus-miner/precommits-info.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "sort" + + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/specs-actors/v7/actors/util/adt" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/urfave/cli/v2" +) + +var sectorPreCommitsCmd = &cli.Command{ + Name: "precommits", + Usage: "Print on-chain precommit info", + Action: func(cctx *cli.Context) error { + ctx := lcli.ReqContext(cctx) + mapi, closer, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + maddr, err := getActorAddress(ctx, cctx) + if err != nil { + return err + } + mact, err := mapi.StateGetActor(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(mapi))) + mst, err := miner.Load(store, mact) + if err != nil { + return err + } + preCommitSector := make([]miner.SectorPreCommitOnChainInfo, 0) + err = mst.ForEachPrecommittedSector(func(info miner.SectorPreCommitOnChainInfo) error { + preCommitSector = append(preCommitSector, info) + return err + }) + less := func(i, j int) bool { + return preCommitSector[i].Info.SectorNumber <= preCommitSector[j].Info.SectorNumber + } + sort.Slice(preCommitSector, less) + for _, info := range preCommitSector { + fmt.Printf("%s: %s\n", info.Info.SectorNumber, info.PreCommitEpoch) + } + + return nil + }, +} diff --git a/cmd/lotus-miner/sealing.go b/cmd/lotus-miner/sealing.go index 6cd4f3073..6ee10dacc 100644 --- a/cmd/lotus-miner/sealing.go +++ b/cmd/lotus-miner/sealing.go @@ -4,20 +4,27 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io" "math" + "net/http" "os" "sort" "strings" "text/tabwriter" "time" + "github.com/dustin/go-humanize" "github.com/fatih/color" "github.com/google/uuid" + "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/go-padreader" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/extern/sector-storage/sealtasks" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + "github.com/filecoin-project/lotus/lib/httpreader" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" @@ -31,6 +38,7 @@ var sealingCmd = &cli.Command{ workersCmd(true), sealingSchedDiagCmd, sealingAbortCmd, + sealingDataCidCmd, }, } @@ -411,3 +419,94 @@ var sealingAbortCmd = &cli.Command{ return nodeApi.SealingAbort(ctx, job.ID) }, } + +var sealingDataCidCmd = &cli.Command{ + Name: "data-cid", + Usage: "Compute data CID using workers", + ArgsUsage: "[file/url] ", + Flags: []cli.Flag{ + &cli.Uint64Flag{ + Name: "file-size", + Usage: "real file size", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() < 1 || cctx.Args().Len() > 2 { + return xerrors.Errorf("expected 1 or 2 arguments") + } + + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := lcli.ReqContext(cctx) + + var r io.Reader + sz := cctx.Uint64("file-size") + + if strings.HasPrefix(cctx.Args().First(), "http://") || strings.HasPrefix(cctx.Args().First(), "https://") { + r = &httpreader.HttpReader{ + URL: cctx.Args().First(), + } + + if !cctx.IsSet("file-size") { + resp, err := http.Head(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("http head: %w", err) + } + + if resp.ContentLength < 0 { + return xerrors.Errorf("head response didn't contain content length; specify --file-size") + } + sz = uint64(resp.ContentLength) + } + } else { + p, err := homedir.Expand(cctx.Args().First()) + if err != nil { + return xerrors.Errorf("expanding path: %w", err) + } + + f, err := os.OpenFile(p, os.O_RDONLY, 0) + if err != nil { + return xerrors.Errorf("opening source file: %w", err) + } + + if !cctx.IsSet("file-size") { + st, err := f.Stat() + if err != nil { + return xerrors.Errorf("stat: %w", err) + } + sz = uint64(st.Size()) + } + + r = f + } + + var psize abi.PaddedPieceSize + if cctx.Args().Len() == 2 { + rps, err := humanize.ParseBytes(cctx.Args().Get(1)) + if err != nil { + return xerrors.Errorf("parsing piece size: %w", err) + } + psize = abi.PaddedPieceSize(rps) + if err := psize.Validate(); err != nil { + return xerrors.Errorf("checking piece size: %w", err) + } + if sz > uint64(psize.Unpadded()) { + return xerrors.Errorf("file larger than the piece") + } + } else { + psize = padreader.PaddedSize(sz).Padded() + } + + pc, err := nodeApi.ComputeDataCid(ctx, psize.Unpadded(), r) + if err != nil { + return xerrors.Errorf("computing data CID: %w", err) + } + + fmt.Println(pc.PieceCID, " ", pc.Size) + return nil + }, +} diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go index 1aa964f7e..0918f62f0 100644 --- a/cmd/lotus-miner/sectors.go +++ b/cmd/lotus-miner/sectors.go @@ -45,6 +45,7 @@ var sectorsCmd = &cli.Command{ sectorsRefsCmd, sectorsUpdateCmd, sectorsPledgeCmd, + sectorPreCommitsCmd, sectorsCheckExpireCmd, sectorsExpiredCmd, sectorsRenewCmd, @@ -58,6 +59,7 @@ var sectorsCmd = &cli.Command{ sectorsCapacityCollateralCmd, sectorsBatching, sectorsRefreshPieceMatchingCmd, + sectorsCompactPartitionsCmd, }, } @@ -2088,3 +2090,89 @@ func yesno(b bool) string { } return color.RedString("NO") } + +var sectorsCompactPartitionsCmd = &cli.Command{ + Name: "compact-partitions", + Usage: "removes dead sectors from partitions and reduces the number of partitions used if possible", + Flags: []cli.Flag{ + &cli.Uint64Flag{ + Name: "deadline", + Usage: "the deadline to compact the partitions in", + Required: true, + }, + &cli.Int64SliceFlag{ + Name: "partitions", + Usage: "list of partitions to compact sectors in", + Required: true, + }, + &cli.BoolFlag{ + Name: "really-do-it", + Usage: "Actually send transaction performing the action", + Value: false, + }, + }, + Action: func(cctx *cli.Context) error { + if !cctx.Bool("really-do-it") { + fmt.Println("Pass --really-do-it to actually execute this action") + return nil + } + + api, acloser, err := lcli.GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer acloser() + + ctx := lcli.ReqContext(cctx) + + maddr, err := getActorAddress(ctx, cctx) + if err != nil { + return err + } + + minfo, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) + if err != nil { + return err + } + + deadline := cctx.Uint64("deadline") + if err != nil { + return err + } + + parts := cctx.Int64Slice("partitions") + if len(parts) <= 0 { + return fmt.Errorf("must include at least one partition to compact") + } + + partitions := bitfield.BitField{} + for _, partition := range parts { + partitions.Set(uint64(partition)) + } + + params := miner5.CompactPartitionsParams{ + Deadline: deadline, + Partitions: partitions, + } + + sp, err := actors.SerializeParams(¶ms) + if err != nil { + return xerrors.Errorf("serializing params: %w", err) + } + + smsg, err := api.MpoolPushMessage(ctx, &types.Message{ + From: minfo.Worker, + To: maddr, + Method: miner.Methods.CompactPartitions, + Value: big.Zero(), + Params: sp, + }, nil) + if err != nil { + return xerrors.Errorf("mpool push: %w", err) + } + + fmt.Println("Message CID:", smsg.Cid()) + + return nil + }, +} diff --git a/cmd/lotus-shed/shedgen/cbor_gen.go b/cmd/lotus-shed/shedgen/cbor_gen.go index 37ed95539..a04d52c8e 100644 --- a/cmd/lotus-shed/shedgen/cbor_gen.go +++ b/cmd/lotus-shed/shedgen/cbor_gen.go @@ -23,18 +23,19 @@ func (t *CarbNode) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{161}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { return err } - scratch := make([]byte, 9) - // t.Sub ([]cid.Cid) (slice) if len("Sub") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Sub\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Sub"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Sub"))); err != nil { return err } if _, err := io.WriteString(w, string("Sub")); err != nil { @@ -45,27 +46,32 @@ func (t *CarbNode) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Sub was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Sub))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Sub))); err != nil { return err } for _, v := range t.Sub { - if err := cbg.WriteCidBuf(scratch, w, v); err != nil { + if err := cbg.WriteCid(w, v); err != nil { return xerrors.Errorf("failed writing cid field t.Sub: %w", err) } } return nil } -func (t *CarbNode) UnmarshalCBOR(r io.Reader) error { +func (t *CarbNode) UnmarshalCBOR(r io.Reader) (err error) { *t = CarbNode{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -80,7 +86,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -92,7 +98,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error { // t.Sub ([]cid.Cid) (slice) case "Sub": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -111,7 +117,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("reading cid field t.Sub failed: %w", err) } diff --git a/cmd/lotus-shed/verifreg.go b/cmd/lotus-shed/verifreg.go index fb2598fda..7725b2f1b 100644 --- a/cmd/lotus-shed/verifreg.go +++ b/cmd/lotus-shed/verifreg.go @@ -193,7 +193,7 @@ var verifRegVerifyClientCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`") + fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`") froms := cctx.String("from") if froms == "" { return fmt.Errorf("must specify from address with --from") @@ -262,7 +262,7 @@ var verifRegListVerifiersCmd = &cli.Command{ Usage: "list all verifiers", Hidden: true, Action: func(cctx *cli.Context) error { - fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`") + fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`") api, closer, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err @@ -294,7 +294,7 @@ var verifRegListClientsCmd = &cli.Command{ Usage: "list all verified clients", Hidden: true, Action: func(cctx *cli.Context) error { - fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`") + fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`") api, closer, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err @@ -326,7 +326,7 @@ var verifRegCheckClientCmd = &cli.Command{ Usage: "check verified client remaining bytes", Hidden: true, Action: func(cctx *cli.Context) error { - fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`") + fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`") if !cctx.Args().Present() { return fmt.Errorf("must specify client address to check") } @@ -362,7 +362,7 @@ var verifRegCheckVerifierCmd = &cli.Command{ Usage: "check verifiers remaining bytes", Hidden: true, Action: func(cctx *cli.Context) error { - fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`") + fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`") if !cctx.Args().Present() { return fmt.Errorf("must specify verifier address to check") } diff --git a/cmd/lotus-worker/main.go b/cmd/lotus-worker/main.go index 83c821105..8faabf70b 100644 --- a/cmd/lotus-worker/main.go +++ b/cmd/lotus-worker/main.go @@ -182,12 +182,16 @@ var runCmd = &cli.Command{ Usage: "enable window post", Value: false, }, - &cli.BoolFlag{ Name: "winningpost", Usage: "enable winning post", Value: false, }, + &cli.BoolFlag{ + Name: "no-default", + Usage: "disable all default compute tasks, use the worker for storage/fetching only", + Value: false, + }, &cli.IntFlag{ Name: "parallel-fetch-limit", Usage: "maximum fetch operations to run in parallel", @@ -308,8 +312,11 @@ var runCmd = &cli.Command{ } if workerType == "" { - workerType = sealtasks.WorkerSealing taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFinalizeReplicaUpdate) + + if !cctx.Bool("no-default") { + workerType = sealtasks.WorkerSealing + } } if (workerType == sealtasks.WorkerSealing || cctx.IsSet("addpiece")) && cctx.Bool("addpiece") { @@ -337,6 +344,10 @@ var runCmd = &cli.Command{ taskTypes = append(taskTypes, sealtasks.TTRegenSectorKey) } + if cctx.Bool("no-default") && workerType == "" { + workerType = sealtasks.WorkerSealing + } + if len(taskTypes) == 0 { return xerrors.Errorf("no task types specified") } diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index d0d1ad627..436849cdc 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -28,18 +28,19 @@ func (t *State) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufState); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufState); err != nil { return err } - scratch := make([]byte, 9) - // t.Value (string) (string) if len(t.Value) > cbg.MaxLength { return xerrors.Errorf("Value in field t.Value was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Value))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Value))); err != nil { return err } if _, err := io.WriteString(w, string(t.Value)); err != nil { @@ -51,27 +52,32 @@ func (t *State) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Unmarshallable was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Unmarshallable))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Unmarshallable))); err != nil { return err } for _, v := range t.Unmarshallable { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } return nil } -func (t *State) UnmarshalCBOR(r io.Reader) error { +func (t *State) UnmarshalCBOR(r io.Reader) (err error) { *t = State{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -83,7 +89,7 @@ func (t *State) UnmarshalCBOR(r io.Reader) error { // t.Value (string) (string) { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -92,7 +98,7 @@ func (t *State) UnmarshalCBOR(r io.Reader) error { } // t.Unmarshallable ([]*chaos.UnmarshallableCBOR) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -112,7 +118,7 @@ func (t *State) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v UnmarshallableCBOR - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -129,19 +135,20 @@ func (t *CallerValidationArgs) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufCallerValidationArgs); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufCallerValidationArgs); err != nil { return err } - scratch := make([]byte, 9) - // t.Branch (chaos.CallerValidationBranch) (int64) if t.Branch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Branch)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Branch)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Branch-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Branch-1)); err != nil { return err } } @@ -151,11 +158,11 @@ func (t *CallerValidationArgs) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Addrs was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Addrs))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Addrs))); err != nil { return err } for _, v := range t.Addrs { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } @@ -165,27 +172,32 @@ func (t *CallerValidationArgs) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Types was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Types))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Types))); err != nil { return err } for _, v := range t.Types { - if err := cbg.WriteCidBuf(scratch, w, v); err != nil { + if err := cbg.WriteCid(w, v); err != nil { return xerrors.Errorf("failed writing cid field t.Types: %w", err) } } return nil } -func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error { +func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) (err error) { *t = CallerValidationArgs{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -196,7 +208,7 @@ func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error { // t.Branch (chaos.CallerValidationBranch) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -221,7 +233,7 @@ func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error { } // t.Addrs ([]address.Address) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -241,7 +253,7 @@ func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v address.Address - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -250,7 +262,7 @@ func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error { // t.Types ([]cid.Cid) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -269,7 +281,7 @@ func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("reading cid field t.Types failed: %w", err) } @@ -286,12 +298,13 @@ func (t *CreateActorArgs) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufCreateActorArgs); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufCreateActorArgs); err != nil { return err } - scratch := make([]byte, 9) - // t.UndefActorCID (bool) (bool) if err := cbg.WriteBool(w, t.UndefActorCID); err != nil { return err @@ -299,7 +312,7 @@ func (t *CreateActorArgs) MarshalCBOR(w io.Writer) error { // t.ActorCID (cid.Cid) (struct) - if err := cbg.WriteCidBuf(scratch, w, t.ActorCID); err != nil { + if err := cbg.WriteCid(cw, t.ActorCID); err != nil { return xerrors.Errorf("failed to write cid field t.ActorCID: %w", err) } @@ -309,22 +322,27 @@ func (t *CreateActorArgs) MarshalCBOR(w io.Writer) error { } // t.Address (address.Address) (struct) - if err := t.Address.MarshalCBOR(w); err != nil { + if err := t.Address.MarshalCBOR(cw); err != nil { return err } return nil } -func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) error { +func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) (err error) { *t = CreateActorArgs{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -335,7 +353,7 @@ func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) error { // t.UndefActorCID (bool) (bool) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -354,7 +372,7 @@ func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.ActorCID: %w", err) } @@ -364,7 +382,7 @@ func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) error { } // t.UndefAddress (bool) (bool) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -383,7 +401,7 @@ func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) error { { - if err := t.Address.UnmarshalCBOR(br); err != nil { + if err := t.Address.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Address: %w", err) } @@ -398,12 +416,15 @@ func (t *ResolveAddressResponse) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufResolveAddressResponse); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufResolveAddressResponse); err != nil { return err } // t.Address (address.Address) (struct) - if err := t.Address.MarshalCBOR(w); err != nil { + if err := t.Address.MarshalCBOR(cw); err != nil { return err } @@ -414,16 +435,21 @@ func (t *ResolveAddressResponse) MarshalCBOR(w io.Writer) error { return nil } -func (t *ResolveAddressResponse) UnmarshalCBOR(r io.Reader) error { +func (t *ResolveAddressResponse) UnmarshalCBOR(r io.Reader) (err error) { *t = ResolveAddressResponse{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -436,14 +462,14 @@ func (t *ResolveAddressResponse) UnmarshalCBOR(r io.Reader) error { { - if err := t.Address.UnmarshalCBOR(br); err != nil { + if err := t.Address.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Address: %w", err) } } // t.Success (bool) (bool) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -468,25 +494,26 @@ func (t *SendArgs) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufSendArgs); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufSendArgs); err != nil { return err } - scratch := make([]byte, 9) - // t.To (address.Address) (struct) - if err := t.To.MarshalCBOR(w); err != nil { + if err := t.To.MarshalCBOR(cw); err != nil { return err } // t.Value (big.Int) (struct) - if err := t.Value.MarshalCBOR(w); err != nil { + if err := t.Value.MarshalCBOR(cw); err != nil { return err } // t.Method (abi.MethodNum) (uint64) - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Method)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Method)); err != nil { return err } @@ -495,26 +522,31 @@ func (t *SendArgs) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.Params was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Params))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Params))); err != nil { return err } - if _, err := w.Write(t.Params[:]); err != nil { + if _, err := cw.Write(t.Params[:]); err != nil { return err } return nil } -func (t *SendArgs) UnmarshalCBOR(r io.Reader) error { +func (t *SendArgs) UnmarshalCBOR(r io.Reader) (err error) { *t = SendArgs{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -527,7 +559,7 @@ func (t *SendArgs) UnmarshalCBOR(r io.Reader) error { { - if err := t.To.UnmarshalCBOR(br); err != nil { + if err := t.To.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.To: %w", err) } @@ -536,7 +568,7 @@ func (t *SendArgs) UnmarshalCBOR(r io.Reader) error { { - if err := t.Value.UnmarshalCBOR(br); err != nil { + if err := t.Value.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Value: %w", err) } @@ -545,7 +577,7 @@ func (t *SendArgs) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -557,7 +589,7 @@ func (t *SendArgs) UnmarshalCBOR(r io.Reader) error { } // t.Params ([]uint8) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -573,7 +605,7 @@ func (t *SendArgs) UnmarshalCBOR(r io.Reader) error { t.Params = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.Params[:]); err != nil { + if _, err := io.ReadFull(cr, t.Params[:]); err != nil { return err } return nil @@ -586,48 +618,54 @@ func (t *SendReturn) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufSendReturn); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufSendReturn); err != nil { return err } - scratch := make([]byte, 9) - // t.Return (builtin.CBORBytes) (slice) if len(t.Return) > cbg.ByteArrayMaxLen { return xerrors.Errorf("Byte array in field t.Return was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Return))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Return))); err != nil { return err } - if _, err := w.Write(t.Return[:]); err != nil { + if _, err := cw.Write(t.Return[:]); err != nil { return err } // t.Code (exitcode.ExitCode) (int64) if t.Code >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Code)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Code)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Code-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Code-1)); err != nil { return err } } return nil } -func (t *SendReturn) UnmarshalCBOR(r io.Reader) error { +func (t *SendReturn) UnmarshalCBOR(r io.Reader) (err error) { *t = SendReturn{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -638,7 +676,7 @@ func (t *SendReturn) UnmarshalCBOR(r io.Reader) error { // t.Return (builtin.CBORBytes) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -654,12 +692,12 @@ func (t *SendReturn) UnmarshalCBOR(r io.Reader) error { t.Return = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.Return[:]); err != nil { + if _, err := io.ReadFull(cr, t.Return[:]); err != nil { return err } // t.Code (exitcode.ExitCode) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -692,18 +730,19 @@ func (t *MutateStateArgs) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufMutateStateArgs); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufMutateStateArgs); err != nil { return err } - scratch := make([]byte, 9) - // t.Value (string) (string) if len(t.Value) > cbg.MaxLength { return xerrors.Errorf("Value in field t.Value was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Value))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Value))); err != nil { return err } if _, err := io.WriteString(w, string(t.Value)); err != nil { @@ -712,27 +751,32 @@ func (t *MutateStateArgs) MarshalCBOR(w io.Writer) error { // t.Branch (chaos.MutateStateBranch) (int64) if t.Branch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Branch)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Branch)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Branch-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Branch-1)); err != nil { return err } } return nil } -func (t *MutateStateArgs) UnmarshalCBOR(r io.Reader) error { +func (t *MutateStateArgs) UnmarshalCBOR(r io.Reader) (err error) { *t = MutateStateArgs{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -744,7 +788,7 @@ func (t *MutateStateArgs) UnmarshalCBOR(r io.Reader) error { // t.Value (string) (string) { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -753,7 +797,7 @@ func (t *MutateStateArgs) UnmarshalCBOR(r io.Reader) error { } // t.Branch (chaos.MutateStateBranch) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -786,19 +830,20 @@ func (t *AbortWithArgs) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufAbortWithArgs); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufAbortWithArgs); err != nil { return err } - scratch := make([]byte, 9) - // t.Code (exitcode.ExitCode) (int64) if t.Code >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Code)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Code)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Code-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Code-1)); err != nil { return err } } @@ -808,7 +853,7 @@ func (t *AbortWithArgs) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.Message was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Message))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Message))); err != nil { return err } if _, err := io.WriteString(w, string(t.Message)); err != nil { @@ -822,16 +867,21 @@ func (t *AbortWithArgs) MarshalCBOR(w io.Writer) error { return nil } -func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error { +func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) (err error) { *t = AbortWithArgs{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -842,7 +892,7 @@ func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error { // t.Code (exitcode.ExitCode) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -868,7 +918,7 @@ func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error { // t.Message (string) (string) { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -877,7 +927,7 @@ func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error { } // t.Uncontrolled (bool) (bool) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -902,60 +952,66 @@ func (t *InspectRuntimeReturn) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufInspectRuntimeReturn); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufInspectRuntimeReturn); err != nil { return err } - scratch := make([]byte, 9) - // t.Caller (address.Address) (struct) - if err := t.Caller.MarshalCBOR(w); err != nil { + if err := t.Caller.MarshalCBOR(cw); err != nil { return err } // t.Receiver (address.Address) (struct) - if err := t.Receiver.MarshalCBOR(w); err != nil { + if err := t.Receiver.MarshalCBOR(cw); err != nil { return err } // t.ValueReceived (big.Int) (struct) - if err := t.ValueReceived.MarshalCBOR(w); err != nil { + if err := t.ValueReceived.MarshalCBOR(cw); err != nil { return err } // t.CurrEpoch (abi.ChainEpoch) (int64) if t.CurrEpoch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.CurrEpoch)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.CurrEpoch)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.CurrEpoch-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.CurrEpoch-1)); err != nil { return err } } // t.CurrentBalance (big.Int) (struct) - if err := t.CurrentBalance.MarshalCBOR(w); err != nil { + if err := t.CurrentBalance.MarshalCBOR(cw); err != nil { return err } // t.State (chaos.State) (struct) - if err := t.State.MarshalCBOR(w); err != nil { + if err := t.State.MarshalCBOR(cw); err != nil { return err } return nil } -func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { +func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) (err error) { *t = InspectRuntimeReturn{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -968,7 +1024,7 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { { - if err := t.Caller.UnmarshalCBOR(br); err != nil { + if err := t.Caller.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Caller: %w", err) } @@ -977,7 +1033,7 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { { - if err := t.Receiver.UnmarshalCBOR(br); err != nil { + if err := t.Receiver.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Receiver: %w", err) } @@ -986,14 +1042,14 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { { - if err := t.ValueReceived.UnmarshalCBOR(br); err != nil { + if err := t.ValueReceived.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.ValueReceived: %w", err) } } // t.CurrEpoch (abi.ChainEpoch) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -1020,7 +1076,7 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { { - if err := t.CurrentBalance.UnmarshalCBOR(br); err != nil { + if err := t.CurrentBalance.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.CurrentBalance: %w", err) } @@ -1029,7 +1085,7 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { { - if err := t.State.UnmarshalCBOR(br); err != nil { + if err := t.State.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.State: %w", err) } diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index dfcfb281c..492bb0055 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -27,6 +27,7 @@ * [DagstoreListShards](#DagstoreListShards) * [DagstoreLookupPieces](#DagstoreLookupPieces) * [DagstoreRecoverShard](#DagstoreRecoverShard) + * [DagstoreRegisterShard](#DagstoreRegisterShard) * [Deals](#Deals) * [DealsConsiderOfflineRetrievalDeals](#DealsConsiderOfflineRetrievalDeals) * [DealsConsiderOfflineStorageDeals](#DealsConsiderOfflineStorageDeals) @@ -639,6 +640,21 @@ Inputs: Response: `{}` +### DagstoreRegisterShard +DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID + + +Perms: admin + +Inputs: +```json +[ + "string value" +] +``` + +Response: `{}` + ## Deals diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md index 5be59a1b6..4122026f8 100644 --- a/documentation/en/cli-lotus-miner.md +++ b/documentation/en/cli-lotus-miner.md @@ -7,7 +7,7 @@ USAGE: lotus-miner [global options] command [command options] [arguments...] VERSION: - 1.15.3-dev + 1.15.4-dev COMMANDS: init Initialize a lotus miner repo @@ -1035,6 +1035,7 @@ USAGE: COMMANDS: list-shards List all shards known to the dagstore, with their current status + register-shard Register a shard initialize-shard Initialize the specified shard recover-shard Attempt to recover a shard in errored state initialize-all Initialize all uninitialized shards, streaming results as they're produced; only shards for unsealed pieces are initialized by default @@ -1061,6 +1062,20 @@ OPTIONS: ``` +### lotus-miner dagstore register-shard +``` +NAME: + lotus-miner dagstore register-shard - Register a shard + +USAGE: + lotus-miner dagstore register-shard [command options] [key] + +OPTIONS: + --color use color in display output (default: depends on output being a TTY) + --help, -h show help (default: false) + +``` + ### lotus-miner dagstore initialize-shard ``` NAME: @@ -1652,6 +1667,7 @@ COMMANDS: refs List References to sectors update-state ADVANCED: manually update the state of a sector, this may aid in error recovery pledge store random data in a sector + precommits Print on-chain precommit info check-expire Inspect expiring sectors expired Get or cleanup expired sectors renew Renew expiring sectors while not exceeding each sector's max life @@ -1665,6 +1681,7 @@ COMMANDS: get-cc-collateral Get the collateral required to pledge a committed capacity sector batching manage batch sector operations match-pending-pieces force a refreshed match of pending pieces to open sectors without manually waiting for more deals + compact-partitions removes dead sectors from partitions and reduces the number of partitions used if possible help, h Shows a list of commands or help for one command OPTIONS: @@ -1750,6 +1767,19 @@ OPTIONS: ``` +### lotus-miner sectors precommits +``` +NAME: + lotus-miner sectors precommits - Print on-chain precommit info + +USAGE: + lotus-miner sectors precommits [command options] [arguments...] + +OPTIONS: + --help, -h show help (default: false) + +``` + ### lotus-miner sectors check-expire ``` NAME: @@ -2006,6 +2036,22 @@ OPTIONS: ``` +### lotus-miner sectors compact-partitions +``` +NAME: + lotus-miner sectors compact-partitions - removes dead sectors from partitions and reduces the number of partitions used if possible + +USAGE: + lotus-miner sectors compact-partitions [command options] [arguments...] + +OPTIONS: + --deadline value the deadline to compact the partitions in (default: 0) + --partitions value list of partitions to compact sectors in + --really-do-it Actually send transaction performing the action (default: false) + --help, -h show help (default: false) + +``` + ## lotus-miner proving ``` NAME: @@ -2285,6 +2331,7 @@ COMMANDS: workers list workers sched-diag Dump internal scheduler state abort Abort a running job + data-cid Compute data CID using workers help, h Shows a list of commands or help for one command OPTIONS: @@ -2347,3 +2394,17 @@ OPTIONS: --help, -h show help (default: false) ``` + +### lotus-miner sealing data-cid +``` +NAME: + lotus-miner sealing data-cid - Compute data CID using workers + +USAGE: + lotus-miner sealing data-cid [command options] [file/url] + +OPTIONS: + --file-size value real file size (default: 0) + --help, -h show help (default: false) + +``` diff --git a/documentation/en/cli-lotus-worker.md b/documentation/en/cli-lotus-worker.md index 68d0d45c2..c99417d42 100644 --- a/documentation/en/cli-lotus-worker.md +++ b/documentation/en/cli-lotus-worker.md @@ -7,7 +7,7 @@ USAGE: lotus-worker [global options] command [command options] [arguments...] VERSION: - 1.15.3-dev + 1.15.4-dev COMMANDS: run Start lotus worker @@ -49,6 +49,7 @@ OPTIONS: --regen-sector-key enable regen sector key (default: true) --windowpost enable window post (default: false) --winningpost enable winning post (default: false) + --no-default disable all default compute tasks, use the worker for storage/fetching only (default: false) --parallel-fetch-limit value maximum fetch operations to run in parallel (default: 5) --post-parallel-reads value maximum number of parallel challenge reads (0 = no limit) (default: 128) --post-read-timeout value time limit for reading PoSt challenges (0 = no limit) (default: 0s) diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 9a87a5008..d1f798f5f 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -7,7 +7,7 @@ USAGE: lotus [global options] command [command options] [arguments...] VERSION: - 1.15.3-dev + 1.15.4-dev COMMANDS: daemon Start a lotus daemon process diff --git a/documentation/en/default-lotus-miner-config.toml b/documentation/en/default-lotus-miner-config.toml index 1e1b0369d..ef3b23b5c 100644 --- a/documentation/en/default-lotus-miner-config.toml +++ b/documentation/en/default-lotus-miner-config.toml @@ -444,12 +444,14 @@ # env var: LOTUS_SEALING_AGGREGATECOMMITS #AggregateCommits = true - # maximum batched commit size - batches will be sent immediately above this size + # minimum batched commit size - batches above this size will eventually be sent on a timeout # # type: int # env var: LOTUS_SEALING_MINCOMMITBATCH #MinCommitBatch = 4 + # maximum batched commit size - batches will be sent immediately above this size + # # type: int # env var: LOTUS_SEALING_MAXCOMMITBATCH #MaxCommitBatch = 819 diff --git a/extern/sector-storage/cbor_gen.go b/extern/sector-storage/cbor_gen.go index 623d75b33..fd99576cc 100644 --- a/extern/sector-storage/cbor_gen.go +++ b/extern/sector-storage/cbor_gen.go @@ -24,25 +24,26 @@ func (t *Call) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{164}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { return err } - scratch := make([]byte, 9) - // t.ID (storiface.CallID) (struct) if len("ID") > cbg.MaxLength { return xerrors.Errorf("Value in field \"ID\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ID"))); err != nil { return err } if _, err := io.WriteString(w, string("ID")); err != nil { return err } - if err := t.ID.MarshalCBOR(w); err != nil { + if err := t.ID.MarshalCBOR(cw); err != nil { return err } @@ -51,7 +52,7 @@ func (t *Call) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"RetType\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("RetType"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RetType"))); err != nil { return err } if _, err := io.WriteString(w, string("RetType")); err != nil { @@ -62,7 +63,7 @@ func (t *Call) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.RetType was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.RetType))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.RetType))); err != nil { return err } if _, err := io.WriteString(w, string(t.RetType)); err != nil { @@ -74,14 +75,14 @@ func (t *Call) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"State\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("State"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("State"))); err != nil { return err } if _, err := io.WriteString(w, string("State")); err != nil { return err } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.State)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.State)); err != nil { return err } @@ -90,29 +91,34 @@ func (t *Call) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Result\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Result"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Result"))); err != nil { return err } if _, err := io.WriteString(w, string("Result")); err != nil { return err } - if err := t.Result.MarshalCBOR(w); err != nil { + if err := t.Result.MarshalCBOR(cw); err != nil { return err } return nil } -func (t *Call) UnmarshalCBOR(r io.Reader) error { +func (t *Call) UnmarshalCBOR(r io.Reader) (err error) { *t = Call{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -127,7 +133,7 @@ func (t *Call) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -141,7 +147,7 @@ func (t *Call) UnmarshalCBOR(r io.Reader) error { { - if err := t.ID.UnmarshalCBOR(br); err != nil { + if err := t.ID.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.ID: %w", err) } @@ -150,7 +156,7 @@ func (t *Call) UnmarshalCBOR(r io.Reader) error { case "RetType": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -162,7 +168,7 @@ func (t *Call) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -177,16 +183,16 @@ func (t *Call) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.Result = new(ManyBytes) - if err := t.Result.UnmarshalCBOR(br); err != nil { + if err := t.Result.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Result pointer: %w", err) } } @@ -206,25 +212,26 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{166}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{166}); err != nil { return err } - scratch := make([]byte, 9) - // t.ID (sectorstorage.WorkID) (struct) if len("ID") > cbg.MaxLength { return xerrors.Errorf("Value in field \"ID\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ID"))); err != nil { return err } if _, err := io.WriteString(w, string("ID")); err != nil { return err } - if err := t.ID.MarshalCBOR(w); err != nil { + if err := t.ID.MarshalCBOR(cw); err != nil { return err } @@ -233,7 +240,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Status\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Status"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Status"))); err != nil { return err } if _, err := io.WriteString(w, string("Status")); err != nil { @@ -244,7 +251,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.Status was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Status))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Status))); err != nil { return err } if _, err := io.WriteString(w, string(t.Status)); err != nil { @@ -256,14 +263,14 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"WorkerCall\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WorkerCall"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("WorkerCall"))); err != nil { return err } if _, err := io.WriteString(w, string("WorkerCall")); err != nil { return err } - if err := t.WorkerCall.MarshalCBOR(w); err != nil { + if err := t.WorkerCall.MarshalCBOR(cw); err != nil { return err } @@ -272,7 +279,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"WorkError\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WorkError"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("WorkError"))); err != nil { return err } if _, err := io.WriteString(w, string("WorkError")); err != nil { @@ -283,7 +290,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.WorkError was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.WorkError))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.WorkError))); err != nil { return err } if _, err := io.WriteString(w, string(t.WorkError)); err != nil { @@ -295,7 +302,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"WorkerHostname\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WorkerHostname"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("WorkerHostname"))); err != nil { return err } if _, err := io.WriteString(w, string("WorkerHostname")); err != nil { @@ -306,7 +313,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.WorkerHostname was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.WorkerHostname))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.WorkerHostname))); err != nil { return err } if _, err := io.WriteString(w, string(t.WorkerHostname)); err != nil { @@ -318,7 +325,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"StartTime\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartTime"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("StartTime"))); err != nil { return err } if _, err := io.WriteString(w, string("StartTime")); err != nil { @@ -326,27 +333,32 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error { } if t.StartTime >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartTime)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.StartTime)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartTime-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.StartTime-1)); err != nil { return err } } return nil } -func (t *WorkState) UnmarshalCBOR(r io.Reader) error { +func (t *WorkState) UnmarshalCBOR(r io.Reader) (err error) { *t = WorkState{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -361,7 +373,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -375,7 +387,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error { { - if err := t.ID.UnmarshalCBOR(br); err != nil { + if err := t.ID.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.ID: %w", err) } @@ -384,7 +396,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error { case "Status": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -396,7 +408,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error { { - if err := t.WorkerCall.UnmarshalCBOR(br); err != nil { + if err := t.WorkerCall.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.WorkerCall: %w", err) } @@ -405,7 +417,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error { case "WorkError": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -416,7 +428,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error { case "WorkerHostname": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -426,7 +438,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error { // t.StartTime (int64) (int64) case "StartTime": { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -463,18 +475,19 @@ func (t *WorkID) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{162}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { return err } - scratch := make([]byte, 9) - // t.Method (sealtasks.TaskType) (string) if len("Method") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Method\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Method"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Method"))); err != nil { return err } if _, err := io.WriteString(w, string("Method")); err != nil { @@ -485,7 +498,7 @@ func (t *WorkID) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.Method was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Method))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Method))); err != nil { return err } if _, err := io.WriteString(w, string(t.Method)); err != nil { @@ -497,7 +510,7 @@ func (t *WorkID) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Params\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Params"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Params"))); err != nil { return err } if _, err := io.WriteString(w, string("Params")); err != nil { @@ -508,7 +521,7 @@ func (t *WorkID) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.Params was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Params))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Params))); err != nil { return err } if _, err := io.WriteString(w, string(t.Params)); err != nil { @@ -517,16 +530,21 @@ func (t *WorkID) MarshalCBOR(w io.Writer) error { return nil } -func (t *WorkID) UnmarshalCBOR(r io.Reader) error { +func (t *WorkID) UnmarshalCBOR(r io.Reader) (err error) { *t = WorkID{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -541,7 +559,7 @@ func (t *WorkID) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -554,7 +572,7 @@ func (t *WorkID) UnmarshalCBOR(r io.Reader) error { case "Method": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -565,7 +583,7 @@ func (t *WorkID) UnmarshalCBOR(r io.Reader) error { case "Params": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } diff --git a/extern/sector-storage/ffiwrapper/sealer_cgo.go b/extern/sector-storage/ffiwrapper/sealer_cgo.go index 400b67211..d63e83c6e 100644 --- a/extern/sector-storage/ffiwrapper/sealer_cgo.go +++ b/extern/sector-storage/ffiwrapper/sealer_cgo.go @@ -31,6 +31,8 @@ import ( "github.com/filecoin-project/lotus/extern/sector-storage/fr32" "github.com/filecoin-project/lotus/extern/sector-storage/partialfile" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" + nr "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" + "github.com/filecoin-project/lotus/lib/nullreader" ) var _ Storage = &Sealer{} @@ -52,6 +54,11 @@ func (sb *Sealer) NewSector(ctx context.Context, sector storage.SectorRef) error } func (sb *Sealer) DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) { + pieceData = io.LimitReader(io.MultiReader( + pieceData, + nullreader.Reader{}, + ), int64(pieceSize)) + // TODO: allow tuning those: chunk := abi.PaddedPieceSize(4 << 20) parallel := runtime.NumCPU() @@ -72,6 +79,7 @@ func (sb *Sealer) DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, for { var read int for rbuf := buf; len(rbuf) > 0; { + n, err := pieceData.Read(rbuf) if err != nil && err != io.EOF { return abi.PieceInfo{}, xerrors.Errorf("pr read error: %w", err) @@ -369,8 +377,8 @@ func (sb *Sealer) pieceCid(spt abi.RegisteredSealProof, in []byte) (cid.Cid, err return pieceCID, werr() } -func (sb *Sealer) tryDecodeUpdatedReplica(ctx context.Context, sector storage.SectorRef, commD cid.Cid, unsealedPath string) (bool, error) { - paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUpdate|storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage) +func (sb *Sealer) tryDecodeUpdatedReplica(ctx context.Context, sector storage.SectorRef, commD cid.Cid, unsealedPath string, randomness abi.SealRandomness) (bool, error) { + replicaPath, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTNone, storiface.PathStorage) if xerrors.Is(err, storiface.ErrSectorNotFound) { return false, nil } else if err != nil { @@ -378,12 +386,47 @@ func (sb *Sealer) tryDecodeUpdatedReplica(ctx context.Context, sector storage.Se } defer done() + sealedPaths, done2, err := sb.AcquireSectorKeyOrRegenerate(ctx, sector, randomness) + if err != nil { + return false, xerrors.Errorf("acquiring sealed sector: %w", err) + } + defer done2() + // Sector data stored in replica update updateProof, err := sector.ProofType.RegisteredUpdateProof() if err != nil { return false, err } - return true, ffi.SectorUpdate.DecodeFrom(updateProof, unsealedPath, paths.Update, paths.Sealed, paths.Cache, commD) + return true, ffi.SectorUpdate.DecodeFrom(updateProof, unsealedPath, replicaPath.Update, sealedPaths.Sealed, sealedPaths.Cache, commD) +} + +func (sb *Sealer) AcquireSectorKeyOrRegenerate(ctx context.Context, sector storage.SectorRef, randomness abi.SealRandomness) (storiface.SectorPaths, func(), error) { + paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage) + if err == nil { + return paths, done, err + } else if !xerrors.Is(err, storiface.ErrSectorNotFound) { + return paths, done, xerrors.Errorf("reading sector key: %w", err) + } + + // Sector key can't be found, so let's regenerate it + sectorSize, err := sector.ProofType.SectorSize() + if err != nil { + return paths, done, xerrors.Errorf("retrieving sector size: %w", err) + } + paddedSize := abi.PaddedPieceSize(sectorSize) + + _, err = sb.AddPiece(ctx, sector, nil, paddedSize.Unpadded(), nr.NewNullReader(paddedSize.Unpadded())) + if err != nil { + return paths, done, xerrors.Errorf("recomputing empty data: %w", err) + } + + err = sb.RegenerateSectorKey(ctx, sector, randomness, []abi.PieceInfo{{PieceCID: zerocomm.ZeroPieceCommitment(paddedSize.Unpadded()), Size: paddedSize}}) + if err != nil { + return paths, done, xerrors.Errorf("during pc1: %w", err) + } + + // Sector key should exist now, let's grab the paths + return sb.sectors.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage) } func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd cid.Cid) error { @@ -437,7 +480,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off } // If piece data stored in updated replica decode whole sector - decoded, err := sb.tryDecodeUpdatedReplica(ctx, sector, commd, unsealedPath.Unsealed) + decoded, err := sb.tryDecodeUpdatedReplica(ctx, sector, commd, unsealedPath.Unsealed, randomness) if err != nil { return xerrors.Errorf("decoding sector from replica: %w", err) } @@ -618,6 +661,51 @@ func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector storag return true, nil } +func (sb *Sealer) RegenerateSectorKey(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) error { + paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed|storiface.FTCache, storiface.FTSealed, storiface.PathSealing) + if err != nil { + return xerrors.Errorf("acquiring sector paths: %w", err) + } + defer done() + + e, err := os.OpenFile(paths.Sealed, os.O_RDWR|os.O_CREATE, 0644) // nolint:gosec + if err != nil { + return xerrors.Errorf("ensuring sealed file exists: %w", err) + } + if err := e.Close(); err != nil { + return err + } + + var sum abi.UnpaddedPieceSize + for _, piece := range pieces { + sum += piece.Size.Unpadded() + } + ssize, err := sector.ProofType.SectorSize() + if err != nil { + return err + } + ussize := abi.PaddedPieceSize(ssize).Unpadded() + if sum != ussize { + return xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize-sum)) + } + + // TODO: context cancellation respect + _, err = ffi.SealPreCommitPhase1( + sector.ProofType, + paths.Cache, + paths.Unsealed, + paths.Sealed, + sector.ID.Number, + sector.ID.Miner, + ticket, + pieces, + ) + if err != nil { + return xerrors.Errorf("presealing sector %d (%s): %w", sector.ID.Number, paths.Unsealed, err) + } + return nil +} + func (sb *Sealer) SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) { paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed, storiface.FTSealed|storiface.FTCache, storiface.PathSealing) if err != nil { diff --git a/extern/sector-storage/manager.go b/extern/sector-storage/manager.go index 4b52f9a1d..8a41aa8f1 100644 --- a/extern/sector-storage/manager.go +++ b/extern/sector-storage/manager.go @@ -273,7 +273,7 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storage.SectorR defer cancel() log.Debugf("acquire unseal sector lock for sector %d", sector.ID) - if err := m.index.StorageLock(ctx, sector.ID, storiface.FTSealed|storiface.FTCache, storiface.FTUnsealed); err != nil { + if err := m.index.StorageLock(ctx, sector.ID, storiface.FTSealed|storiface.FTCache|storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTUnsealed); err != nil { return xerrors.Errorf("acquiring unseal sector lock: %w", err) } @@ -281,8 +281,11 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storage.SectorR // put it in the sealing scratch space. sealFetch := func(ctx context.Context, worker Worker) error { log.Debugf("copy sealed/cache sector data for sector %d", sector.ID) - if _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy)); err != nil { - return xerrors.Errorf("copy sealed/cache sector data: %w", err) + _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy)) + _, err2 := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, storiface.AcquireCopy)) + + if err != nil && err2 != nil { + return xerrors.Errorf("cannot unseal piece. error fetching sealed data: %w. error fetching replica data: %w", err, err2) } return nil diff --git a/extern/sector-storage/manager_test.go b/extern/sector-storage/manager_test.go index 9c844292e..d9f8af461 100644 --- a/extern/sector-storage/manager_test.go +++ b/extern/sector-storage/manager_test.go @@ -315,6 +315,12 @@ func TestSnapDeals(t *testing.T) { require.NoError(t, m.GenerateSectorKeyFromData(ctx, sid, out.NewUnsealed)) fmt.Printf("GSK duration (%s): %s\n", ss.ShortString(), time.Since(startGSK)) + fmt.Printf("Remove data\n") + require.NoError(t, m.FinalizeSector(ctx, sid, nil)) + fmt.Printf("Release Sector Key\n") + require.NoError(t, m.ReleaseSectorKey(ctx, sid)) + fmt.Printf("Unseal Replica\n") + require.NoError(t, m.SectorsUnsealPiece(ctx, sid, 0, p1.Size.Unpadded(), ticket, &out.NewUnsealed)) } func TestRedoPC1(t *testing.T) { diff --git a/extern/sector-storage/storiface/cbor_gen.go b/extern/sector-storage/storiface/cbor_gen.go index 44b5ae9ee..5b4623175 100644 --- a/extern/sector-storage/storiface/cbor_gen.go +++ b/extern/sector-storage/storiface/cbor_gen.go @@ -23,25 +23,26 @@ func (t *CallID) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{162}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { return err } - scratch := make([]byte, 9) - // t.Sector (abi.SectorID) (struct) if len("Sector") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Sector\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Sector"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Sector"))); err != nil { return err } if _, err := io.WriteString(w, string("Sector")); err != nil { return err } - if err := t.Sector.MarshalCBOR(w); err != nil { + if err := t.Sector.MarshalCBOR(cw); err != nil { return err } @@ -50,7 +51,7 @@ func (t *CallID) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"ID\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ID"))); err != nil { return err } if _, err := io.WriteString(w, string("ID")); err != nil { @@ -61,26 +62,31 @@ func (t *CallID) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.ID was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.ID))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.ID))); err != nil { return err } - if _, err := w.Write(t.ID[:]); err != nil { + if _, err := cw.Write(t.ID[:]); err != nil { return err } return nil } -func (t *CallID) UnmarshalCBOR(r io.Reader) error { +func (t *CallID) UnmarshalCBOR(r io.Reader) (err error) { *t = CallID{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -95,7 +101,7 @@ func (t *CallID) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -109,7 +115,7 @@ func (t *CallID) UnmarshalCBOR(r io.Reader) error { { - if err := t.Sector.UnmarshalCBOR(br); err != nil { + if err := t.Sector.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Sector: %w", err) } @@ -117,7 +123,7 @@ func (t *CallID) UnmarshalCBOR(r io.Reader) error { // t.ID (uuid.UUID) (array) case "ID": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -135,7 +141,7 @@ func (t *CallID) UnmarshalCBOR(r io.Reader) error { t.ID = [16]uint8{} - if _, err := io.ReadFull(br, t.ID[:]); err != nil { + if _, err := io.ReadFull(cr, t.ID[:]); err != nil { return err } diff --git a/extern/sector-storage/worker_local.go b/extern/sector-storage/worker_local.go index 9a14e42b5..355915a34 100644 --- a/extern/sector-storage/worker_local.go +++ b/extern/sector-storage/worker_local.go @@ -289,7 +289,7 @@ func (l *LocalWorker) asyncCall(ctx context.Context, sector storage.SectorRef, r log.Errorf("get hostname err: %+v", err) } - err = xerrors.Errorf("%w [Hostname: %s]", err.Error(), hostname) + err = xerrors.Errorf("%w [Hostname: %s]", err, hostname) } if doReturn(ctx, rt, ci, l.ret, res, toCallError(err)) { diff --git a/extern/storage-sealing/cbor_gen.go b/extern/storage-sealing/cbor_gen.go index c1e2b08fa..8794fdf4d 100644 --- a/extern/storage-sealing/cbor_gen.go +++ b/extern/storage-sealing/cbor_gen.go @@ -26,25 +26,26 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{162}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { return err } - scratch := make([]byte, 9) - // t.Piece (abi.PieceInfo) (struct) if len("Piece") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Piece\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Piece"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Piece"))); err != nil { return err } if _, err := io.WriteString(w, string("Piece")); err != nil { return err } - if err := t.Piece.MarshalCBOR(w); err != nil { + if err := t.Piece.MarshalCBOR(cw); err != nil { return err } @@ -53,29 +54,34 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"DealInfo\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealInfo"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealInfo"))); err != nil { return err } if _, err := io.WriteString(w, string("DealInfo")); err != nil { return err } - if err := t.DealInfo.MarshalCBOR(w); err != nil { + if err := t.DealInfo.MarshalCBOR(cw); err != nil { return err } return nil } -func (t *Piece) UnmarshalCBOR(r io.Reader) error { +func (t *Piece) UnmarshalCBOR(r io.Reader) (err error) { *t = Piece{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -90,7 +96,7 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -104,7 +110,7 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { { - if err := t.Piece.UnmarshalCBOR(br); err != nil { + if err := t.Piece.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Piece: %w", err) } @@ -114,16 +120,16 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.DealInfo = new(api.PieceDealInfo) - if err := t.DealInfo.UnmarshalCBOR(br); err != nil { + if err := t.DealInfo.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.DealInfo pointer: %w", err) } } @@ -143,18 +149,19 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{184, 32}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{184, 32}); err != nil { return err } - scratch := make([]byte, 9) - // t.State (sealing.SectorState) (string) if len("State") > cbg.MaxLength { return xerrors.Errorf("Value in field \"State\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("State"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("State"))); err != nil { return err } if _, err := io.WriteString(w, string("State")); err != nil { @@ -165,7 +172,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.State was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.State))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.State))); err != nil { return err } if _, err := io.WriteString(w, string(t.State)); err != nil { @@ -177,14 +184,14 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"SectorNumber\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorNumber"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("SectorNumber"))); err != nil { return err } if _, err := io.WriteString(w, string("SectorNumber")); err != nil { return err } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SectorNumber)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.SectorNumber)); err != nil { return err } @@ -193,7 +200,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"SectorType\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorType"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("SectorType"))); err != nil { return err } if _, err := io.WriteString(w, string("SectorType")); err != nil { @@ -201,11 +208,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.SectorType >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SectorType)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.SectorType)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.SectorType-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.SectorType-1)); err != nil { return err } } @@ -215,7 +222,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"CreationTime\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("CreationTime"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("CreationTime"))); err != nil { return err } if _, err := io.WriteString(w, string("CreationTime")); err != nil { @@ -223,11 +230,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.CreationTime >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.CreationTime)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.CreationTime)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.CreationTime-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.CreationTime-1)); err != nil { return err } } @@ -237,7 +244,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Pieces\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Pieces"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Pieces"))); err != nil { return err } if _, err := io.WriteString(w, string("Pieces")); err != nil { @@ -248,11 +255,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Pieces was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Pieces))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Pieces))); err != nil { return err } for _, v := range t.Pieces { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } @@ -262,7 +269,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"TicketValue\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("TicketValue"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("TicketValue"))); err != nil { return err } if _, err := io.WriteString(w, string("TicketValue")); err != nil { @@ -273,11 +280,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.TicketValue was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.TicketValue))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.TicketValue))); err != nil { return err } - if _, err := w.Write(t.TicketValue[:]); err != nil { + if _, err := cw.Write(t.TicketValue[:]); err != nil { return err } @@ -286,7 +293,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"TicketEpoch\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("TicketEpoch"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("TicketEpoch"))); err != nil { return err } if _, err := io.WriteString(w, string("TicketEpoch")); err != nil { @@ -294,11 +301,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.TicketEpoch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.TicketEpoch)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.TicketEpoch)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.TicketEpoch-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.TicketEpoch-1)); err != nil { return err } } @@ -308,7 +315,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"PreCommit1Out\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PreCommit1Out"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PreCommit1Out"))); err != nil { return err } if _, err := io.WriteString(w, string("PreCommit1Out")); err != nil { @@ -319,11 +326,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.PreCommit1Out was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.PreCommit1Out))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.PreCommit1Out))); err != nil { return err } - if _, err := w.Write(t.PreCommit1Out[:]); err != nil { + if _, err := cw.Write(t.PreCommit1Out[:]); err != nil { return err } @@ -332,7 +339,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"CommD\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("CommD"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("CommD"))); err != nil { return err } if _, err := io.WriteString(w, string("CommD")); err != nil { @@ -340,11 +347,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.CommD == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.CommD); err != nil { + if err := cbg.WriteCid(cw, *t.CommD); err != nil { return xerrors.Errorf("failed to write cid field t.CommD: %w", err) } } @@ -354,7 +361,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"CommR\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("CommR"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("CommR"))); err != nil { return err } if _, err := io.WriteString(w, string("CommR")); err != nil { @@ -362,11 +369,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.CommR == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.CommR); err != nil { + if err := cbg.WriteCid(cw, *t.CommR); err != nil { return xerrors.Errorf("failed to write cid field t.CommR: %w", err) } } @@ -376,7 +383,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Proof\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Proof"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Proof"))); err != nil { return err } if _, err := io.WriteString(w, string("Proof")); err != nil { @@ -387,11 +394,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.Proof was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Proof))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Proof))); err != nil { return err } - if _, err := w.Write(t.Proof[:]); err != nil { + if _, err := cw.Write(t.Proof[:]); err != nil { return err } @@ -400,14 +407,14 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"PreCommitInfo\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PreCommitInfo"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PreCommitInfo"))); err != nil { return err } if _, err := io.WriteString(w, string("PreCommitInfo")); err != nil { return err } - if err := t.PreCommitInfo.MarshalCBOR(w); err != nil { + if err := t.PreCommitInfo.MarshalCBOR(cw); err != nil { return err } @@ -416,14 +423,14 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"PreCommitDeposit\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PreCommitDeposit"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PreCommitDeposit"))); err != nil { return err } if _, err := io.WriteString(w, string("PreCommitDeposit")); err != nil { return err } - if err := t.PreCommitDeposit.MarshalCBOR(w); err != nil { + if err := t.PreCommitDeposit.MarshalCBOR(cw); err != nil { return err } @@ -432,7 +439,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"PreCommitMessage\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PreCommitMessage"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PreCommitMessage"))); err != nil { return err } if _, err := io.WriteString(w, string("PreCommitMessage")); err != nil { @@ -440,11 +447,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.PreCommitMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.PreCommitMessage); err != nil { + if err := cbg.WriteCid(cw, *t.PreCommitMessage); err != nil { return xerrors.Errorf("failed to write cid field t.PreCommitMessage: %w", err) } } @@ -454,7 +461,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"PreCommitTipSet\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PreCommitTipSet"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PreCommitTipSet"))); err != nil { return err } if _, err := io.WriteString(w, string("PreCommitTipSet")); err != nil { @@ -465,11 +472,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.PreCommitTipSet was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.PreCommitTipSet))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.PreCommitTipSet))); err != nil { return err } - if _, err := w.Write(t.PreCommitTipSet[:]); err != nil { + if _, err := cw.Write(t.PreCommitTipSet[:]); err != nil { return err } @@ -478,14 +485,14 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"PreCommit2Fails\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PreCommit2Fails"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PreCommit2Fails"))); err != nil { return err } if _, err := io.WriteString(w, string("PreCommit2Fails")); err != nil { return err } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.PreCommit2Fails)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.PreCommit2Fails)); err != nil { return err } @@ -494,7 +501,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"SeedValue\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SeedValue"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("SeedValue"))); err != nil { return err } if _, err := io.WriteString(w, string("SeedValue")); err != nil { @@ -505,11 +512,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.SeedValue was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.SeedValue))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.SeedValue))); err != nil { return err } - if _, err := w.Write(t.SeedValue[:]); err != nil { + if _, err := cw.Write(t.SeedValue[:]); err != nil { return err } @@ -518,7 +525,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"SeedEpoch\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SeedEpoch"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("SeedEpoch"))); err != nil { return err } if _, err := io.WriteString(w, string("SeedEpoch")); err != nil { @@ -526,11 +533,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.SeedEpoch >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SeedEpoch)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.SeedEpoch)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.SeedEpoch-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.SeedEpoch-1)); err != nil { return err } } @@ -540,7 +547,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"CommitMessage\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("CommitMessage"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("CommitMessage"))); err != nil { return err } if _, err := io.WriteString(w, string("CommitMessage")); err != nil { @@ -548,11 +555,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.CommitMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.CommitMessage); err != nil { + if err := cbg.WriteCid(cw, *t.CommitMessage); err != nil { return xerrors.Errorf("failed to write cid field t.CommitMessage: %w", err) } } @@ -562,14 +569,14 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"InvalidProofs\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("InvalidProofs"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("InvalidProofs"))); err != nil { return err } if _, err := io.WriteString(w, string("InvalidProofs")); err != nil { return err } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.InvalidProofs)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.InvalidProofs)); err != nil { return err } @@ -578,7 +585,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"CCUpdate\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("CCUpdate"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("CCUpdate"))); err != nil { return err } if _, err := io.WriteString(w, string("CCUpdate")); err != nil { @@ -594,7 +601,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"CCPieces\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("CCPieces"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("CCPieces"))); err != nil { return err } if _, err := io.WriteString(w, string("CCPieces")); err != nil { @@ -605,11 +612,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.CCPieces was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.CCPieces))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.CCPieces))); err != nil { return err } for _, v := range t.CCPieces { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } @@ -619,7 +626,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"UpdateSealed\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("UpdateSealed"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("UpdateSealed"))); err != nil { return err } if _, err := io.WriteString(w, string("UpdateSealed")); err != nil { @@ -627,11 +634,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.UpdateSealed == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.UpdateSealed); err != nil { + if err := cbg.WriteCid(cw, *t.UpdateSealed); err != nil { return xerrors.Errorf("failed to write cid field t.UpdateSealed: %w", err) } } @@ -641,7 +648,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"UpdateUnsealed\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("UpdateUnsealed"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("UpdateUnsealed"))); err != nil { return err } if _, err := io.WriteString(w, string("UpdateUnsealed")); err != nil { @@ -649,11 +656,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.UpdateUnsealed == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.UpdateUnsealed); err != nil { + if err := cbg.WriteCid(cw, *t.UpdateUnsealed); err != nil { return xerrors.Errorf("failed to write cid field t.UpdateUnsealed: %w", err) } } @@ -663,7 +670,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"ReplicaUpdateProof\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ReplicaUpdateProof"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ReplicaUpdateProof"))); err != nil { return err } if _, err := io.WriteString(w, string("ReplicaUpdateProof")); err != nil { @@ -674,11 +681,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.ReplicaUpdateProof was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.ReplicaUpdateProof))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.ReplicaUpdateProof))); err != nil { return err } - if _, err := w.Write(t.ReplicaUpdateProof[:]); err != nil { + if _, err := cw.Write(t.ReplicaUpdateProof[:]); err != nil { return err } @@ -687,7 +694,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"ReplicaUpdateMessage\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ReplicaUpdateMessage"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ReplicaUpdateMessage"))); err != nil { return err } if _, err := io.WriteString(w, string("ReplicaUpdateMessage")); err != nil { @@ -695,11 +702,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.ReplicaUpdateMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.ReplicaUpdateMessage); err != nil { + if err := cbg.WriteCid(cw, *t.ReplicaUpdateMessage); err != nil { return xerrors.Errorf("failed to write cid field t.ReplicaUpdateMessage: %w", err) } } @@ -709,7 +716,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"FaultReportMsg\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("FaultReportMsg"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("FaultReportMsg"))); err != nil { return err } if _, err := io.WriteString(w, string("FaultReportMsg")); err != nil { @@ -717,11 +724,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.FaultReportMsg == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.FaultReportMsg); err != nil { + if err := cbg.WriteCid(cw, *t.FaultReportMsg); err != nil { return xerrors.Errorf("failed to write cid field t.FaultReportMsg: %w", err) } } @@ -731,7 +738,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Return\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Return"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Return"))); err != nil { return err } if _, err := io.WriteString(w, string("Return")); err != nil { @@ -742,7 +749,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.Return was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Return))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Return))); err != nil { return err } if _, err := io.WriteString(w, string(t.Return)); err != nil { @@ -754,7 +761,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"TerminateMessage\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("TerminateMessage"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("TerminateMessage"))); err != nil { return err } if _, err := io.WriteString(w, string("TerminateMessage")); err != nil { @@ -762,11 +769,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.TerminateMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.TerminateMessage); err != nil { + if err := cbg.WriteCid(cw, *t.TerminateMessage); err != nil { return xerrors.Errorf("failed to write cid field t.TerminateMessage: %w", err) } } @@ -776,7 +783,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"TerminatedAt\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("TerminatedAt"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("TerminatedAt"))); err != nil { return err } if _, err := io.WriteString(w, string("TerminatedAt")); err != nil { @@ -784,11 +791,11 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } if t.TerminatedAt >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.TerminatedAt)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.TerminatedAt)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.TerminatedAt-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.TerminatedAt-1)); err != nil { return err } } @@ -798,7 +805,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"LastErr\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("LastErr"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LastErr"))); err != nil { return err } if _, err := io.WriteString(w, string("LastErr")); err != nil { @@ -809,7 +816,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.LastErr was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.LastErr))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LastErr))); err != nil { return err } if _, err := io.WriteString(w, string(t.LastErr)); err != nil { @@ -821,7 +828,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Log\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Log"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Log"))); err != nil { return err } if _, err := io.WriteString(w, string("Log")); err != nil { @@ -832,27 +839,32 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Log was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Log))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Log))); err != nil { return err } for _, v := range t.Log { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } return nil } -func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { +func (t *SectorInfo) UnmarshalCBOR(r io.Reader) (err error) { *t = SectorInfo{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -867,7 +879,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -880,7 +892,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { case "State": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -892,7 +904,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -905,7 +917,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.SectorType (abi.RegisteredSealProof) (int64) case "SectorType": { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -931,7 +943,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.CreationTime (int64) (int64) case "CreationTime": { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -957,7 +969,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.Pieces ([]sealing.Piece) (slice) case "Pieces": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -977,7 +989,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v Piece - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -987,7 +999,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.TicketValue (abi.SealRandomness) (slice) case "TicketValue": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1003,13 +1015,13 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { t.TicketValue = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.TicketValue[:]); err != nil { + if _, err := io.ReadFull(cr, t.TicketValue[:]); err != nil { return err } // t.TicketEpoch (abi.ChainEpoch) (int64) case "TicketEpoch": { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -1035,7 +1047,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.PreCommit1Out (storage.PreCommit1Out) (slice) case "PreCommit1Out": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1051,7 +1063,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { t.PreCommit1Out = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.PreCommit1Out[:]); err != nil { + if _, err := io.ReadFull(cr, t.PreCommit1Out[:]); err != nil { return err } // t.CommD (cid.Cid) (struct) @@ -1059,16 +1071,16 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.CommD: %w", err) } @@ -1082,16 +1094,16 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.CommR: %w", err) } @@ -1103,7 +1115,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.Proof ([]uint8) (slice) case "Proof": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1119,7 +1131,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { t.Proof = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.Proof[:]); err != nil { + if _, err := io.ReadFull(cr, t.Proof[:]); err != nil { return err } // t.PreCommitInfo (miner.SectorPreCommitInfo) (struct) @@ -1127,16 +1139,16 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.PreCommitInfo = new(miner.SectorPreCommitInfo) - if err := t.PreCommitInfo.UnmarshalCBOR(br); err != nil { + if err := t.PreCommitInfo.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.PreCommitInfo pointer: %w", err) } } @@ -1147,7 +1159,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - if err := t.PreCommitDeposit.UnmarshalCBOR(br); err != nil { + if err := t.PreCommitDeposit.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.PreCommitDeposit: %w", err) } @@ -1157,16 +1169,16 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.PreCommitMessage: %w", err) } @@ -1178,7 +1190,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.PreCommitTipSet (sealing.TipSetToken) (slice) case "PreCommitTipSet": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1194,7 +1206,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { t.PreCommitTipSet = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.PreCommitTipSet[:]); err != nil { + if _, err := io.ReadFull(cr, t.PreCommitTipSet[:]); err != nil { return err } // t.PreCommit2Fails (uint64) (uint64) @@ -1202,7 +1214,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1215,7 +1227,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.SeedValue (abi.InteractiveSealRandomness) (slice) case "SeedValue": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1231,13 +1243,13 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { t.SeedValue = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.SeedValue[:]); err != nil { + if _, err := io.ReadFull(cr, t.SeedValue[:]); err != nil { return err } // t.SeedEpoch (abi.ChainEpoch) (int64) case "SeedEpoch": { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -1265,16 +1277,16 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err) } @@ -1288,7 +1300,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1301,7 +1313,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.CCUpdate (bool) (bool) case "CCUpdate": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1319,7 +1331,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.CCPieces ([]sealing.Piece) (slice) case "CCPieces": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1339,7 +1351,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v Piece - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -1351,16 +1363,16 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.UpdateSealed: %w", err) } @@ -1374,16 +1386,16 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.UpdateUnsealed: %w", err) } @@ -1395,7 +1407,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.ReplicaUpdateProof (storage.ReplicaUpdateProof) (slice) case "ReplicaUpdateProof": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1411,7 +1423,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { t.ReplicaUpdateProof = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.ReplicaUpdateProof[:]); err != nil { + if _, err := io.ReadFull(cr, t.ReplicaUpdateProof[:]); err != nil { return err } // t.ReplicaUpdateMessage (cid.Cid) (struct) @@ -1419,16 +1431,16 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.ReplicaUpdateMessage: %w", err) } @@ -1442,16 +1454,16 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.FaultReportMsg: %w", err) } @@ -1464,7 +1476,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { case "Return": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -1476,16 +1488,16 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.TerminateMessage: %w", err) } @@ -1497,7 +1509,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.TerminatedAt (abi.ChainEpoch) (int64) case "TerminatedAt": { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -1524,7 +1536,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { case "LastErr": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -1534,7 +1546,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.Log ([]sealing.Log) (slice) case "Log": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1554,7 +1566,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v Log - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -1574,25 +1586,26 @@ func (t *Log) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{164}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { return err } - scratch := make([]byte, 9) - // t.Timestamp (uint64) (uint64) if len("Timestamp") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Timestamp\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Timestamp"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Timestamp"))); err != nil { return err } if _, err := io.WriteString(w, string("Timestamp")); err != nil { return err } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Timestamp)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Timestamp)); err != nil { return err } @@ -1601,7 +1614,7 @@ func (t *Log) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Trace\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Trace"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Trace"))); err != nil { return err } if _, err := io.WriteString(w, string("Trace")); err != nil { @@ -1612,7 +1625,7 @@ func (t *Log) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.Trace was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Trace))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Trace))); err != nil { return err } if _, err := io.WriteString(w, string(t.Trace)); err != nil { @@ -1624,7 +1637,7 @@ func (t *Log) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Message\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Message"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Message"))); err != nil { return err } if _, err := io.WriteString(w, string("Message")); err != nil { @@ -1635,7 +1648,7 @@ func (t *Log) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.Message was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Message))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Message))); err != nil { return err } if _, err := io.WriteString(w, string(t.Message)); err != nil { @@ -1647,7 +1660,7 @@ func (t *Log) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Kind\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Kind"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Kind"))); err != nil { return err } if _, err := io.WriteString(w, string("Kind")); err != nil { @@ -1658,7 +1671,7 @@ func (t *Log) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.Kind was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Kind))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Kind))); err != nil { return err } if _, err := io.WriteString(w, string(t.Kind)); err != nil { @@ -1667,16 +1680,21 @@ func (t *Log) MarshalCBOR(w io.Writer) error { return nil } -func (t *Log) UnmarshalCBOR(r io.Reader) error { +func (t *Log) UnmarshalCBOR(r io.Reader) (err error) { *t = Log{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -1691,7 +1709,7 @@ func (t *Log) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -1705,7 +1723,7 @@ func (t *Log) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -1719,7 +1737,7 @@ func (t *Log) UnmarshalCBOR(r io.Reader) error { case "Trace": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -1730,7 +1748,7 @@ func (t *Log) UnmarshalCBOR(r io.Reader) error { case "Message": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -1741,7 +1759,7 @@ func (t *Log) UnmarshalCBOR(r io.Reader) error { case "Kind": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } diff --git a/extern/storage-sealing/input.go b/extern/storage-sealing/input.go index 945de51b7..ca7ae7dd3 100644 --- a/extern/storage-sealing/input.go +++ b/extern/storage-sealing/input.go @@ -22,6 +22,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" "github.com/filecoin-project/lotus/extern/storage-sealing/sealiface" ) @@ -220,7 +221,7 @@ func (m *Sealing) handleAddPiece(ctx statemachine.Context, sector SectorInfo) er m.minerSector(sector.SectorType, sector.SectorNumber), pieceSizes, p.Unpadded(), - NewNullReader(p.Unpadded())) + nullreader.NewNullReader(p.Unpadded())) if err != nil { err = xerrors.Errorf("writing padding piece: %w", err) deal.accepted(sector.SectorNumber, offset, err) diff --git a/extern/storage-sealing/lib/nullreader/nullreader.go b/extern/storage-sealing/lib/nullreader/nullreader.go index dc3537ad7..9e4bc523c 100644 --- a/extern/storage-sealing/lib/nullreader/nullreader.go +++ b/extern/storage-sealing/lib/nullreader/nullreader.go @@ -1,5 +1,23 @@ package nullreader +import ( + "io" + + "github.com/filecoin-project/go-state-types/abi" +) + +type NullReader struct { + *io.LimitedReader +} + +func NewNullReader(size abi.UnpaddedPieceSize) io.Reader { + return &NullReader{(io.LimitReader(&Reader{}, int64(size))).(*io.LimitedReader)} +} + +func (m NullReader) NullBytes() int64 { + return m.N +} + // TODO: extract this to someplace where it can be shared with lotus type Reader struct{} diff --git a/extern/storage-sealing/nullreader.go b/extern/storage-sealing/nullreader.go deleted file mode 100644 index 5987a4145..000000000 --- a/extern/storage-sealing/nullreader.go +++ /dev/null @@ -1,20 +0,0 @@ -package sealing - -import ( - "io" - - "github.com/filecoin-project/go-state-types/abi" - nr "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" -) - -type NullReader struct { - *io.LimitedReader -} - -func NewNullReader(size abi.UnpaddedPieceSize) io.Reader { - return &NullReader{(io.LimitReader(&nr.Reader{}, int64(size))).(*io.LimitedReader)} -} - -func (m NullReader) NullBytes() int64 { - return m.N -} diff --git a/extern/storage-sealing/states_sealing.go b/extern/storage-sealing/states_sealing.go index 1b2556943..d20bbf696 100644 --- a/extern/storage-sealing/states_sealing.go +++ b/extern/storage-sealing/states_sealing.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/actors/policy" + "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" ) var DealSectorPriority = 1024 @@ -91,7 +92,7 @@ func (m *Sealing) padSector(ctx context.Context, sectorID storage.SectorRef, exi for i, size := range sizes { expectCid := zerocomm.ZeroPieceCommitment(size) - ppi, err := m.sealer.AddPiece(ctx, sectorID, existingPieceSizes, size, NewNullReader(size)) + ppi, err := m.sealer.AddPiece(ctx, sectorID, existingPieceSizes, size, nullreader.NewNullReader(size)) if err != nil { return nil, xerrors.Errorf("add piece: %w", err) } diff --git a/go.mod b/go.mod index e11e69a80..dc25cfa19 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/filecoin-project/go-legs v0.3.7 github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-paramfetch v0.0.4 - github.com/filecoin-project/go-state-types v0.1.3 + github.com/filecoin-project/go-state-types v0.1.4 github.com/filecoin-project/go-statemachine v1.0.2 github.com/filecoin-project/go-statestore v0.2.0 github.com/filecoin-project/go-storedcounter v0.1.0 @@ -109,7 +109,7 @@ require ( github.com/koalacxr/quantile v0.0.1 github.com/libp2p/go-buffer-pool v0.0.2 github.com/libp2p/go-eventbus v0.2.1 - github.com/libp2p/go-libp2p v0.19.0 + github.com/libp2p/go-libp2p v0.19.3 github.com/libp2p/go-libp2p-connmgr v0.3.1 github.com/libp2p/go-libp2p-core v0.15.1 github.com/libp2p/go-libp2p-discovery v0.6.0 @@ -145,7 +145,7 @@ require ( github.com/syndtr/goleveldb v1.0.0 github.com/urfave/cli/v2 v2.3.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14 + github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799 github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 @@ -277,7 +277,7 @@ require ( github.com/libp2p/go-tcp-transport v0.5.1 // indirect github.com/libp2p/go-ws-transport v0.6.0 // indirect github.com/libp2p/go-yamux/v3 v3.1.1 // indirect - github.com/lucas-clemente/quic-go v0.27.0 // indirect + github.com/lucas-clemente/quic-go v0.27.1 // indirect github.com/lucasb-eyer/go-colorful v1.0.3 // indirect github.com/magefile/mage v1.9.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect diff --git a/go.sum b/go.sum index 5112c7bd3..56211d9df 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,9 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.1-0.20210810190654-139e0e79e69e/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.1/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= -github.com/filecoin-project/go-state-types v0.1.3 h1:rzIJyQo5HO2ptc8Jcu8P0qTutnI7NWwTle54eAHoNO0= github.com/filecoin-project/go-state-types v0.1.3/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.1.4 h1:NU0veVNxtDiLD/eRyKHV9lv3njSzfTh/sJGxxvcYcpg= +github.com/filecoin-project/go-state-types v0.1.4/go.mod h1:xCA/WfKlC2zcn3fUmDv4IrzznwS98X5XW/irUP3Lhxg= github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= github.com/filecoin-project/go-statemachine v1.0.1/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= github.com/filecoin-project/go-statemachine v1.0.2-0.20220322104818-27f8fbb86dfd/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= @@ -1086,8 +1087,8 @@ github.com/libp2p/go-libp2p v0.16.0/go.mod h1:ump42BsirwAWxKzsCiFnTtN1Yc+DuPu76f github.com/libp2p/go-libp2p v0.17.0/go.mod h1:Fkin50rsGdv5mm5BshBUtPRZknt9esfmYXBOYcwOTgw= github.com/libp2p/go-libp2p v0.18.0-rc1/go.mod h1:RgYlH7IIWHXREimC92bw5Lg1V2R5XmSzuLHb5fTnr+8= github.com/libp2p/go-libp2p v0.18.0-rc3/go.mod h1:WYL+Xw1iuwi6rdfzw5VIEpD+HqzYucHZ6fcUuumbI3M= -github.com/libp2p/go-libp2p v0.19.0 h1:zosskMbaobL7UDCVLEe1m5CGs1TaFNFoN/M5XLiKg0U= -github.com/libp2p/go-libp2p v0.19.0/go.mod h1:Ki9jJXLO2YqrTIFxofV7Twyd3INWPT97+r8hGt7XPjI= +github.com/libp2p/go-libp2p v0.19.3 h1:LqjvuBWdyYSqvkH4VVYxA78Fkphzg2Pq86VMnilqgkw= +github.com/libp2p/go-libp2p v0.19.3/go.mod h1:AGlPVLjh0+6jvEtf+a2gZEux7yHJrYXnG9IC7wcQ2NY= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= github.com/libp2p/go-libp2p-asn-util v0.1.0 h1:rABPCO77SjdbJ/eJ/ynIo8vWICy1VEnL5JAxJbQLo1E= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= @@ -1457,8 +1458,9 @@ github.com/lucas-clemente/quic-go v0.21.2/go.mod h1:vF5M1XqhBAHgbjKcJOXY3JZz3GP0 github.com/lucas-clemente/quic-go v0.23.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= github.com/lucas-clemente/quic-go v0.24.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg= -github.com/lucas-clemente/quic-go v0.27.0 h1:v6WY87q9zD4dKASbG8hy/LpzAVNzEQzw8sEIeloJsc4= github.com/lucas-clemente/quic-go v0.27.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= +github.com/lucas-clemente/quic-go v0.27.1 h1:sOw+4kFSVrdWOYmUjufQ9GBVPqZ+tu+jMtXxXNmRJyk= +github.com/lucas-clemente/quic-go v0.27.1/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= @@ -1981,8 +1983,9 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2/go.mod h1:f github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210713220151-be142a5ae1a8/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20220224212727-7a699437a831/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14 h1:vo2wkP2ceHyGyZwFFtAabpot03EeSxxwAe57pOI9E/4= github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799 h1:DOOT2B85S0tHoLGTzV+FakaSSihgRCVwZkjqKQP5L/w= +github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-ctrlnet v0.0.0-20180313164037-f564fbbdaa95/go.mod h1:SJqKCCPXRfBFCwXjfNT/skfsceF7+MBFLI2OrvuRA7g= diff --git a/itests/worker_test.go b/itests/worker_test.go index c1fba2600..0d20c2f19 100644 --- a/itests/worker_test.go +++ b/itests/worker_test.go @@ -49,17 +49,23 @@ func TestWorkerDataCid(t *testing.T) { e, err := worker.Enabled(ctx) require.NoError(t, err) require.True(t, e) - /* - pi, err := miner.ComputeDataCid(ctx, 1016, strings.NewReader(strings.Repeat("a", 1016))) - require.NoError(t, err) - require.Equal(t, abi.PaddedPieceSize(1024), pi.Size) - require.Equal(t, "baga6ea4seaqlhznlutptgfwhffupyer6txswamerq5fc2jlwf2lys2mm5jtiaeq", pi.PieceCID.String()) - */ + + pi, err := miner.ComputeDataCid(ctx, 1016, strings.NewReader(strings.Repeat("a", 1016))) + require.NoError(t, err) + require.Equal(t, abi.PaddedPieceSize(1024), pi.Size) + require.Equal(t, "baga6ea4seaqlhznlutptgfwhffupyer6txswamerq5fc2jlwf2lys2mm5jtiaeq", pi.PieceCID.String()) + bigPiece := abi.PaddedPieceSize(16 << 20).Unpadded() - pi, err := miner.ComputeDataCid(ctx, bigPiece, strings.NewReader(strings.Repeat("a", int(bigPiece)))) + pi, err = miner.ComputeDataCid(ctx, bigPiece, strings.NewReader(strings.Repeat("a", int(bigPiece)))) require.NoError(t, err) require.Equal(t, bigPiece.Padded(), pi.Size) require.Equal(t, "baga6ea4seaqmhoxl2ybw5m2wyd3pt3h4zmp7j52yumzu2rar26twns3uocq7yfa", pi.PieceCID.String()) + + nonFullPiece := abi.PaddedPieceSize(10 << 20).Unpadded() + pi, err = miner.ComputeDataCid(ctx, bigPiece, strings.NewReader(strings.Repeat("a", int(nonFullPiece)))) + require.NoError(t, err) + require.Equal(t, bigPiece.Padded(), pi.Size) + require.Equal(t, "baga6ea4seaqbxib4pdxs5cqdn3fmtj4rcxk6rx6ztiqmrx7fcpo3ymuxbp2rodi", pi.PieceCID.String()) } func TestWinningPostWorker(t *testing.T) { diff --git a/lib/httpreader/httpreader.go b/lib/httpreader/httpreader.go new file mode 100644 index 000000000..62338e76e --- /dev/null +++ b/lib/httpreader/httpreader.go @@ -0,0 +1,47 @@ +package httpreader + +import ( + "io" + "net/http" + + "golang.org/x/xerrors" +) + +// HttpReader is a reader which will read a http resource with a simple get request. +// Before first Read it will be passed over JsonRPC as a URL. +type HttpReader struct { + URL string + + reader io.ReadCloser +} + +func (h *HttpReader) Close() error { + h.URL = "" + if h.reader != nil { + return h.reader.Close() + } + return nil +} + +func (h *HttpReader) Read(p []byte) (n int, err error) { + if h.reader == nil { + res, err := http.Get(h.URL) + if err != nil { + return 0, err + } + if res.StatusCode != http.StatusOK { + return 0, xerrors.Errorf("unexpected http status %d", res.StatusCode) + } + + // mark the reader as reading + h.URL = "" + h.reader = res.Body + } + if h.reader == nil { + return 0, xerrors.Errorf("http reader closed") + } + + return h.reader.Read(p) +} + +var _ io.ReadCloser = &HttpReader{} diff --git a/lib/rpcenc/reader.go b/lib/rpcenc/reader.go index 61ae80ba9..a4f16982e 100644 --- a/lib/rpcenc/reader.go +++ b/lib/rpcenc/reader.go @@ -21,7 +21,9 @@ import ( "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-state-types/abi" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + + "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" + "github.com/filecoin-project/lotus/lib/httpreader" ) var log = logging.Logger("rpcenc") @@ -33,6 +35,7 @@ type StreamType string const ( Null StreamType = "null" PushStream StreamType = "push" + HTTP StreamType = "http" // TODO: Data transfer handoff to workers? ) @@ -101,9 +104,12 @@ func ReaderParamEncoder(addr string) jsonrpc.Option { return jsonrpc.WithParamEncoder(new(io.Reader), func(value reflect.Value) (reflect.Value, error) { r := value.Interface().(io.Reader) - if r, ok := r.(*sealing.NullReader); ok { + if r, ok := r.(*nullreader.NullReader); ok { return reflect.ValueOf(ReaderStream{Type: Null, Info: fmt.Sprint(r.N)}), nil } + if r, ok := r.(*httpreader.HttpReader); ok && r.URL != "" { + return reflect.ValueOf(ReaderStream{Type: HTTP, Info: r.URL}), nil + } reqID := uuid.New() u, err := url.Parse(addr) @@ -412,13 +418,16 @@ func ReaderParamDecoder() (http.HandlerFunc, jsonrpc.ServerOption) { return reflect.Value{}, xerrors.Errorf("unmarshaling reader id: %w", err) } - if rs.Type == Null { + switch rs.Type { + case Null: n, err := strconv.ParseInt(rs.Info, 10, 64) if err != nil { return reflect.Value{}, xerrors.Errorf("parsing null byte count: %w", err) } - return reflect.ValueOf(sealing.NewNullReader(abi.UnpaddedPieceSize(n))), nil + return reflect.ValueOf(nullreader.NewNullReader(abi.UnpaddedPieceSize(n))), nil + case HTTP: + return reflect.ValueOf(&httpreader.HttpReader{URL: rs.Info}), nil } u, err := uuid.Parse(rs.Info) diff --git a/lib/rpcenc/reader_test.go b/lib/rpcenc/reader_test.go index 302908df0..881cbeaae 100644 --- a/lib/rpcenc/reader_test.go +++ b/lib/rpcenc/reader_test.go @@ -14,7 +14,8 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-jsonrpc" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + + "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader" ) type ReaderHandler struct { @@ -57,7 +58,7 @@ func (h *ReaderHandler) ReadAll(ctx context.Context, r io.Reader) ([]byte, error } func (h *ReaderHandler) ReadNullLen(ctx context.Context, r io.Reader) (int64, error) { - return r.(*sealing.NullReader).N, nil + return r.(*nullreader.NullReader).N, nil } func (h *ReaderHandler) ReadUrl(ctx context.Context, u string) (string, error) { @@ -118,7 +119,7 @@ func TestNullReaderProxy(t *testing.T) { defer closer() - n, err := client.ReadNullLen(context.TODO(), sealing.NewNullReader(1016)) + n, err := client.ReadNullLen(context.TODO(), nullreader.NewNullReader(1016)) require.NoError(t, err) require.Equal(t, int64(1016), n) } diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index cf51fb13e..757d80932 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -897,13 +897,13 @@ This is useful for forcing all deals to be assigned as snap deals to sectors mar Name: "MinCommitBatch", Type: "int", - Comment: `maximum batched commit size - batches will be sent immediately above this size`, + Comment: `minimum batched commit size - batches above this size will eventually be sent on a timeout`, }, { Name: "MaxCommitBatch", Type: "int", - Comment: ``, + Comment: `maximum batched commit size - batches will be sent immediately above this size`, }, { Name: "CommitBatchWait", diff --git a/node/config/types.go b/node/config/types.go index b5b1fae7e..1505ecb39 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -290,8 +290,9 @@ type SealingConfig struct { // enable / disable commit aggregation (takes effect after nv13) AggregateCommits bool - // maximum batched commit size - batches will be sent immediately above this size + // minimum batched commit size - batches above this size will eventually be sent on a timeout MinCommitBatch int + // maximum batched commit size - batches will be sent immediately above this size MaxCommitBatch int // how long to wait before submitting a batch after crossing the minimum batch size CommitBatchWait Duration diff --git a/node/hello/cbor_gen.go b/node/hello/cbor_gen.go index 145a19728..68c82e2bb 100644 --- a/node/hello/cbor_gen.go +++ b/node/hello/cbor_gen.go @@ -26,61 +26,67 @@ func (t *HelloMessage) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufHelloMessage); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufHelloMessage); err != nil { return err } - scratch := make([]byte, 9) - // t.HeaviestTipSet ([]cid.Cid) (slice) if len(t.HeaviestTipSet) > cbg.MaxLength { return xerrors.Errorf("Slice value in field t.HeaviestTipSet was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.HeaviestTipSet))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.HeaviestTipSet))); err != nil { return err } for _, v := range t.HeaviestTipSet { - if err := cbg.WriteCidBuf(scratch, w, v); err != nil { + if err := cbg.WriteCid(w, v); err != nil { return xerrors.Errorf("failed writing cid field t.HeaviestTipSet: %w", err) } } // t.HeaviestTipSetHeight (abi.ChainEpoch) (int64) if t.HeaviestTipSetHeight >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.HeaviestTipSetHeight)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.HeaviestTipSetHeight)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.HeaviestTipSetHeight-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.HeaviestTipSetHeight-1)); err != nil { return err } } // t.HeaviestTipSetWeight (big.Int) (struct) - if err := t.HeaviestTipSetWeight.MarshalCBOR(w); err != nil { + if err := t.HeaviestTipSetWeight.MarshalCBOR(cw); err != nil { return err } // t.GenesisHash (cid.Cid) (struct) - if err := cbg.WriteCidBuf(scratch, w, t.GenesisHash); err != nil { + if err := cbg.WriteCid(cw, t.GenesisHash); err != nil { return xerrors.Errorf("failed to write cid field t.GenesisHash: %w", err) } return nil } -func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error { +func (t *HelloMessage) UnmarshalCBOR(r io.Reader) (err error) { *t = HelloMessage{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -91,7 +97,7 @@ func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error { // t.HeaviestTipSet ([]cid.Cid) (slice) - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -110,7 +116,7 @@ func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("reading cid field t.HeaviestTipSet failed: %w", err) } @@ -119,7 +125,7 @@ func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error { // t.HeaviestTipSetHeight (abi.ChainEpoch) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -146,7 +152,7 @@ func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error { { - if err := t.HeaviestTipSetWeight.UnmarshalCBOR(br); err != nil { + if err := t.HeaviestTipSetWeight.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.HeaviestTipSetWeight: %w", err) } @@ -155,7 +161,7 @@ func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.GenesisHash: %w", err) } @@ -173,46 +179,52 @@ func (t *LatencyMessage) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write(lengthBufLatencyMessage); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write(lengthBufLatencyMessage); err != nil { return err } - scratch := make([]byte, 9) - // t.TArrival (int64) (int64) if t.TArrival >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.TArrival)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.TArrival)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.TArrival-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.TArrival-1)); err != nil { return err } } // t.TSent (int64) (int64) if t.TSent >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.TSent)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.TSent)); err != nil { return err } } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.TSent-1)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.TSent-1)); err != nil { return err } } return nil } -func (t *LatencyMessage) UnmarshalCBOR(r io.Reader) error { +func (t *LatencyMessage) UnmarshalCBOR(r io.Reader) (err error) { *t = LatencyMessage{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajArray { return fmt.Errorf("cbor input should be of type array") } @@ -223,7 +235,7 @@ func (t *LatencyMessage) UnmarshalCBOR(r io.Reader) error { // t.TArrival (int64) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err @@ -248,7 +260,7 @@ func (t *LatencyMessage) UnmarshalCBOR(r io.Reader) error { } // t.TSent (int64) (int64) { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() var extraI int64 if err != nil { return err diff --git a/node/impl/storminer.go b/node/impl/storminer.go index f34761d89..eb8952004 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -37,8 +37,10 @@ import ( "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" + filmktsstore "github.com/filecoin-project/go-fil-markets/stores" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" + mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage" "github.com/filecoin-project/lotus/extern/sector-storage/fsutil" @@ -83,6 +85,7 @@ type StorageMinerAPI struct { SectorBlocks *sectorblocks.SectorBlocks `optional:"true"` Host host.Host `optional:"true"` DAGStore *dagstore.DAGStore `optional:"true"` + DAGStoreWrapper *mktsdagstore.Wrapper `optional:"true"` // Miner / storage Miner *storage.Miner `optional:"true"` @@ -792,6 +795,35 @@ func (sm *StorageMinerAPI) DagstoreListShards(ctx context.Context) ([]api.Dagsto return ret, nil } +func (sm *StorageMinerAPI) DagstoreRegisterShard(ctx context.Context, key string) error { + if sm.DAGStore == nil { + return fmt.Errorf("dagstore not available on this node") + } + + // First check if the shard has already been registered + k := shard.KeyFromString(key) + _, err := sm.DAGStore.GetShardInfo(k) + if err == nil { + // Shard already registered, nothing further to do + return nil + } + // If the shard is not registered we would expect ErrShardUnknown + if !errors.Is(err, dagstore.ErrShardUnknown) { + return fmt.Errorf("getting shard info from DAG store: %w", err) + } + + pieceCid, err := cid.Parse(key) + if err != nil { + return fmt.Errorf("parsing shard key as piece cid: %w", err) + } + + if err = filmktsstore.RegisterShardSync(ctx, sm.DAGStoreWrapper, pieceCid, "", true); err != nil { + return fmt.Errorf("failed to register shard: %w", err) + } + + return nil +} + func (sm *StorageMinerAPI) DagstoreInitializeShard(ctx context.Context, key string) error { if sm.DAGStore == nil { return fmt.Errorf("dagstore not available on this node") diff --git a/paychmgr/cbor_gen.go b/paychmgr/cbor_gen.go index 428c09a9e..9f12b60a2 100644 --- a/paychmgr/cbor_gen.go +++ b/paychmgr/cbor_gen.go @@ -25,25 +25,26 @@ func (t *VoucherInfo) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{163}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { return err } - scratch := make([]byte, 9) - // t.Voucher (paych.SignedVoucher) (struct) if len("Voucher") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Voucher\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Voucher"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Voucher"))); err != nil { return err } if _, err := io.WriteString(w, string("Voucher")); err != nil { return err } - if err := t.Voucher.MarshalCBOR(w); err != nil { + if err := t.Voucher.MarshalCBOR(cw); err != nil { return err } @@ -52,7 +53,7 @@ func (t *VoucherInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Proof\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Proof"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Proof"))); err != nil { return err } if _, err := io.WriteString(w, string("Proof")); err != nil { @@ -63,11 +64,11 @@ func (t *VoucherInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Byte array in field t.Proof was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Proof))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Proof))); err != nil { return err } - if _, err := w.Write(t.Proof[:]); err != nil { + if _, err := cw.Write(t.Proof[:]); err != nil { return err } @@ -76,7 +77,7 @@ func (t *VoucherInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Submitted\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Submitted"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Submitted"))); err != nil { return err } if _, err := io.WriteString(w, string("Submitted")); err != nil { @@ -89,16 +90,21 @@ func (t *VoucherInfo) MarshalCBOR(w io.Writer) error { return nil } -func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error { +func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) (err error) { *t = VoucherInfo{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -113,7 +119,7 @@ func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -127,16 +133,16 @@ func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.Voucher = new(paych.SignedVoucher) - if err := t.Voucher.UnmarshalCBOR(br); err != nil { + if err := t.Voucher.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Voucher pointer: %w", err) } } @@ -145,7 +151,7 @@ func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error { // t.Proof ([]uint8) (slice) case "Proof": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -161,13 +167,13 @@ func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error { t.Proof = make([]uint8, extra) } - if _, err := io.ReadFull(br, t.Proof[:]); err != nil { + if _, err := io.ReadFull(cr, t.Proof[:]); err != nil { return err } // t.Submitted (bool) (bool) case "Submitted": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -196,18 +202,19 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{174}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{174}); err != nil { return err } - scratch := make([]byte, 9) - // t.ChannelID (string) (string) if len("ChannelID") > cbg.MaxLength { return xerrors.Errorf("Value in field \"ChannelID\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ChannelID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ChannelID"))); err != nil { return err } if _, err := io.WriteString(w, string("ChannelID")); err != nil { @@ -218,7 +225,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.ChannelID was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.ChannelID))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.ChannelID))); err != nil { return err } if _, err := io.WriteString(w, string(t.ChannelID)); err != nil { @@ -230,14 +237,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Channel\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Channel"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Channel"))); err != nil { return err } if _, err := io.WriteString(w, string("Channel")); err != nil { return err } - if err := t.Channel.MarshalCBOR(w); err != nil { + if err := t.Channel.MarshalCBOR(cw); err != nil { return err } @@ -246,14 +253,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Control\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Control"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Control"))); err != nil { return err } if _, err := io.WriteString(w, string("Control")); err != nil { return err } - if err := t.Control.MarshalCBOR(w); err != nil { + if err := t.Control.MarshalCBOR(cw); err != nil { return err } @@ -262,14 +269,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Target\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Target"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Target"))); err != nil { return err } if _, err := io.WriteString(w, string("Target")); err != nil { return err } - if err := t.Target.MarshalCBOR(w); err != nil { + if err := t.Target.MarshalCBOR(cw); err != nil { return err } @@ -278,14 +285,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Direction\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Direction"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Direction"))); err != nil { return err } if _, err := io.WriteString(w, string("Direction")); err != nil { return err } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Direction)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Direction)); err != nil { return err } @@ -294,7 +301,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Vouchers\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Vouchers"))); err != nil { return err } if _, err := io.WriteString(w, string("Vouchers")); err != nil { @@ -305,11 +312,11 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Slice value in field t.Vouchers was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Vouchers))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Vouchers))); err != nil { return err } for _, v := range t.Vouchers { - if err := v.MarshalCBOR(w); err != nil { + if err := v.MarshalCBOR(cw); err != nil { return err } } @@ -319,14 +326,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"NextLane\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("NextLane"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("NextLane"))); err != nil { return err } if _, err := io.WriteString(w, string("NextLane")); err != nil { return err } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.NextLane)); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.NextLane)); err != nil { return err } @@ -335,14 +342,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Amount\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Amount"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Amount"))); err != nil { return err } if _, err := io.WriteString(w, string("Amount")); err != nil { return err } - if err := t.Amount.MarshalCBOR(w); err != nil { + if err := t.Amount.MarshalCBOR(cw); err != nil { return err } @@ -351,14 +358,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"AvailableAmount\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("AvailableAmount"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("AvailableAmount"))); err != nil { return err } if _, err := io.WriteString(w, string("AvailableAmount")); err != nil { return err } - if err := t.AvailableAmount.MarshalCBOR(w); err != nil { + if err := t.AvailableAmount.MarshalCBOR(cw); err != nil { return err } @@ -367,14 +374,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"PendingAvailableAmount\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PendingAvailableAmount"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PendingAvailableAmount"))); err != nil { return err } if _, err := io.WriteString(w, string("PendingAvailableAmount")); err != nil { return err } - if err := t.PendingAvailableAmount.MarshalCBOR(w); err != nil { + if err := t.PendingAvailableAmount.MarshalCBOR(cw); err != nil { return err } @@ -383,14 +390,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"PendingAmount\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PendingAmount"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PendingAmount"))); err != nil { return err } if _, err := io.WriteString(w, string("PendingAmount")); err != nil { return err } - if err := t.PendingAmount.MarshalCBOR(w); err != nil { + if err := t.PendingAmount.MarshalCBOR(cw); err != nil { return err } @@ -399,7 +406,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"CreateMsg\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("CreateMsg"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("CreateMsg"))); err != nil { return err } if _, err := io.WriteString(w, string("CreateMsg")); err != nil { @@ -407,11 +414,11 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { } if t.CreateMsg == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.CreateMsg); err != nil { + if err := cbg.WriteCid(cw, *t.CreateMsg); err != nil { return xerrors.Errorf("failed to write cid field t.CreateMsg: %w", err) } } @@ -421,7 +428,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"AddFundsMsg\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("AddFundsMsg"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("AddFundsMsg"))); err != nil { return err } if _, err := io.WriteString(w, string("AddFundsMsg")); err != nil { @@ -429,11 +436,11 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { } if t.AddFundsMsg == nil { - if _, err := w.Write(cbg.CborNull); err != nil { + if _, err := cw.Write(cbg.CborNull); err != nil { return err } } else { - if err := cbg.WriteCidBuf(scratch, w, *t.AddFundsMsg); err != nil { + if err := cbg.WriteCid(cw, *t.AddFundsMsg); err != nil { return xerrors.Errorf("failed to write cid field t.AddFundsMsg: %w", err) } } @@ -443,7 +450,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Settling\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Settling"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Settling"))); err != nil { return err } if _, err := io.WriteString(w, string("Settling")); err != nil { @@ -456,16 +463,21 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { return nil } -func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { +func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) (err error) { *t = ChannelInfo{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -480,7 +492,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -493,7 +505,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { case "ChannelID": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -505,16 +517,16 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } t.Channel = new(address.Address) - if err := t.Channel.UnmarshalCBOR(br); err != nil { + if err := t.Channel.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Channel pointer: %w", err) } } @@ -525,7 +537,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { { - if err := t.Control.UnmarshalCBOR(br); err != nil { + if err := t.Control.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Control: %w", err) } @@ -535,7 +547,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { { - if err := t.Target.UnmarshalCBOR(br); err != nil { + if err := t.Target.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Target: %w", err) } @@ -545,7 +557,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -558,7 +570,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { // t.Vouchers ([]*paychmgr.VoucherInfo) (slice) case "Vouchers": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -578,7 +590,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { for i := 0; i < int(extra); i++ { var v VoucherInfo - if err := v.UnmarshalCBOR(br); err != nil { + if err := v.UnmarshalCBOR(cr); err != nil { return err } @@ -590,7 +602,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { { - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -605,7 +617,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { { - if err := t.Amount.UnmarshalCBOR(br); err != nil { + if err := t.Amount.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Amount: %w", err) } @@ -615,7 +627,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { { - if err := t.AvailableAmount.UnmarshalCBOR(br); err != nil { + if err := t.AvailableAmount.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.AvailableAmount: %w", err) } @@ -625,7 +637,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { { - if err := t.PendingAvailableAmount.UnmarshalCBOR(br); err != nil { + if err := t.PendingAvailableAmount.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.PendingAvailableAmount: %w", err) } @@ -635,7 +647,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { { - if err := t.PendingAmount.UnmarshalCBOR(br); err != nil { + if err := t.PendingAmount.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.PendingAmount: %w", err) } @@ -645,16 +657,16 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.CreateMsg: %w", err) } @@ -668,16 +680,16 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { { - b, err := br.ReadByte() + b, err := cr.ReadByte() if err != nil { return err } if b != cbg.CborNull[0] { - if err := br.UnreadByte(); err != nil { + if err := cr.UnreadByte(); err != nil { return err } - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.AddFundsMsg: %w", err) } @@ -689,7 +701,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { // t.Settling (bool) (bool) case "Settling": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -718,18 +730,19 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{164}); err != nil { + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { return err } - scratch := make([]byte, 9) - // t.ChannelID (string) (string) if len("ChannelID") > cbg.MaxLength { return xerrors.Errorf("Value in field \"ChannelID\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ChannelID"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ChannelID"))); err != nil { return err } if _, err := io.WriteString(w, string("ChannelID")); err != nil { @@ -740,7 +753,7 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.ChannelID was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.ChannelID))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.ChannelID))); err != nil { return err } if _, err := io.WriteString(w, string(t.ChannelID)); err != nil { @@ -752,14 +765,14 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"MsgCid\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("MsgCid"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("MsgCid"))); err != nil { return err } if _, err := io.WriteString(w, string("MsgCid")); err != nil { return err } - if err := cbg.WriteCidBuf(scratch, w, t.MsgCid); err != nil { + if err := cbg.WriteCid(cw, t.MsgCid); err != nil { return xerrors.Errorf("failed to write cid field t.MsgCid: %w", err) } @@ -768,7 +781,7 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Received\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Received"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Received"))); err != nil { return err } if _, err := io.WriteString(w, string("Received")); err != nil { @@ -784,7 +797,7 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field \"Err\" was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Err"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Err"))); err != nil { return err } if _, err := io.WriteString(w, string("Err")); err != nil { @@ -795,7 +808,7 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("Value in field t.Err was too long") } - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Err))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Err))); err != nil { return err } if _, err := io.WriteString(w, string(t.Err)); err != nil { @@ -804,16 +817,21 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error { return nil } -func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error { +func (t *MsgInfo) UnmarshalCBOR(r io.Reader) (err error) { *t = MsgInfo{} - br := cbg.GetPeeker(r) - scratch := make([]byte, 8) + cr := cbg.NewCborReader(r) - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err := cr.ReadHeader() if err != nil { return err } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + if maj != cbg.MajMap { return fmt.Errorf("cbor input should be of type map") } @@ -828,7 +846,7 @@ func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error { for i := uint64(0); i < n; i++ { { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -841,7 +859,7 @@ func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error { case "ChannelID": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err } @@ -853,7 +871,7 @@ func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) + c, err := cbg.ReadCid(cr) if err != nil { return xerrors.Errorf("failed to read cid field t.MsgCid: %w", err) } @@ -864,7 +882,7 @@ func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error { // t.Received (bool) (bool) case "Received": - maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + maj, extra, err = cr.ReadHeader() if err != nil { return err } @@ -883,7 +901,7 @@ func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error { case "Err": { - sval, err := cbg.ReadStringBuf(br, scratch) + sval, err := cbg.ReadString(cr) if err != nil { return err }