Merge remote-tracking branch 'origin/master' into feat/worker-task-count-limits

.github/workflows/testground-on-push.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: testground run
-        uses: coryschwartz/testground-github-action@v1.1
+        uses: testground/testground-github-action@v1
         with:
           backend_addr: ${{ matrix.backend_addr }}
           backend_proto: ${{ matrix.backend_proto }}

api/api_storage.go

@@ -227,6 +227,9 @@ type StorageMiner interface {
 	// DagstoreGC runs garbage collection on the DAG store.
 	DagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin
 
+	// DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID
+	DagstoreRegisterShard(ctx context.Context, key string) error //perm:admin
+
 	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
 	// so they can download its index
 	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin

api/cbor_gen.go

@@ -26,25 +26,26 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{163}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{163}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Channel (address.Address) (struct)
 	if len("Channel") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Channel\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Channel"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Channel"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Channel")); err != nil {
 		return err
 	}
 
-	if err := t.Channel.MarshalCBOR(w); err != nil {
+	if err := t.Channel.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -53,14 +54,14 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"WaitSentinel\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("WaitSentinel"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("WaitSentinel")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteCidBuf(scratch, w, t.WaitSentinel); err != nil {
+	if err := cbg.WriteCid(cw, t.WaitSentinel); err != nil {
 		return xerrors.Errorf("failed to write cid field t.WaitSentinel: %w", err)
 	}
 
@@ -69,7 +70,7 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Vouchers\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Vouchers")); err != nil {
@@ -80,27 +81,32 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Vouchers was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Vouchers))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Vouchers))); err != nil {
 		return err
 	}
 	for _, v := range t.Vouchers {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
+func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = PaymentInfo{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -115,7 +121,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -129,7 +135,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.Channel.UnmarshalCBOR(br); err != nil {
+				if err := t.Channel.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.Channel: %w", err)
 				}
 
@@ -139,7 +145,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				c, err := cbg.ReadCid(br)
+				c, err := cbg.ReadCid(cr)
 				if err != nil {
 					return xerrors.Errorf("failed to read cid field t.WaitSentinel: %w", err)
 				}
@@ -150,7 +156,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 			// t.Vouchers ([]*paych.SignedVoucher) (slice)
 		case "Vouchers":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -170,7 +176,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 			for i := 0; i < int(extra); i++ {
 
 				var v paych.SignedVoucher
-				if err := v.UnmarshalCBOR(br); err != nil {
+				if err := v.UnmarshalCBOR(cr); err != nil {
 					return err
 				}
 
@@ -190,25 +196,26 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{163}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{163}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.SectorID (abi.SectorNumber) (uint64)
 	if len("SectorID") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"SectorID\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("SectorID"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("SectorID"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("SectorID")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.SectorID)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.SectorID)); err != nil {
 		return err
 	}
 
@@ -217,14 +224,14 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Offset\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Offset"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Offset"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Offset")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Offset)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Offset)); err != nil {
 		return err
 	}
 
@@ -233,30 +240,35 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Size\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Size"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Size"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Size")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Size)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Size)); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
+func (t *SealedRef) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = SealedRef{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -271,7 +283,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -285,7 +297,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err = cr.ReadHeader()
 				if err != nil {
 					return err
 				}
@@ -300,7 +312,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err = cr.ReadHeader()
 				if err != nil {
 					return err
 				}
@@ -315,7 +327,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err = cr.ReadHeader()
 				if err != nil {
 					return err
 				}
@@ -339,18 +351,19 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{161}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{161}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Refs ([]api.SealedRef) (slice)
 	if len("Refs") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Refs\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Refs"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Refs"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Refs")); err != nil {
@@ -361,27 +374,32 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Refs was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Refs))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Refs))); err != nil {
 		return err
 	}
 	for _, v := range t.Refs {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
+func (t *SealedRefs) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = SealedRefs{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -396,7 +414,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -408,7 +426,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
 		// t.Refs ([]api.SealedRef) (slice)
 		case "Refs":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -428,7 +446,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
 			for i := 0; i < int(extra); i++ {
 
 				var v SealedRef
-				if err := v.UnmarshalCBOR(br); err != nil {
+				if err := v.UnmarshalCBOR(cr); err != nil {
 					return err
 				}
 
@@ -448,18 +466,19 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{162}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Value (abi.SealRandomness) (slice)
 	if len("Value") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Value\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Value")); err != nil {
@@ -470,11 +489,11 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Byte array in field t.Value was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Value))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Value))); err != nil {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := cw.Write(t.Value[:]); err != nil {
 		return err
 	}
 
@@ -483,7 +502,7 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Epoch\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Epoch")); err != nil {
@@ -491,27 +510,32 @@ func (t *SealTicket) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.Epoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
+func (t *SealTicket) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = SealTicket{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -526,7 +550,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -538,7 +562,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 		// t.Value (abi.SealRandomness) (slice)
 		case "Value":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -554,13 +578,13 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error {
 				t.Value = make([]uint8, extra)
 			}
 
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
+			if _, err := io.ReadFull(cr, t.Value[:]); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)
 		case "Epoch":
 			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err := cr.ReadHeader()
 				var extraI int64
 				if err != nil {
 					return err
@@ -597,18 +621,19 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{162}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Value (abi.InteractiveSealRandomness) (slice)
 	if len("Value") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Value\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Value"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Value"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Value")); err != nil {
@@ -619,11 +644,11 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Byte array in field t.Value was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Value))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Value))); err != nil {
 		return err
 	}
 
-	if _, err := w.Write(t.Value[:]); err != nil {
+	if _, err := cw.Write(t.Value[:]); err != nil {
 		return err
 	}
 
@@ -632,7 +657,7 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Epoch\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Epoch"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Epoch"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Epoch")); err != nil {
@@ -640,27 +665,32 @@ func (t *SealSeed) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.Epoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Epoch)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Epoch-1)); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
+func (t *SealSeed) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = SealSeed{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -675,7 +705,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -687,7 +717,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 		// t.Value (abi.InteractiveSealRandomness) (slice)
 		case "Value":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -703,13 +733,13 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error {
 				t.Value = make([]uint8, extra)
 			}
 
-			if _, err := io.ReadFull(br, t.Value[:]); err != nil {
+			if _, err := io.ReadFull(cr, t.Value[:]); err != nil {
 				return err
 			}
 			// t.Epoch (abi.ChainEpoch) (int64)
 		case "Epoch":
 			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err := cr.ReadHeader()
 				var extraI int64
 				if err != nil {
 					return err
@@ -746,18 +776,19 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{165}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{165}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.PublishCid (cid.Cid) (struct)
 	if len("PublishCid") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"PublishCid\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PublishCid"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PublishCid"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("PublishCid")); err != nil {
@@ -765,11 +796,11 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.PublishCid == nil {
-		if _, err := w.Write(cbg.CborNull); err != nil {
+		if _, err := cw.Write(cbg.CborNull); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteCidBuf(scratch, w, *t.PublishCid); err != nil {
+		if err := cbg.WriteCid(cw, *t.PublishCid); err != nil {
 			return xerrors.Errorf("failed to write cid field t.PublishCid: %w", err)
 		}
 	}
@@ -779,14 +810,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"DealID\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealID"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealID"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("DealID")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.DealID)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.DealID)); err != nil {
 		return err
 	}
 
@@ -795,14 +826,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"DealProposal\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealProposal"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealProposal"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("DealProposal")); err != nil {
 		return err
 	}
 
-	if err := t.DealProposal.MarshalCBOR(w); err != nil {
+	if err := t.DealProposal.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -811,14 +842,14 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"DealSchedule\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("DealSchedule"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DealSchedule"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("DealSchedule")); err != nil {
 		return err
 	}
 
-	if err := t.DealSchedule.MarshalCBOR(w); err != nil {
+	if err := t.DealSchedule.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -827,7 +858,7 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"KeepUnsealed\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("KeepUnsealed"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("KeepUnsealed")); err != nil {
@@ -840,16 +871,21 @@ func (t *PieceDealInfo) MarshalCBOR(w io.Writer) error {
 	return nil
 }
 
-func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
+func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = PieceDealInfo{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -864,7 +900,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -878,16 +914,16 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				b, err := br.ReadByte()
+				b, err := cr.ReadByte()
 				if err != nil {
 					return err
 				}
 				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
+					if err := cr.UnreadByte(); err != nil {
 						return err
 					}
 
-					c, err := cbg.ReadCid(br)
+					c, err := cbg.ReadCid(cr)
 					if err != nil {
 						return xerrors.Errorf("failed to read cid field t.PublishCid: %w", err)
 					}
@@ -901,7 +937,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err = cr.ReadHeader()
 				if err != nil {
 					return err
 				}
@@ -916,16 +952,16 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				b, err := br.ReadByte()
+				b, err := cr.ReadByte()
 				if err != nil {
 					return err
 				}
 				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
+					if err := cr.UnreadByte(); err != nil {
 						return err
 					}
 					t.DealProposal = new(market.DealProposal)
-					if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
+					if err := t.DealProposal.UnmarshalCBOR(cr); err != nil {
 						return xerrors.Errorf("unmarshaling t.DealProposal pointer: %w", err)
 					}
 				}
@@ -936,7 +972,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.DealSchedule.UnmarshalCBOR(br); err != nil {
+				if err := t.DealSchedule.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.DealSchedule: %w", err)
 				}
 
@@ -944,7 +980,7 @@ func (t *PieceDealInfo) UnmarshalCBOR(r io.Reader) error {
 			// t.KeepUnsealed (bool) (bool)
 		case "KeepUnsealed":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -973,18 +1009,19 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{162}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.StartEpoch (abi.ChainEpoch) (int64)
 	if len("StartEpoch") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"StartEpoch\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartEpoch"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("StartEpoch"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("StartEpoch")); err != nil {
@@ -992,11 +1029,11 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.StartEpoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.StartEpoch)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.StartEpoch-1)); err != nil {
 			return err
 		}
 	}
@@ -1006,7 +1043,7 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"EndEpoch\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("EndEpoch"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("EndEpoch"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("EndEpoch")); err != nil {
@@ -1014,27 +1051,32 @@ func (t *DealSchedule) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.EndEpoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.EndEpoch)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.EndEpoch-1)); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
+func (t *DealSchedule) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = DealSchedule{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -1049,7 +1091,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -1061,7 +1103,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
 		// t.StartEpoch (abi.ChainEpoch) (int64)
 		case "StartEpoch":
 			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err := cr.ReadHeader()
 				var extraI int64
 				if err != nil {
 					return err
@@ -1087,7 +1129,7 @@ func (t *DealSchedule) UnmarshalCBOR(r io.Reader) error {
 			// t.EndEpoch (abi.ChainEpoch) (int64)
 		case "EndEpoch":
 			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err := cr.ReadHeader()
 				var extraI int64
 				if err != nil {
 					return err

api/proxy_gen.go

@@ -663,6 +663,8 @@ type StorageMinerStruct struct {
 
 		DagstoreRecoverShard func(p0 context.Context, p1 string) error `perm:"write"`
 
+		DagstoreRegisterShard func(p0 context.Context, p1 string) error `perm:"admin"`
+
 		DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"`
 
 		DealsConsiderOfflineStorageDeals func(p0 context.Context) (bool, error) `perm:"admin"`
@@ -3990,6 +3992,17 @@ func (s *StorageMinerStub) DagstoreRecoverShard(p0 context.Context, p1 string) e
 	return ErrNotSupported
 }
 
+func (s *StorageMinerStruct) DagstoreRegisterShard(p0 context.Context, p1 string) error {
+	if s.Internal.DagstoreRegisterShard == nil {
+		return ErrNotSupported
+	}
+	return s.Internal.DagstoreRegisterShard(p0, p1)
+}
+
+func (s *StorageMinerStub) DagstoreRegisterShard(p0 context.Context, p1 string) error {
+	return ErrNotSupported
+}
+
 func (s *StorageMinerStruct) DealsConsiderOfflineRetrievalDeals(p0 context.Context) (bool, error) {
 	if s.Internal.DealsConsiderOfflineRetrievalDeals == nil {
 		return false, ErrNotSupported

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.15.3-dev"
+const BuildVersion = "1.15.4-dev"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

chain/exchange/cbor_gen.go

@@ -26,51 +26,57 @@ func (t *Request) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufRequest); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufRequest); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Head ([]cid.Cid) (slice)
 	if len(t.Head) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.Head was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Head))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Head))); err != nil {
 		return err
 	}
 	for _, v := range t.Head {
-		if err := cbg.WriteCidBuf(scratch, w, v); err != nil {
+		if err := cbg.WriteCid(w, v); err != nil {
 			return xerrors.Errorf("failed writing cid field t.Head: %w", err)
 		}
 	}
 
 	// t.Length (uint64) (uint64)
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Length)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Length)); err != nil {
 		return err
 	}
 
 	// t.Options (uint64) (uint64)
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Options)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Options)); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (t *Request) UnmarshalCBOR(r io.Reader) error {
+func (t *Request) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = Request{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -81,7 +87,7 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error {
 
 	// t.Head ([]cid.Cid) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -100,7 +106,7 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error {
 
 	for i := 0; i < int(extra); i++ {
 
-		c, err := cbg.ReadCid(br)
+		c, err := cbg.ReadCid(cr)
 		if err != nil {
 			return xerrors.Errorf("reading cid field t.Head failed: %w", err)
 		}
@@ -111,7 +117,7 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err = cr.ReadHeader()
 		if err != nil {
 			return err
 		}
@@ -125,7 +131,7 @@ func (t *Request) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err = cr.ReadHeader()
 		if err != nil {
 			return err
 		}
@@ -145,15 +151,16 @@ func (t *Response) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufResponse); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufResponse); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Status (exchange.status) (uint64)
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Status)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Status)); err != nil {
 		return err
 	}
 
@@ -162,7 +169,7 @@ func (t *Response) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.ErrorMessage was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.ErrorMessage))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.ErrorMessage))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.ErrorMessage)); err != nil {
@@ -174,27 +181,32 @@ func (t *Response) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Chain was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Chain))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Chain))); err != nil {
 		return err
 	}
 	for _, v := range t.Chain {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *Response) UnmarshalCBOR(r io.Reader) error {
+func (t *Response) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = Response{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -207,7 +219,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err = cr.ReadHeader()
 		if err != nil {
 			return err
 		}
@@ -220,7 +232,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error {
 	// t.ErrorMessage (string) (string)
 
 	{
-		sval, err := cbg.ReadStringBuf(br, scratch)
+		sval, err := cbg.ReadString(cr)
 		if err != nil {
 			return err
 		}
@@ -229,7 +241,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.Chain ([]*exchange.BSTipSet) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -249,7 +261,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error {
 	for i := 0; i < int(extra); i++ {
 
 		var v BSTipSet
-		if err := v.UnmarshalCBOR(br); err != nil {
+		if err := v.UnmarshalCBOR(cr); err != nil {
 			return err
 		}
 
@@ -266,22 +278,23 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufCompactedMessages); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufCompactedMessages); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Bls ([]*types.Message) (slice)
 	if len(t.Bls) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.Bls was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Bls))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Bls))); err != nil {
 		return err
 	}
 	for _, v := range t.Bls {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
@@ -291,7 +304,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.BlsIncludes was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.BlsIncludes))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.BlsIncludes))); err != nil {
 		return err
 	}
 	for _, v := range t.BlsIncludes {
@@ -299,11 +312,11 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 			return xerrors.Errorf("Slice value in field v was too long")
 		}
 
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil {
 			return err
 		}
 		for _, v := range v {
-			if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil {
+			if err := cw.CborWriteHeader(cbg.MajUnsignedInt, uint64(v)); err != nil {
 				return err
 			}
 		}
@@ -314,11 +327,11 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Secpk was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Secpk))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Secpk))); err != nil {
 		return err
 	}
 	for _, v := range t.Secpk {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
@@ -328,7 +341,7 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.SecpkIncludes was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.SecpkIncludes))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.SecpkIncludes))); err != nil {
 		return err
 	}
 	for _, v := range t.SecpkIncludes {
@@ -336,11 +349,11 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 			return xerrors.Errorf("Slice value in field v was too long")
 		}
 
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(v))); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(v))); err != nil {
 			return err
 		}
 		for _, v := range v {
-			if err := cbg.CborWriteHeader(w, cbg.MajUnsignedInt, uint64(v)); err != nil {
+			if err := cw.CborWriteHeader(cbg.MajUnsignedInt, uint64(v)); err != nil {
 				return err
 			}
 		}
@@ -348,16 +361,21 @@ func (t *CompactedMessages) MarshalCBOR(w io.Writer) error {
 	return nil
 }
 
-func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
+func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = CompactedMessages{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -368,7 +386,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
 
 	// t.Bls ([]*types.Message) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -388,7 +406,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
 	for i := 0; i < int(extra); i++ {
 
 		var v types.Message
-		if err := v.UnmarshalCBOR(br); err != nil {
+		if err := v.UnmarshalCBOR(cr); err != nil {
 			return err
 		}
 
@@ -397,7 +415,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
 
 	// t.BlsIncludes ([][]uint64) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -420,7 +438,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
 			var extra uint64
 			var err error
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -439,7 +457,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
 
 			for j := 0; j < int(extra); j++ {
 
-				maj, val, err := cbg.CborReadHeaderBuf(br, scratch)
+				maj, val, err := cr.ReadHeader()
 				if err != nil {
 					return xerrors.Errorf("failed to read uint64 for t.BlsIncludes[i] slice: %w", err)
 				}
@@ -456,7 +474,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
 
 	// t.Secpk ([]*types.SignedMessage) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -476,7 +494,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
 	for i := 0; i < int(extra); i++ {
 
 		var v types.SignedMessage
-		if err := v.UnmarshalCBOR(br); err != nil {
+		if err := v.UnmarshalCBOR(cr); err != nil {
 			return err
 		}
 
@@ -485,7 +503,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
 
 	// t.SecpkIncludes ([][]uint64) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -508,7 +526,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
 			var extra uint64
 			var err error
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -527,7 +545,7 @@ func (t *CompactedMessages) UnmarshalCBOR(r io.Reader) error {
 
 			for j := 0; j < int(extra); j++ {
 
-				maj, val, err := cbg.CborReadHeaderBuf(br, scratch)
+				maj, val, err := cr.ReadHeader()
 				if err != nil {
 					return xerrors.Errorf("failed to read uint64 for t.SecpkIncludes[i] slice: %w", err)
 				}
@@ -552,43 +570,49 @@ func (t *BSTipSet) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufBSTipSet); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufBSTipSet); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Blocks ([]*types.BlockHeader) (slice)
 	if len(t.Blocks) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.Blocks was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Blocks))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Blocks))); err != nil {
 		return err
 	}
 	for _, v := range t.Blocks {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
 
 	// t.Messages (exchange.CompactedMessages) (struct)
-	if err := t.Messages.MarshalCBOR(w); err != nil {
+	if err := t.Messages.MarshalCBOR(cw); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error {
+func (t *BSTipSet) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = BSTipSet{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -599,7 +623,7 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error {
 
 	// t.Blocks ([]*types.BlockHeader) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -619,7 +643,7 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error {
 	for i := 0; i < int(extra); i++ {
 
 		var v types.BlockHeader
-		if err := v.UnmarshalCBOR(br); err != nil {
+		if err := v.UnmarshalCBOR(cr); err != nil {
 			return err
 		}
 
@@ -630,16 +654,16 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		b, err := br.ReadByte()
+		b, err := cr.ReadByte()
 		if err != nil {
 			return err
 		}
 		if b != cbg.CborNull[0] {
-			if err := br.UnreadByte(); err != nil {
+			if err := cr.UnreadByte(); err != nil {
 				return err
 			}
 			t.Messages = new(CompactedMessages)
-			if err := t.Messages.UnmarshalCBOR(br); err != nil {
+			if err := t.Messages.UnmarshalCBOR(cr); err != nil {
 				return xerrors.Errorf("unmarshaling t.Messages pointer: %w", err)
 			}
 		}

chain/market/cbor_gen.go

@@ -25,30 +25,31 @@ func (t *FundedAddressState) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufFundedAddressState); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufFundedAddressState); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Addr (address.Address) (struct)
-	if err := t.Addr.MarshalCBOR(w); err != nil {
+	if err := t.Addr.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
 	// t.AmtReserved (big.Int) (struct)
-	if err := t.AmtReserved.MarshalCBOR(w); err != nil {
+	if err := t.AmtReserved.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
 	// t.MsgCid (cid.Cid) (struct)
 
 	if t.MsgCid == nil {
-		if _, err := w.Write(cbg.CborNull); err != nil {
+		if _, err := cw.Write(cbg.CborNull); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteCidBuf(scratch, w, *t.MsgCid); err != nil {
+		if err := cbg.WriteCid(cw, *t.MsgCid); err != nil {
 			return xerrors.Errorf("failed to write cid field t.MsgCid: %w", err)
 		}
 	}
@@ -56,16 +57,21 @@ func (t *FundedAddressState) MarshalCBOR(w io.Writer) error {
 	return nil
 }
 
-func (t *FundedAddressState) UnmarshalCBOR(r io.Reader) error {
+func (t *FundedAddressState) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = FundedAddressState{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -78,7 +84,7 @@ func (t *FundedAddressState) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.Addr.UnmarshalCBOR(br); err != nil {
+		if err := t.Addr.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.Addr: %w", err)
 		}
 
@@ -87,7 +93,7 @@ func (t *FundedAddressState) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.AmtReserved.UnmarshalCBOR(br); err != nil {
+		if err := t.AmtReserved.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.AmtReserved: %w", err)
 		}
 
@@ -96,16 +102,16 @@ func (t *FundedAddressState) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		b, err := br.ReadByte()
+		b, err := cr.ReadByte()
 		if err != nil {
 			return err
 		}
 		if b != cbg.CborNull[0] {
-			if err := br.UnreadByte(); err != nil {
+			if err := cr.UnreadByte(); err != nil {
 				return err
 			}
 
-			c, err := cbg.ReadCid(br)
+			c, err := cbg.ReadCid(cr)
 			if err != nil {
 				return xerrors.Errorf("failed to read cid field t.MsgCid: %w", err)
 			}

chain/store/snapshot.go

@@ -8,6 +8,7 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/ipld/go-car"
 	carutil "github.com/ipld/go-car/util"
+	mh "github.com/multiformats/go-multihash"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 
@@ -142,7 +143,18 @@ func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRe
 
 		for _, c := range out {
 			if seen.Visit(c) {
-				if c.Prefix().Codec != cid.DagCBOR {
+				prefix := c.Prefix()
+
+				// Don't include identity CIDs.
+				if prefix.MhType == mh.IDENTITY {
+					continue
+				}
+
+				// We only include raw and dagcbor, for now.
+				// Raw for "code" CIDs.
+				switch prefix.Codec {
+				case cid.Raw, cid.DagCBOR:
+				default:
 					continue
 				}
 

chain/vm/cbor_gen.go

@@ -26,19 +26,20 @@ func (t *FvmExecutionTrace) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufFvmExecutionTrace); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufFvmExecutionTrace); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Msg (types.Message) (struct)
-	if err := t.Msg.MarshalCBOR(w); err != nil {
+	if err := t.Msg.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
 	// t.MsgRct (types.MessageReceipt) (struct)
-	if err := t.MsgRct.MarshalCBOR(w); err != nil {
+	if err := t.MsgRct.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -47,7 +48,7 @@ func (t *FvmExecutionTrace) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.Error was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Error))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Error))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.Error)); err != nil {
@@ -59,27 +60,32 @@ func (t *FvmExecutionTrace) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Subcalls was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Subcalls))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Subcalls))); err != nil {
 		return err
 	}
 	for _, v := range t.Subcalls {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error {
+func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = FvmExecutionTrace{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -92,16 +98,16 @@ func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		b, err := br.ReadByte()
+		b, err := cr.ReadByte()
 		if err != nil {
 			return err
 		}
 		if b != cbg.CborNull[0] {
-			if err := br.UnreadByte(); err != nil {
+			if err := cr.UnreadByte(); err != nil {
 				return err
 			}
 			t.Msg = new(types.Message)
-			if err := t.Msg.UnmarshalCBOR(br); err != nil {
+			if err := t.Msg.UnmarshalCBOR(cr); err != nil {
 				return xerrors.Errorf("unmarshaling t.Msg pointer: %w", err)
 			}
 		}
@@ -111,16 +117,16 @@ func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		b, err := br.ReadByte()
+		b, err := cr.ReadByte()
 		if err != nil {
 			return err
 		}
 		if b != cbg.CborNull[0] {
-			if err := br.UnreadByte(); err != nil {
+			if err := cr.UnreadByte(); err != nil {
 				return err
 			}
 			t.MsgRct = new(types.MessageReceipt)
-			if err := t.MsgRct.UnmarshalCBOR(br); err != nil {
+			if err := t.MsgRct.UnmarshalCBOR(cr); err != nil {
 				return xerrors.Errorf("unmarshaling t.MsgRct pointer: %w", err)
 			}
 		}
@@ -129,7 +135,7 @@ func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error {
 	// t.Error (string) (string)
 
 	{
-		sval, err := cbg.ReadStringBuf(br, scratch)
+		sval, err := cbg.ReadString(cr)
 		if err != nil {
 			return err
 		}
@@ -138,7 +144,7 @@ func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.Subcalls ([]vm.FvmExecutionTrace) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -158,7 +164,7 @@ func (t *FvmExecutionTrace) UnmarshalCBOR(r io.Reader) error {
 	for i := 0; i < int(extra); i++ {
 
 		var v FvmExecutionTrace
-		if err := v.UnmarshalCBOR(br); err != nil {
+		if err := v.UnmarshalCBOR(cr); err != nil {
 			return err
 		}
 

cmd/lotus-miner/dagstore.go

@@ -19,6 +19,7 @@ var dagstoreCmd = &cli.Command{
 	Usage: "Manage the dagstore on the markets subsystem",
 	Subcommands: []*cli.Command{
 		dagstoreListShardsCmd,
+		dagstoreRegisterShardCmd,
 		dagstoreInitializeShardCmd,
 		dagstoreRecoverShardCmd,
 		dagstoreInitializeAllCmd,
@@ -59,6 +60,45 @@ var dagstoreListShardsCmd = &cli.Command{
 	},
 }
 
+var dagstoreRegisterShardCmd = &cli.Command{
+	Name:      "register-shard",
+	ArgsUsage: "[key]",
+	Usage:     "Register a shard",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		if cctx.NArg() != 1 {
+			return fmt.Errorf("must provide a single shard key")
+		}
+
+		marketsAPI, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		shardKey := cctx.Args().First()
+		err = marketsAPI.DagstoreRegisterShard(ctx, shardKey)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("Registered shard " + shardKey)
+		return nil
+	},
+}
+
 var dagstoreInitializeShardCmd = &cli.Command{
 	Name:      "initialize-shard",
 	ArgsUsage: "[key]",

cmd/lotus-miner/precommits-info.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/specs-actors/v7/actors/util/adt"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	"github.com/urfave/cli/v2"
+)
+
+var sectorPreCommitsCmd = &cli.Command{
+	Name:  "precommits",
+	Usage: "Print on-chain precommit info",
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+		mapi, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		maddr, err := getActorAddress(ctx, cctx)
+		if err != nil {
+			return err
+		}
+		mact, err := mapi.StateGetActor(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+		store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(mapi)))
+		mst, err := miner.Load(store, mact)
+		if err != nil {
+			return err
+		}
+		preCommitSector := make([]miner.SectorPreCommitOnChainInfo, 0)
+		err = mst.ForEachPrecommittedSector(func(info miner.SectorPreCommitOnChainInfo) error {
+			preCommitSector = append(preCommitSector, info)
+			return err
+		})
+		less := func(i, j int) bool {
+			return preCommitSector[i].Info.SectorNumber <= preCommitSector[j].Info.SectorNumber
+		}
+		sort.Slice(preCommitSector, less)
+		for _, info := range preCommitSector {
+			fmt.Printf("%s: %s\n", info.Info.SectorNumber, info.PreCommitEpoch)
+		}
+
+		return nil
+	},
+}

cmd/lotus-miner/sealing.go

@@ -4,20 +4,27 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"math"
+	"net/http"
 	"os"
 	"sort"
 	"strings"
 	"text/tabwriter"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/fatih/color"
 	"github.com/google/uuid"
+	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-padreader"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	"github.com/filecoin-project/lotus/lib/httpreader"
 
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
@@ -31,6 +38,7 @@ var sealingCmd = &cli.Command{
 		workersCmd(true),
 		sealingSchedDiagCmd,
 		sealingAbortCmd,
+		sealingDataCidCmd,
 	},
 }
 
@@ -411,3 +419,94 @@ var sealingAbortCmd = &cli.Command{
 		return nodeApi.SealingAbort(ctx, job.ID)
 	},
 }
+
+var sealingDataCidCmd = &cli.Command{
+	Name:      "data-cid",
+	Usage:     "Compute data CID using workers",
+	ArgsUsage: "[file/url] <padded piece size>",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:  "file-size",
+			Usage: "real file size",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() < 1 || cctx.Args().Len() > 2 {
+			return xerrors.Errorf("expected 1 or 2 arguments")
+		}
+
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		var r io.Reader
+		sz := cctx.Uint64("file-size")
+
+		if strings.HasPrefix(cctx.Args().First(), "http://") || strings.HasPrefix(cctx.Args().First(), "https://") {
+			r = &httpreader.HttpReader{
+				URL: cctx.Args().First(),
+			}
+
+			if !cctx.IsSet("file-size") {
+				resp, err := http.Head(cctx.Args().First())
+				if err != nil {
+					return xerrors.Errorf("http head: %w", err)
+				}
+
+				if resp.ContentLength < 0 {
+					return xerrors.Errorf("head response didn't contain content length; specify --file-size")
+				}
+				sz = uint64(resp.ContentLength)
+			}
+		} else {
+			p, err := homedir.Expand(cctx.Args().First())
+			if err != nil {
+				return xerrors.Errorf("expanding path: %w", err)
+			}
+
+			f, err := os.OpenFile(p, os.O_RDONLY, 0)
+			if err != nil {
+				return xerrors.Errorf("opening source file: %w", err)
+			}
+
+			if !cctx.IsSet("file-size") {
+				st, err := f.Stat()
+				if err != nil {
+					return xerrors.Errorf("stat: %w", err)
+				}
+				sz = uint64(st.Size())
+			}
+
+			r = f
+		}
+
+		var psize abi.PaddedPieceSize
+		if cctx.Args().Len() == 2 {
+			rps, err := humanize.ParseBytes(cctx.Args().Get(1))
+			if err != nil {
+				return xerrors.Errorf("parsing piece size: %w", err)
+			}
+			psize = abi.PaddedPieceSize(rps)
+			if err := psize.Validate(); err != nil {
+				return xerrors.Errorf("checking piece size: %w", err)
+			}
+			if sz > uint64(psize.Unpadded()) {
+				return xerrors.Errorf("file larger than the piece")
+			}
+		} else {
+			psize = padreader.PaddedSize(sz).Padded()
+		}
+
+		pc, err := nodeApi.ComputeDataCid(ctx, psize.Unpadded(), r)
+		if err != nil {
+			return xerrors.Errorf("computing data CID: %w", err)
+		}
+
+		fmt.Println(pc.PieceCID, " ", pc.Size)
+		return nil
+	},
+}

cmd/lotus-miner/sectors.go

@@ -45,6 +45,7 @@ var sectorsCmd = &cli.Command{
 		sectorsRefsCmd,
 		sectorsUpdateCmd,
 		sectorsPledgeCmd,
+		sectorPreCommitsCmd,
 		sectorsCheckExpireCmd,
 		sectorsExpiredCmd,
 		sectorsRenewCmd,
@@ -58,6 +59,7 @@ var sectorsCmd = &cli.Command{
 		sectorsCapacityCollateralCmd,
 		sectorsBatching,
 		sectorsRefreshPieceMatchingCmd,
+		sectorsCompactPartitionsCmd,
 	},
 }
 
@@ -2088,3 +2090,89 @@ func yesno(b bool) string {
 	}
 	return color.RedString("NO")
 }
+
+var sectorsCompactPartitionsCmd = &cli.Command{
+	Name:  "compact-partitions",
+	Usage: "removes dead sectors from partitions and reduces the number of partitions used if possible",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:     "deadline",
+			Usage:    "the deadline to compact the partitions in",
+			Required: true,
+		},
+		&cli.Int64SliceFlag{
+			Name:     "partitions",
+			Usage:    "list of partitions to compact sectors in",
+			Required: true,
+		},
+		&cli.BoolFlag{
+			Name:  "really-do-it",
+			Usage: "Actually send transaction performing the action",
+			Value: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Bool("really-do-it") {
+			fmt.Println("Pass --really-do-it to actually execute this action")
+			return nil
+		}
+
+		api, acloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer acloser()
+
+		ctx := lcli.ReqContext(cctx)
+
+		maddr, err := getActorAddress(ctx, cctx)
+		if err != nil {
+			return err
+		}
+
+		minfo, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		deadline := cctx.Uint64("deadline")
+		if err != nil {
+			return err
+		}
+
+		parts := cctx.Int64Slice("partitions")
+		if len(parts) <= 0 {
+			return fmt.Errorf("must include at least one partition to compact")
+		}
+
+		partitions := bitfield.BitField{}
+		for _, partition := range parts {
+			partitions.Set(uint64(partition))
+		}
+
+		params := miner5.CompactPartitionsParams{
+			Deadline:   deadline,
+			Partitions: partitions,
+		}
+
+		sp, err := actors.SerializeParams(&params)
+		if err != nil {
+			return xerrors.Errorf("serializing params: %w", err)
+		}
+
+		smsg, err := api.MpoolPushMessage(ctx, &types.Message{
+			From:   minfo.Worker,
+			To:     maddr,
+			Method: miner.Methods.CompactPartitions,
+			Value:  big.Zero(),
+			Params: sp,
+		}, nil)
+		if err != nil {
+			return xerrors.Errorf("mpool push: %w", err)
+		}
+
+		fmt.Println("Message CID:", smsg.Cid())
+
+		return nil
+	},
+}

cmd/lotus-shed/shedgen/cbor_gen.go

@@ -23,18 +23,19 @@ func (t *CarbNode) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{161}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{161}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Sub ([]cid.Cid) (slice)
 	if len("Sub") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Sub\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Sub"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Sub"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Sub")); err != nil {
@@ -45,27 +46,32 @@ func (t *CarbNode) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Sub was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Sub))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Sub))); err != nil {
 		return err
 	}
 	for _, v := range t.Sub {
-		if err := cbg.WriteCidBuf(scratch, w, v); err != nil {
+		if err := cbg.WriteCid(w, v); err != nil {
 			return xerrors.Errorf("failed writing cid field t.Sub: %w", err)
 		}
 	}
 	return nil
 }
 
-func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
+func (t *CarbNode) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = CarbNode{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -80,7 +86,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -92,7 +98,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
 		// t.Sub ([]cid.Cid) (slice)
 		case "Sub":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -111,7 +117,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
 
 			for i := 0; i < int(extra); i++ {
 
-				c, err := cbg.ReadCid(br)
+				c, err := cbg.ReadCid(cr)
 				if err != nil {
 					return xerrors.Errorf("reading cid field t.Sub failed: %w", err)
 				}

cmd/lotus-shed/verifreg.go

@@ -193,7 +193,7 @@ var verifRegVerifyClientCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
+		fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
 		froms := cctx.String("from")
 		if froms == "" {
 			return fmt.Errorf("must specify from address with --from")
@@ -262,7 +262,7 @@ var verifRegListVerifiersCmd = &cli.Command{
 	Usage:  "list all verifiers",
 	Hidden: true,
 	Action: func(cctx *cli.Context) error {
-		fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
+		fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
 		api, closer, err := lcli.GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -294,7 +294,7 @@ var verifRegListClientsCmd = &cli.Command{
 	Usage:  "list all verified clients",
 	Hidden: true,
 	Action: func(cctx *cli.Context) error {
-		fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
+		fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
 		api, closer, err := lcli.GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -326,7 +326,7 @@ var verifRegCheckClientCmd = &cli.Command{
 	Usage:  "check verified client remaining bytes",
 	Hidden: true,
 	Action: func(cctx *cli.Context) error {
-		fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
+		fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
 		if !cctx.Args().Present() {
 			return fmt.Errorf("must specify client address to check")
 		}
@@ -362,7 +362,7 @@ var verifRegCheckVerifierCmd = &cli.Command{
 	Usage:  "check verifiers remaining bytes",
 	Hidden: true,
 	Action: func(cctx *cli.Context) error {
-		fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
+		fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
 		if !cctx.Args().Present() {
 			return fmt.Errorf("must specify verifier address to check")
 		}

cmd/lotus-worker/main.go

@@ -182,12 +182,16 @@ var runCmd = &cli.Command{
 			Usage: "enable window post",
 			Value: false,
 		},
-
 		&cli.BoolFlag{
 			Name:  "winningpost",
 			Usage: "enable winning post",
 			Value: false,
 		},
+		&cli.BoolFlag{
+			Name:  "no-default",
+			Usage: "disable all default compute tasks, use the worker for storage/fetching only",
+			Value: false,
+		},
 		&cli.IntFlag{
 			Name:  "parallel-fetch-limit",
 			Usage: "maximum fetch operations to run in parallel",
@@ -308,8 +312,11 @@ var runCmd = &cli.Command{
 		}
 
 		if workerType == "" {
-			workerType = sealtasks.WorkerSealing
 			taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFinalizeReplicaUpdate)
+
+			if !cctx.Bool("no-default") {
+				workerType = sealtasks.WorkerSealing
+			}
 		}
 
 		if (workerType == sealtasks.WorkerSealing || cctx.IsSet("addpiece")) && cctx.Bool("addpiece") {
@@ -337,6 +344,10 @@ var runCmd = &cli.Command{
 			taskTypes = append(taskTypes, sealtasks.TTRegenSectorKey)
 		}
 
+		if cctx.Bool("no-default") && workerType == "" {
+			workerType = sealtasks.WorkerSealing
+		}
+
 		if len(taskTypes) == 0 {
 			return xerrors.Errorf("no task types specified")
 		}

conformance/chaos/cbor_gen.go

@@ -28,18 +28,19 @@ func (t *State) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufState); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufState); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Value (string) (string)
 	if len(t.Value) > cbg.MaxLength {
 		return xerrors.Errorf("Value in field t.Value was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Value))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Value))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.Value)); err != nil {
@@ -51,27 +52,32 @@ func (t *State) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Unmarshallable was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Unmarshallable))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Unmarshallable))); err != nil {
 		return err
 	}
 	for _, v := range t.Unmarshallable {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *State) UnmarshalCBOR(r io.Reader) error {
+func (t *State) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = State{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -83,7 +89,7 @@ func (t *State) UnmarshalCBOR(r io.Reader) error {
 	// t.Value (string) (string)
 
 	{
-		sval, err := cbg.ReadStringBuf(br, scratch)
+		sval, err := cbg.ReadString(cr)
 		if err != nil {
 			return err
 		}
@@ -92,7 +98,7 @@ func (t *State) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.Unmarshallable ([]*chaos.UnmarshallableCBOR) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -112,7 +118,7 @@ func (t *State) UnmarshalCBOR(r io.Reader) error {
 	for i := 0; i < int(extra); i++ {
 
 		var v UnmarshallableCBOR
-		if err := v.UnmarshalCBOR(br); err != nil {
+		if err := v.UnmarshalCBOR(cr); err != nil {
 			return err
 		}
 
@@ -129,19 +135,20 @@ func (t *CallerValidationArgs) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufCallerValidationArgs); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufCallerValidationArgs); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Branch (chaos.CallerValidationBranch) (int64)
 	if t.Branch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Branch)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Branch)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Branch-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Branch-1)); err != nil {
 			return err
 		}
 	}
@@ -151,11 +158,11 @@ func (t *CallerValidationArgs) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Addrs was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Addrs))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Addrs))); err != nil {
 		return err
 	}
 	for _, v := range t.Addrs {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
@@ -165,27 +172,32 @@ func (t *CallerValidationArgs) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Types was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Types))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Types))); err != nil {
 		return err
 	}
 	for _, v := range t.Types {
-		if err := cbg.WriteCidBuf(scratch, w, v); err != nil {
+		if err := cbg.WriteCid(w, v); err != nil {
 			return xerrors.Errorf("failed writing cid field t.Types: %w", err)
 		}
 	}
 	return nil
 }
 
-func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error {
+func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = CallerValidationArgs{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -196,7 +208,7 @@ func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error {
 
 	// t.Branch (chaos.CallerValidationBranch) (int64)
 	{
-		maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err := cr.ReadHeader()
 		var extraI int64
 		if err != nil {
 			return err
@@ -221,7 +233,7 @@ func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.Addrs ([]address.Address) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -241,7 +253,7 @@ func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error {
 	for i := 0; i < int(extra); i++ {
 
 		var v address.Address
-		if err := v.UnmarshalCBOR(br); err != nil {
+		if err := v.UnmarshalCBOR(cr); err != nil {
 			return err
 		}
 
@@ -250,7 +262,7 @@ func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error {
 
 	// t.Types ([]cid.Cid) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -269,7 +281,7 @@ func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error {
 
 	for i := 0; i < int(extra); i++ {
 
-		c, err := cbg.ReadCid(br)
+		c, err := cbg.ReadCid(cr)
 		if err != nil {
 			return xerrors.Errorf("reading cid field t.Types failed: %w", err)
 		}
@@ -286,12 +298,13 @@ func (t *CreateActorArgs) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufCreateActorArgs); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufCreateActorArgs); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.UndefActorCID (bool) (bool)
 	if err := cbg.WriteBool(w, t.UndefActorCID); err != nil {
 		return err
@@ -299,7 +312,7 @@ func (t *CreateActorArgs) MarshalCBOR(w io.Writer) error {
 
 	// t.ActorCID (cid.Cid) (struct)
 
-	if err := cbg.WriteCidBuf(scratch, w, t.ActorCID); err != nil {
+	if err := cbg.WriteCid(cw, t.ActorCID); err != nil {
 		return xerrors.Errorf("failed to write cid field t.ActorCID: %w", err)
 	}
 
@@ -309,22 +322,27 @@ func (t *CreateActorArgs) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Address (address.Address) (struct)
-	if err := t.Address.MarshalCBOR(w); err != nil {
+	if err := t.Address.MarshalCBOR(cw); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) error {
+func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = CreateActorArgs{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -335,7 +353,7 @@ func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) error {
 
 	// t.UndefActorCID (bool) (bool)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -354,7 +372,7 @@ func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		c, err := cbg.ReadCid(br)
+		c, err := cbg.ReadCid(cr)
 		if err != nil {
 			return xerrors.Errorf("failed to read cid field t.ActorCID: %w", err)
 		}
@@ -364,7 +382,7 @@ func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.UndefAddress (bool) (bool)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -383,7 +401,7 @@ func (t *CreateActorArgs) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.Address.UnmarshalCBOR(br); err != nil {
+		if err := t.Address.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.Address: %w", err)
 		}
 
@@ -398,12 +416,15 @@ func (t *ResolveAddressResponse) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufResolveAddressResponse); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufResolveAddressResponse); err != nil {
 		return err
 	}
 
 	// t.Address (address.Address) (struct)
-	if err := t.Address.MarshalCBOR(w); err != nil {
+	if err := t.Address.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -414,16 +435,21 @@ func (t *ResolveAddressResponse) MarshalCBOR(w io.Writer) error {
 	return nil
 }
 
-func (t *ResolveAddressResponse) UnmarshalCBOR(r io.Reader) error {
+func (t *ResolveAddressResponse) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = ResolveAddressResponse{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -436,14 +462,14 @@ func (t *ResolveAddressResponse) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.Address.UnmarshalCBOR(br); err != nil {
+		if err := t.Address.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.Address: %w", err)
 		}
 
 	}
 	// t.Success (bool) (bool)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -468,25 +494,26 @@ func (t *SendArgs) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufSendArgs); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufSendArgs); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.To (address.Address) (struct)
-	if err := t.To.MarshalCBOR(w); err != nil {
+	if err := t.To.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
 	// t.Value (big.Int) (struct)
-	if err := t.Value.MarshalCBOR(w); err != nil {
+	if err := t.Value.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
 	// t.Method (abi.MethodNum) (uint64)
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Method)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Method)); err != nil {
 		return err
 	}
 
@@ -495,26 +522,31 @@ func (t *SendArgs) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Byte array in field t.Params was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Params))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Params))); err != nil {
 		return err
 	}
 
-	if _, err := w.Write(t.Params[:]); err != nil {
+	if _, err := cw.Write(t.Params[:]); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (t *SendArgs) UnmarshalCBOR(r io.Reader) error {
+func (t *SendArgs) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = SendArgs{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -527,7 +559,7 @@ func (t *SendArgs) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.To.UnmarshalCBOR(br); err != nil {
+		if err := t.To.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.To: %w", err)
 		}
 
@@ -536,7 +568,7 @@ func (t *SendArgs) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.Value.UnmarshalCBOR(br); err != nil {
+		if err := t.Value.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.Value: %w", err)
 		}
 
@@ -545,7 +577,7 @@ func (t *SendArgs) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err = cr.ReadHeader()
 		if err != nil {
 			return err
 		}
@@ -557,7 +589,7 @@ func (t *SendArgs) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.Params ([]uint8) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -573,7 +605,7 @@ func (t *SendArgs) UnmarshalCBOR(r io.Reader) error {
 		t.Params = make([]uint8, extra)
 	}
 
-	if _, err := io.ReadFull(br, t.Params[:]); err != nil {
+	if _, err := io.ReadFull(cr, t.Params[:]); err != nil {
 		return err
 	}
 	return nil
@@ -586,48 +618,54 @@ func (t *SendReturn) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufSendReturn); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufSendReturn); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Return (builtin.CBORBytes) (slice)
 	if len(t.Return) > cbg.ByteArrayMaxLen {
 		return xerrors.Errorf("Byte array in field t.Return was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Return))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Return))); err != nil {
 		return err
 	}
 
-	if _, err := w.Write(t.Return[:]); err != nil {
+	if _, err := cw.Write(t.Return[:]); err != nil {
 		return err
 	}
 
 	// t.Code (exitcode.ExitCode) (int64)
 	if t.Code >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Code)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Code)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Code-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Code-1)); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *SendReturn) UnmarshalCBOR(r io.Reader) error {
+func (t *SendReturn) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = SendReturn{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -638,7 +676,7 @@ func (t *SendReturn) UnmarshalCBOR(r io.Reader) error {
 
 	// t.Return (builtin.CBORBytes) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -654,12 +692,12 @@ func (t *SendReturn) UnmarshalCBOR(r io.Reader) error {
 		t.Return = make([]uint8, extra)
 	}
 
-	if _, err := io.ReadFull(br, t.Return[:]); err != nil {
+	if _, err := io.ReadFull(cr, t.Return[:]); err != nil {
 		return err
 	}
 	// t.Code (exitcode.ExitCode) (int64)
 	{
-		maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err := cr.ReadHeader()
 		var extraI int64
 		if err != nil {
 			return err
@@ -692,18 +730,19 @@ func (t *MutateStateArgs) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufMutateStateArgs); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufMutateStateArgs); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Value (string) (string)
 	if len(t.Value) > cbg.MaxLength {
 		return xerrors.Errorf("Value in field t.Value was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Value))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Value))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.Value)); err != nil {
@@ -712,27 +751,32 @@ func (t *MutateStateArgs) MarshalCBOR(w io.Writer) error {
 
 	// t.Branch (chaos.MutateStateBranch) (int64)
 	if t.Branch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Branch)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Branch)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Branch-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Branch-1)); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *MutateStateArgs) UnmarshalCBOR(r io.Reader) error {
+func (t *MutateStateArgs) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = MutateStateArgs{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -744,7 +788,7 @@ func (t *MutateStateArgs) UnmarshalCBOR(r io.Reader) error {
 	// t.Value (string) (string)
 
 	{
-		sval, err := cbg.ReadStringBuf(br, scratch)
+		sval, err := cbg.ReadString(cr)
 		if err != nil {
 			return err
 		}
@@ -753,7 +797,7 @@ func (t *MutateStateArgs) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.Branch (chaos.MutateStateBranch) (int64)
 	{
-		maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err := cr.ReadHeader()
 		var extraI int64
 		if err != nil {
 			return err
@@ -786,19 +830,20 @@ func (t *AbortWithArgs) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufAbortWithArgs); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufAbortWithArgs); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Code (exitcode.ExitCode) (int64)
 	if t.Code >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Code)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Code)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Code-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Code-1)); err != nil {
 			return err
 		}
 	}
@@ -808,7 +853,7 @@ func (t *AbortWithArgs) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.Message was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Message))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Message))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.Message)); err != nil {
@@ -822,16 +867,21 @@ func (t *AbortWithArgs) MarshalCBOR(w io.Writer) error {
 	return nil
 }
 
-func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error {
+func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = AbortWithArgs{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -842,7 +892,7 @@ func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error {
 
 	// t.Code (exitcode.ExitCode) (int64)
 	{
-		maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err := cr.ReadHeader()
 		var extraI int64
 		if err != nil {
 			return err
@@ -868,7 +918,7 @@ func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error {
 	// t.Message (string) (string)
 
 	{
-		sval, err := cbg.ReadStringBuf(br, scratch)
+		sval, err := cbg.ReadString(cr)
 		if err != nil {
 			return err
 		}
@@ -877,7 +927,7 @@ func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.Uncontrolled (bool) (bool)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -902,60 +952,66 @@ func (t *InspectRuntimeReturn) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufInspectRuntimeReturn); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufInspectRuntimeReturn); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Caller (address.Address) (struct)
-	if err := t.Caller.MarshalCBOR(w); err != nil {
+	if err := t.Caller.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
 	// t.Receiver (address.Address) (struct)
-	if err := t.Receiver.MarshalCBOR(w); err != nil {
+	if err := t.Receiver.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
 	// t.ValueReceived (big.Int) (struct)
-	if err := t.ValueReceived.MarshalCBOR(w); err != nil {
+	if err := t.ValueReceived.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
 	// t.CurrEpoch (abi.ChainEpoch) (int64)
 	if t.CurrEpoch >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.CurrEpoch)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.CurrEpoch)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.CurrEpoch-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.CurrEpoch-1)); err != nil {
 			return err
 		}
 	}
 
 	// t.CurrentBalance (big.Int) (struct)
-	if err := t.CurrentBalance.MarshalCBOR(w); err != nil {
+	if err := t.CurrentBalance.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
 	// t.State (chaos.State) (struct)
-	if err := t.State.MarshalCBOR(w); err != nil {
+	if err := t.State.MarshalCBOR(cw); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error {
+func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = InspectRuntimeReturn{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -968,7 +1024,7 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.Caller.UnmarshalCBOR(br); err != nil {
+		if err := t.Caller.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.Caller: %w", err)
 		}
 
@@ -977,7 +1033,7 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.Receiver.UnmarshalCBOR(br); err != nil {
+		if err := t.Receiver.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.Receiver: %w", err)
 		}
 
@@ -986,14 +1042,14 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.ValueReceived.UnmarshalCBOR(br); err != nil {
+		if err := t.ValueReceived.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.ValueReceived: %w", err)
 		}
 
 	}
 	// t.CurrEpoch (abi.ChainEpoch) (int64)
 	{
-		maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err := cr.ReadHeader()
 		var extraI int64
 		if err != nil {
 			return err
@@ -1020,7 +1076,7 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.CurrentBalance.UnmarshalCBOR(br); err != nil {
+		if err := t.CurrentBalance.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.CurrentBalance: %w", err)
 		}
 
@@ -1029,7 +1085,7 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.State.UnmarshalCBOR(br); err != nil {
+		if err := t.State.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.State: %w", err)
 		}
 

documentation/en/api-v0-methods-miner.md

@@ -27,6 +27,7 @@
   * [DagstoreListShards](#DagstoreListShards)
   * [DagstoreLookupPieces](#DagstoreLookupPieces)
   * [DagstoreRecoverShard](#DagstoreRecoverShard)
+  * [DagstoreRegisterShard](#DagstoreRegisterShard)
 * [Deals](#Deals)
   * [DealsConsiderOfflineRetrievalDeals](#DealsConsiderOfflineRetrievalDeals)
   * [DealsConsiderOfflineStorageDeals](#DealsConsiderOfflineStorageDeals)
@@ -639,6 +640,21 @@ Inputs:
 
 Response: `{}`
 
+### DagstoreRegisterShard
+DagstoreRegisterShard registers a shard manually with dagstore with given pieceCID
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  "string value"
+]
+```
+
+Response: `{}`
+
 ## Deals
 
 

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.3-dev
+   1.15.4-dev
 
 COMMANDS:
    init     Initialize a lotus miner repo
@@ -1035,6 +1035,7 @@ USAGE:
 
 COMMANDS:
    list-shards       List all shards known to the dagstore, with their current status
+   register-shard    Register a shard
    initialize-shard  Initialize the specified shard
    recover-shard     Attempt to recover a shard in errored state
    initialize-all    Initialize all uninitialized shards, streaming results as they're produced; only shards for unsealed pieces are initialized by default
@@ -1061,6 +1062,20 @@ OPTIONS:
    
 ```
 
+### lotus-miner dagstore register-shard
+```
+NAME:
+   lotus-miner dagstore register-shard - Register a shard
+
+USAGE:
+   lotus-miner dagstore register-shard [command options] [key]
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+   
+```
+
 ### lotus-miner dagstore initialize-shard
 ```
 NAME:
@@ -1652,6 +1667,7 @@ COMMANDS:
    refs                  List References to sectors
    update-state          ADVANCED: manually update the state of a sector, this may aid in error recovery
    pledge                store random data in a sector
+   precommits            Print on-chain precommit info
    check-expire          Inspect expiring sectors
    expired               Get or cleanup expired sectors
    renew                 Renew expiring sectors while not exceeding each sector's max life
@@ -1665,6 +1681,7 @@ COMMANDS:
    get-cc-collateral     Get the collateral required to pledge a committed capacity sector
    batching              manage batch sector operations
    match-pending-pieces  force a refreshed match of pending pieces to open sectors without manually waiting for more deals
+   compact-partitions    removes dead sectors from partitions and reduces the number of partitions used if possible
    help, h               Shows a list of commands or help for one command
 
 OPTIONS:
@@ -1750,6 +1767,19 @@ OPTIONS:
    
 ```
 
+### lotus-miner sectors precommits
+```
+NAME:
+   lotus-miner sectors precommits - Print on-chain precommit info
+
+USAGE:
+   lotus-miner sectors precommits [command options] [arguments...]
+
+OPTIONS:
+   --help, -h  show help (default: false)
+   
+```
+
 ### lotus-miner sectors check-expire
 ```
 NAME:
@@ -2006,6 +2036,22 @@ OPTIONS:
    
 ```
 
+### lotus-miner sectors compact-partitions
+```
+NAME:
+   lotus-miner sectors compact-partitions - removes dead sectors from partitions and reduces the number of partitions used if possible
+
+USAGE:
+   lotus-miner sectors compact-partitions [command options] [arguments...]
+
+OPTIONS:
+   --deadline value    the deadline to compact the partitions in (default: 0)
+   --partitions value  list of partitions to compact sectors in
+   --really-do-it      Actually send transaction performing the action (default: false)
+   --help, -h          show help (default: false)
+   
+```
+
 ## lotus-miner proving
 ```
 NAME:
@@ -2285,6 +2331,7 @@ COMMANDS:
    workers     list workers
    sched-diag  Dump internal scheduler state
    abort       Abort a running job
+   data-cid    Compute data CID using workers
    help, h     Shows a list of commands or help for one command
 
 OPTIONS:
@@ -2347,3 +2394,17 @@ OPTIONS:
    --help, -h  show help (default: false)
    
 ```
+
+### lotus-miner sealing data-cid
+```
+NAME:
+   lotus-miner sealing data-cid - Compute data CID using workers
+
+USAGE:
+   lotus-miner sealing data-cid [command options] [file/url] <padded piece size>
+
+OPTIONS:
+   --file-size value  real file size (default: 0)
+   --help, -h         show help (default: false)
+   
+```

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.3-dev
+   1.15.4-dev
 
 COMMANDS:
    run         Start lotus worker
@@ -49,6 +49,7 @@ OPTIONS:
    --regen-sector-key            enable regen sector key (default: true)
    --windowpost                  enable window post (default: false)
    --winningpost                 enable winning post (default: false)
+   --no-default                  disable all default compute tasks, use the worker for storage/fetching only (default: false)
    --parallel-fetch-limit value  maximum fetch operations to run in parallel (default: 5)
    --post-parallel-reads value   maximum number of parallel challenge reads (0 = no limit) (default: 128)
    --post-read-timeout value     time limit for reading PoSt challenges (0 = no limit) (default: 0s)

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.15.3-dev
+   1.15.4-dev
 
 COMMANDS:
    daemon   Start a lotus daemon process

documentation/en/default-lotus-miner-config.toml

@@ -444,12 +444,14 @@
   # env var: LOTUS_SEALING_AGGREGATECOMMITS
   #AggregateCommits = true
 
-  # maximum batched commit size - batches will be sent immediately above this size
+  # minimum batched commit size - batches above this size will eventually be sent on a timeout
   #
   # type: int
   # env var: LOTUS_SEALING_MINCOMMITBATCH
   #MinCommitBatch = 4
 
+  # maximum batched commit size - batches will be sent immediately above this size
+  #
   # type: int
   # env var: LOTUS_SEALING_MAXCOMMITBATCH
   #MaxCommitBatch = 819

extern/sector-storage/cbor_gen.go

@@ -24,25 +24,26 @@ func (t *Call) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{164}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{164}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.ID (storiface.CallID) (struct)
 	if len("ID") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"ID\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ID"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ID"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("ID")); err != nil {
 		return err
 	}
 
-	if err := t.ID.MarshalCBOR(w); err != nil {
+	if err := t.ID.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -51,7 +52,7 @@ func (t *Call) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"RetType\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("RetType"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RetType"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("RetType")); err != nil {
@@ -62,7 +63,7 @@ func (t *Call) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.RetType was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.RetType))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.RetType))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.RetType)); err != nil {
@@ -74,14 +75,14 @@ func (t *Call) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"State\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("State"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("State"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("State")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.State)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.State)); err != nil {
 		return err
 	}
 
@@ -90,29 +91,34 @@ func (t *Call) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Result\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Result"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Result"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Result")); err != nil {
 		return err
 	}
 
-	if err := t.Result.MarshalCBOR(w); err != nil {
+	if err := t.Result.MarshalCBOR(cw); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (t *Call) UnmarshalCBOR(r io.Reader) error {
+func (t *Call) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = Call{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -127,7 +133,7 @@ func (t *Call) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -141,7 +147,7 @@ func (t *Call) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.ID.UnmarshalCBOR(br); err != nil {
+				if err := t.ID.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.ID: %w", err)
 				}
 
@@ -150,7 +156,7 @@ func (t *Call) UnmarshalCBOR(r io.Reader) error {
 		case "RetType":
 
 			{
-				sval, err := cbg.ReadStringBuf(br, scratch)
+				sval, err := cbg.ReadString(cr)
 				if err != nil {
 					return err
 				}
@@ -162,7 +168,7 @@ func (t *Call) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err = cr.ReadHeader()
 				if err != nil {
 					return err
 				}
@@ -177,16 +183,16 @@ func (t *Call) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				b, err := br.ReadByte()
+				b, err := cr.ReadByte()
 				if err != nil {
 					return err
 				}
 				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
+					if err := cr.UnreadByte(); err != nil {
 						return err
 					}
 					t.Result = new(ManyBytes)
-					if err := t.Result.UnmarshalCBOR(br); err != nil {
+					if err := t.Result.UnmarshalCBOR(cr); err != nil {
 						return xerrors.Errorf("unmarshaling t.Result pointer: %w", err)
 					}
 				}
@@ -206,25 +212,26 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{166}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{166}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.ID (sectorstorage.WorkID) (struct)
 	if len("ID") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"ID\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ID"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ID"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("ID")); err != nil {
 		return err
 	}
 
-	if err := t.ID.MarshalCBOR(w); err != nil {
+	if err := t.ID.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -233,7 +240,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Status\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Status"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Status"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Status")); err != nil {
@@ -244,7 +251,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.Status was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Status))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Status))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.Status)); err != nil {
@@ -256,14 +263,14 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"WorkerCall\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WorkerCall"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("WorkerCall"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("WorkerCall")); err != nil {
 		return err
 	}
 
-	if err := t.WorkerCall.MarshalCBOR(w); err != nil {
+	if err := t.WorkerCall.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -272,7 +279,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"WorkError\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WorkError"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("WorkError"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("WorkError")); err != nil {
@@ -283,7 +290,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.WorkError was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.WorkError))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.WorkError))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.WorkError)); err != nil {
@@ -295,7 +302,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"WorkerHostname\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("WorkerHostname"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("WorkerHostname"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("WorkerHostname")); err != nil {
@@ -306,7 +313,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.WorkerHostname was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.WorkerHostname))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.WorkerHostname))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.WorkerHostname)); err != nil {
@@ -318,7 +325,7 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"StartTime\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("StartTime"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("StartTime"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("StartTime")); err != nil {
@@ -326,27 +333,32 @@ func (t *WorkState) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.StartTime >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.StartTime)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.StartTime)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.StartTime-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.StartTime-1)); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *WorkState) UnmarshalCBOR(r io.Reader) error {
+func (t *WorkState) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = WorkState{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -361,7 +373,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -375,7 +387,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.ID.UnmarshalCBOR(br); err != nil {
+				if err := t.ID.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.ID: %w", err)
 				}
 
@@ -384,7 +396,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error {
 		case "Status":
 
 			{
-				sval, err := cbg.ReadStringBuf(br, scratch)
+				sval, err := cbg.ReadString(cr)
 				if err != nil {
 					return err
 				}
@@ -396,7 +408,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.WorkerCall.UnmarshalCBOR(br); err != nil {
+				if err := t.WorkerCall.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.WorkerCall: %w", err)
 				}
 
@@ -405,7 +417,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error {
 		case "WorkError":
 
 			{
-				sval, err := cbg.ReadStringBuf(br, scratch)
+				sval, err := cbg.ReadString(cr)
 				if err != nil {
 					return err
 				}
@@ -416,7 +428,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error {
 		case "WorkerHostname":
 
 			{
-				sval, err := cbg.ReadStringBuf(br, scratch)
+				sval, err := cbg.ReadString(cr)
 				if err != nil {
 					return err
 				}
@@ -426,7 +438,7 @@ func (t *WorkState) UnmarshalCBOR(r io.Reader) error {
 			// t.StartTime (int64) (int64)
 		case "StartTime":
 			{
-				maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err := cr.ReadHeader()
 				var extraI int64
 				if err != nil {
 					return err
@@ -463,18 +475,19 @@ func (t *WorkID) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{162}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Method (sealtasks.TaskType) (string)
 	if len("Method") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Method\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Method"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Method"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Method")); err != nil {
@@ -485,7 +498,7 @@ func (t *WorkID) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.Method was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Method))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Method))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.Method)); err != nil {
@@ -497,7 +510,7 @@ func (t *WorkID) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Params\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Params"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Params"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Params")); err != nil {
@@ -508,7 +521,7 @@ func (t *WorkID) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.Params was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Params))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Params))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.Params)); err != nil {
@@ -517,16 +530,21 @@ func (t *WorkID) MarshalCBOR(w io.Writer) error {
 	return nil
 }
 
-func (t *WorkID) UnmarshalCBOR(r io.Reader) error {
+func (t *WorkID) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = WorkID{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -541,7 +559,7 @@ func (t *WorkID) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -554,7 +572,7 @@ func (t *WorkID) UnmarshalCBOR(r io.Reader) error {
 		case "Method":
 
 			{
-				sval, err := cbg.ReadStringBuf(br, scratch)
+				sval, err := cbg.ReadString(cr)
 				if err != nil {
 					return err
 				}
@@ -565,7 +583,7 @@ func (t *WorkID) UnmarshalCBOR(r io.Reader) error {
 		case "Params":
 
 			{
-				sval, err := cbg.ReadStringBuf(br, scratch)
+				sval, err := cbg.ReadString(cr)
 				if err != nil {
 					return err
 				}

extern/sector-storage/ffiwrapper/sealer_cgo.go

@@ -31,6 +31,8 @@ import (
 	"github.com/filecoin-project/lotus/extern/sector-storage/fr32"
 	"github.com/filecoin-project/lotus/extern/sector-storage/partialfile"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
+	nr "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader"
+	"github.com/filecoin-project/lotus/lib/nullreader"
 )
 
 var _ Storage = &Sealer{}
@@ -52,6 +54,11 @@ func (sb *Sealer) NewSector(ctx context.Context, sector storage.SectorRef) error
 }
 
 func (sb *Sealer) DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData storage.Data) (abi.PieceInfo, error) {
+	pieceData = io.LimitReader(io.MultiReader(
+		pieceData,
+		nullreader.Reader{},
+	), int64(pieceSize))
+
 	// TODO: allow tuning those:
 	chunk := abi.PaddedPieceSize(4 << 20)
 	parallel := runtime.NumCPU()
@@ -72,6 +79,7 @@ func (sb *Sealer) DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize,
 	for {
 		var read int
 		for rbuf := buf; len(rbuf) > 0; {
+
 			n, err := pieceData.Read(rbuf)
 			if err != nil && err != io.EOF {
 				return abi.PieceInfo{}, xerrors.Errorf("pr read error: %w", err)
@@ -369,8 +377,8 @@ func (sb *Sealer) pieceCid(spt abi.RegisteredSealProof, in []byte) (cid.Cid, err
 	return pieceCID, werr()
 }
 
-func (sb *Sealer) tryDecodeUpdatedReplica(ctx context.Context, sector storage.SectorRef, commD cid.Cid, unsealedPath string) (bool, error) {
-	paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUpdate|storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage)
+func (sb *Sealer) tryDecodeUpdatedReplica(ctx context.Context, sector storage.SectorRef, commD cid.Cid, unsealedPath string, randomness abi.SealRandomness) (bool, error) {
+	replicaPath, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTNone, storiface.PathStorage)
 	if xerrors.Is(err, storiface.ErrSectorNotFound) {
 		return false, nil
 	} else if err != nil {
@@ -378,12 +386,47 @@ func (sb *Sealer) tryDecodeUpdatedReplica(ctx context.Context, sector storage.Se
 	}
 	defer done()
 
+	sealedPaths, done2, err := sb.AcquireSectorKeyOrRegenerate(ctx, sector, randomness)
+	if err != nil {
+		return false, xerrors.Errorf("acquiring sealed sector: %w", err)
+	}
+	defer done2()
+
 	// Sector data stored in replica update
 	updateProof, err := sector.ProofType.RegisteredUpdateProof()
 	if err != nil {
 		return false, err
 	}
-	return true, ffi.SectorUpdate.DecodeFrom(updateProof, unsealedPath, paths.Update, paths.Sealed, paths.Cache, commD)
+	return true, ffi.SectorUpdate.DecodeFrom(updateProof, unsealedPath, replicaPath.Update, sealedPaths.Sealed, sealedPaths.Cache, commD)
+}
+
+func (sb *Sealer) AcquireSectorKeyOrRegenerate(ctx context.Context, sector storage.SectorRef, randomness abi.SealRandomness) (storiface.SectorPaths, func(), error) {
+	paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage)
+	if err == nil {
+		return paths, done, err
+	} else if !xerrors.Is(err, storiface.ErrSectorNotFound) {
+		return paths, done, xerrors.Errorf("reading sector key: %w", err)
+	}
+
+	// Sector key can't be found, so let's regenerate it
+	sectorSize, err := sector.ProofType.SectorSize()
+	if err != nil {
+		return paths, done, xerrors.Errorf("retrieving sector size: %w", err)
+	}
+	paddedSize := abi.PaddedPieceSize(sectorSize)
+
+	_, err = sb.AddPiece(ctx, sector, nil, paddedSize.Unpadded(), nr.NewNullReader(paddedSize.Unpadded()))
+	if err != nil {
+		return paths, done, xerrors.Errorf("recomputing empty data: %w", err)
+	}
+
+	err = sb.RegenerateSectorKey(ctx, sector, randomness, []abi.PieceInfo{{PieceCID: zerocomm.ZeroPieceCommitment(paddedSize.Unpadded()), Size: paddedSize}})
+	if err != nil {
+		return paths, done, xerrors.Errorf("during pc1: %w", err)
+	}
+
+	// Sector key should exist now, let's grab the paths
+	return sb.sectors.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage)
 }
 
 func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd cid.Cid) error {
@@ -437,7 +480,7 @@ func (sb *Sealer) UnsealPiece(ctx context.Context, sector storage.SectorRef, off
 	}
 
 	// If piece data stored in updated replica decode whole sector
-	decoded, err := sb.tryDecodeUpdatedReplica(ctx, sector, commd, unsealedPath.Unsealed)
+	decoded, err := sb.tryDecodeUpdatedReplica(ctx, sector, commd, unsealedPath.Unsealed, randomness)
 	if err != nil {
 		return xerrors.Errorf("decoding sector from replica: %w", err)
 	}
@@ -618,6 +661,51 @@ func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector storag
 	return true, nil
 }
 
+func (sb *Sealer) RegenerateSectorKey(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) error {
+	paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed|storiface.FTCache, storiface.FTSealed, storiface.PathSealing)
+	if err != nil {
+		return xerrors.Errorf("acquiring sector paths: %w", err)
+	}
+	defer done()
+
+	e, err := os.OpenFile(paths.Sealed, os.O_RDWR|os.O_CREATE, 0644) // nolint:gosec
+	if err != nil {
+		return xerrors.Errorf("ensuring sealed file exists: %w", err)
+	}
+	if err := e.Close(); err != nil {
+		return err
+	}
+
+	var sum abi.UnpaddedPieceSize
+	for _, piece := range pieces {
+		sum += piece.Size.Unpadded()
+	}
+	ssize, err := sector.ProofType.SectorSize()
+	if err != nil {
+		return err
+	}
+	ussize := abi.PaddedPieceSize(ssize).Unpadded()
+	if sum != ussize {
+		return xerrors.Errorf("aggregated piece sizes don't match sector size: %d != %d (%d)", sum, ussize, int64(ussize-sum))
+	}
+
+	// TODO: context cancellation respect
+	_, err = ffi.SealPreCommitPhase1(
+		sector.ProofType,
+		paths.Cache,
+		paths.Unsealed,
+		paths.Sealed,
+		sector.ID.Number,
+		sector.ID.Miner,
+		ticket,
+		pieces,
+	)
+	if err != nil {
+		return xerrors.Errorf("presealing sector %d (%s): %w", sector.ID.Number, paths.Unsealed, err)
+	}
+	return nil
+}
+
 func (sb *Sealer) SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (out storage.PreCommit1Out, err error) {
 	paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTUnsealed, storiface.FTSealed|storiface.FTCache, storiface.PathSealing)
 	if err != nil {

extern/sector-storage/manager.go

@@ -273,7 +273,7 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storage.SectorR
 	defer cancel()
 
 	log.Debugf("acquire unseal sector lock for sector %d", sector.ID)
-	if err := m.index.StorageLock(ctx, sector.ID, storiface.FTSealed|storiface.FTCache, storiface.FTUnsealed); err != nil {
+	if err := m.index.StorageLock(ctx, sector.ID, storiface.FTSealed|storiface.FTCache|storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTUnsealed); err != nil {
 		return xerrors.Errorf("acquiring unseal sector lock: %w", err)
 	}
 
@@ -281,8 +281,11 @@ func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storage.SectorR
 	// put it in the sealing scratch space.
 	sealFetch := func(ctx context.Context, worker Worker) error {
 		log.Debugf("copy sealed/cache sector data for sector %d", sector.ID)
-		if _, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy)); err != nil {
-			return xerrors.Errorf("copy sealed/cache sector data: %w", err)
+		_, err := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.PathSealing, storiface.AcquireCopy))
+		_, err2 := m.waitSimpleCall(ctx)(worker.Fetch(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.PathSealing, storiface.AcquireCopy))
+
+		if err != nil && err2 != nil {
+			return xerrors.Errorf("cannot unseal piece. error fetching sealed data: %w. error fetching replica data: %w", err, err2)
 		}
 
 		return nil

extern/sector-storage/manager_test.go

@@ -315,6 +315,12 @@ func TestSnapDeals(t *testing.T) {
 	require.NoError(t, m.GenerateSectorKeyFromData(ctx, sid, out.NewUnsealed))
 	fmt.Printf("GSK duration (%s): %s\n", ss.ShortString(), time.Since(startGSK))
 
+	fmt.Printf("Remove data\n")
+	require.NoError(t, m.FinalizeSector(ctx, sid, nil))
+	fmt.Printf("Release Sector Key\n")
+	require.NoError(t, m.ReleaseSectorKey(ctx, sid))
+	fmt.Printf("Unseal Replica\n")
+	require.NoError(t, m.SectorsUnsealPiece(ctx, sid, 0, p1.Size.Unpadded(), ticket, &out.NewUnsealed))
 }
 
 func TestRedoPC1(t *testing.T) {

extern/sector-storage/storiface/cbor_gen.go

@@ -23,25 +23,26 @@ func (t *CallID) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{162}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Sector (abi.SectorID) (struct)
 	if len("Sector") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Sector\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Sector"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Sector"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Sector")); err != nil {
 		return err
 	}
 
-	if err := t.Sector.MarshalCBOR(w); err != nil {
+	if err := t.Sector.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -50,7 +51,7 @@ func (t *CallID) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"ID\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ID"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ID"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("ID")); err != nil {
@@ -61,26 +62,31 @@ func (t *CallID) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Byte array in field t.ID was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.ID))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.ID))); err != nil {
 		return err
 	}
 
-	if _, err := w.Write(t.ID[:]); err != nil {
+	if _, err := cw.Write(t.ID[:]); err != nil {
 		return err
 	}
 	return nil
 }
 
-func (t *CallID) UnmarshalCBOR(r io.Reader) error {
+func (t *CallID) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = CallID{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -95,7 +101,7 @@ func (t *CallID) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -109,7 +115,7 @@ func (t *CallID) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.Sector.UnmarshalCBOR(br); err != nil {
+				if err := t.Sector.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.Sector: %w", err)
 				}
 
@@ -117,7 +123,7 @@ func (t *CallID) UnmarshalCBOR(r io.Reader) error {
 			// t.ID (uuid.UUID) (array)
 		case "ID":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -135,7 +141,7 @@ func (t *CallID) UnmarshalCBOR(r io.Reader) error {
 
 			t.ID = [16]uint8{}
 
-			if _, err := io.ReadFull(br, t.ID[:]); err != nil {
+			if _, err := io.ReadFull(cr, t.ID[:]); err != nil {
 				return err
 			}
 

extern/sector-storage/worker_local.go

@@ -289,7 +289,7 @@ func (l *LocalWorker) asyncCall(ctx context.Context, sector storage.SectorRef, r
 				log.Errorf("get hostname err: %+v", err)
 			}
 
-			err = xerrors.Errorf("%w [Hostname: %s]", err.Error(), hostname)
+			err = xerrors.Errorf("%w [Hostname: %s]", err, hostname)
 		}
 
 		if doReturn(ctx, rt, ci, l.ret, res, toCallError(err)) {

extern/storage-sealing/input.go

@@ -22,6 +22,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader"
 	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 )
 
@@ -220,7 +221,7 @@ func (m *Sealing) handleAddPiece(ctx statemachine.Context, sector SectorInfo) er
 				m.minerSector(sector.SectorType, sector.SectorNumber),
 				pieceSizes,
 				p.Unpadded(),
-				NewNullReader(p.Unpadded()))
+				nullreader.NewNullReader(p.Unpadded()))
 			if err != nil {
 				err = xerrors.Errorf("writing padding piece: %w", err)
 				deal.accepted(sector.SectorNumber, offset, err)

extern/storage-sealing/lib/nullreader/nullreader.go

@@ -1,5 +1,23 @@
 package nullreader
 
+import (
+	"io"
+
+	"github.com/filecoin-project/go-state-types/abi"
+)
+
+type NullReader struct {
+	*io.LimitedReader
+}
+
+func NewNullReader(size abi.UnpaddedPieceSize) io.Reader {
+	return &NullReader{(io.LimitReader(&Reader{}, int64(size))).(*io.LimitedReader)}
+}
+
+func (m NullReader) NullBytes() int64 {
+	return m.N
+}
+
 // TODO: extract this to someplace where it can be shared with lotus
 type Reader struct{}
 

extern/storage-sealing/nullreader.go

@@ -1,20 +0,0 @@
-package sealing
-
-import (
-	"io"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	nr "github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader"
-)
-
-type NullReader struct {
-	*io.LimitedReader
-}
-
-func NewNullReader(size abi.UnpaddedPieceSize) io.Reader {
-	return &NullReader{(io.LimitReader(&nr.Reader{}, int64(size))).(*io.LimitedReader)}
-}
-
-func (m NullReader) NullBytes() int64 {
-	return m.N
-}

extern/storage-sealing/states_sealing.go

@@ -21,6 +21,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
+	"github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader"
 )
 
 var DealSectorPriority = 1024
@@ -91,7 +92,7 @@ func (m *Sealing) padSector(ctx context.Context, sectorID storage.SectorRef, exi
 	for i, size := range sizes {
 		expectCid := zerocomm.ZeroPieceCommitment(size)
 
-		ppi, err := m.sealer.AddPiece(ctx, sectorID, existingPieceSizes, size, NewNullReader(size))
+		ppi, err := m.sealer.AddPiece(ctx, sectorID, existingPieceSizes, size, nullreader.NewNullReader(size))
 		if err != nil {
 			return nil, xerrors.Errorf("add piece: %w", err)
 		}

go.mod

@@ -41,7 +41,7 @@ require (
 	github.com/filecoin-project/go-legs v0.3.7
 	github.com/filecoin-project/go-padreader v0.0.1
 	github.com/filecoin-project/go-paramfetch v0.0.4
-	github.com/filecoin-project/go-state-types v0.1.3
+	github.com/filecoin-project/go-state-types v0.1.4
 	github.com/filecoin-project/go-statemachine v1.0.2
 	github.com/filecoin-project/go-statestore v0.2.0
 	github.com/filecoin-project/go-storedcounter v0.1.0
@@ -109,7 +109,7 @@ require (
 	github.com/koalacxr/quantile v0.0.1
 	github.com/libp2p/go-buffer-pool v0.0.2
 	github.com/libp2p/go-eventbus v0.2.1
-	github.com/libp2p/go-libp2p v0.19.0
+	github.com/libp2p/go-libp2p v0.19.3
 	github.com/libp2p/go-libp2p-connmgr v0.3.1
 	github.com/libp2p/go-libp2p-core v0.15.1
 	github.com/libp2p/go-libp2p-discovery v0.6.0
@@ -145,7 +145,7 @@ require (
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/urfave/cli/v2 v2.3.0
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
-	github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14
+	github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799
 	github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
 	github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542
@@ -277,7 +277,7 @@ require (
 	github.com/libp2p/go-tcp-transport v0.5.1 // indirect
 	github.com/libp2p/go-ws-transport v0.6.0 // indirect
 	github.com/libp2p/go-yamux/v3 v3.1.1 // indirect
-	github.com/lucas-clemente/quic-go v0.27.0 // indirect
+	github.com/lucas-clemente/quic-go v0.27.1 // indirect
 	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
 	github.com/magefile/mage v1.9.0 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect

go.sum

@@ -368,8 +368,9 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go
 github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.1-0.20210810190654-139e0e79e69e/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.1/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
-github.com/filecoin-project/go-state-types v0.1.3 h1:rzIJyQo5HO2ptc8Jcu8P0qTutnI7NWwTle54eAHoNO0=
 github.com/filecoin-project/go-state-types v0.1.3/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
+github.com/filecoin-project/go-state-types v0.1.4 h1:NU0veVNxtDiLD/eRyKHV9lv3njSzfTh/sJGxxvcYcpg=
+github.com/filecoin-project/go-state-types v0.1.4/go.mod h1:xCA/WfKlC2zcn3fUmDv4IrzznwS98X5XW/irUP3Lhxg=
 github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
 github.com/filecoin-project/go-statemachine v1.0.1/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54=
 github.com/filecoin-project/go-statemachine v1.0.2-0.20220322104818-27f8fbb86dfd/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54=
@@ -1086,8 +1087,8 @@ github.com/libp2p/go-libp2p v0.16.0/go.mod h1:ump42BsirwAWxKzsCiFnTtN1Yc+DuPu76f
 github.com/libp2p/go-libp2p v0.17.0/go.mod h1:Fkin50rsGdv5mm5BshBUtPRZknt9esfmYXBOYcwOTgw=
 github.com/libp2p/go-libp2p v0.18.0-rc1/go.mod h1:RgYlH7IIWHXREimC92bw5Lg1V2R5XmSzuLHb5fTnr+8=
 github.com/libp2p/go-libp2p v0.18.0-rc3/go.mod h1:WYL+Xw1iuwi6rdfzw5VIEpD+HqzYucHZ6fcUuumbI3M=
-github.com/libp2p/go-libp2p v0.19.0 h1:zosskMbaobL7UDCVLEe1m5CGs1TaFNFoN/M5XLiKg0U=
-github.com/libp2p/go-libp2p v0.19.0/go.mod h1:Ki9jJXLO2YqrTIFxofV7Twyd3INWPT97+r8hGt7XPjI=
+github.com/libp2p/go-libp2p v0.19.3 h1:LqjvuBWdyYSqvkH4VVYxA78Fkphzg2Pq86VMnilqgkw=
+github.com/libp2p/go-libp2p v0.19.3/go.mod h1:AGlPVLjh0+6jvEtf+a2gZEux7yHJrYXnG9IC7wcQ2NY=
 github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo=
 github.com/libp2p/go-libp2p-asn-util v0.1.0 h1:rABPCO77SjdbJ/eJ/ynIo8vWICy1VEnL5JAxJbQLo1E=
 github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I=
@@ -1457,8 +1458,9 @@ github.com/lucas-clemente/quic-go v0.21.2/go.mod h1:vF5M1XqhBAHgbjKcJOXY3JZz3GP0
 github.com/lucas-clemente/quic-go v0.23.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0=
 github.com/lucas-clemente/quic-go v0.24.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0=
 github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg=
-github.com/lucas-clemente/quic-go v0.27.0 h1:v6WY87q9zD4dKASbG8hy/LpzAVNzEQzw8sEIeloJsc4=
 github.com/lucas-clemente/quic-go v0.27.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
+github.com/lucas-clemente/quic-go v0.27.1 h1:sOw+4kFSVrdWOYmUjufQ9GBVPqZ+tu+jMtXxXNmRJyk=
+github.com/lucas-clemente/quic-go v0.27.1/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI=
 github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac=
 github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg=
@@ -1981,8 +1983,9 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2/go.mod h1:f
 github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20210713220151-be142a5ae1a8/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/cbor-gen v0.0.0-20220224212727-7a699437a831/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
-github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14 h1:vo2wkP2ceHyGyZwFFtAabpot03EeSxxwAe57pOI9E/4=
 github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
+github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799 h1:DOOT2B85S0tHoLGTzV+FakaSSihgRCVwZkjqKQP5L/w=
+github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
 github.com/whyrusleeping/go-ctrlnet v0.0.0-20180313164037-f564fbbdaa95/go.mod h1:SJqKCCPXRfBFCwXjfNT/skfsceF7+MBFLI2OrvuRA7g=

itests/worker_test.go

@@ -49,17 +49,23 @@ func TestWorkerDataCid(t *testing.T) {
 	e, err := worker.Enabled(ctx)
 	require.NoError(t, err)
 	require.True(t, e)
-	/*
-		pi, err := miner.ComputeDataCid(ctx, 1016, strings.NewReader(strings.Repeat("a", 1016)))
-		require.NoError(t, err)
-		require.Equal(t, abi.PaddedPieceSize(1024), pi.Size)
-		require.Equal(t, "baga6ea4seaqlhznlutptgfwhffupyer6txswamerq5fc2jlwf2lys2mm5jtiaeq", pi.PieceCID.String())
-	*/
+
+	pi, err := miner.ComputeDataCid(ctx, 1016, strings.NewReader(strings.Repeat("a", 1016)))
+	require.NoError(t, err)
+	require.Equal(t, abi.PaddedPieceSize(1024), pi.Size)
+	require.Equal(t, "baga6ea4seaqlhznlutptgfwhffupyer6txswamerq5fc2jlwf2lys2mm5jtiaeq", pi.PieceCID.String())
+
 	bigPiece := abi.PaddedPieceSize(16 << 20).Unpadded()
-	pi, err := miner.ComputeDataCid(ctx, bigPiece, strings.NewReader(strings.Repeat("a", int(bigPiece))))
+	pi, err = miner.ComputeDataCid(ctx, bigPiece, strings.NewReader(strings.Repeat("a", int(bigPiece))))
 	require.NoError(t, err)
 	require.Equal(t, bigPiece.Padded(), pi.Size)
 	require.Equal(t, "baga6ea4seaqmhoxl2ybw5m2wyd3pt3h4zmp7j52yumzu2rar26twns3uocq7yfa", pi.PieceCID.String())
+
+	nonFullPiece := abi.PaddedPieceSize(10 << 20).Unpadded()
+	pi, err = miner.ComputeDataCid(ctx, bigPiece, strings.NewReader(strings.Repeat("a", int(nonFullPiece))))
+	require.NoError(t, err)
+	require.Equal(t, bigPiece.Padded(), pi.Size)
+	require.Equal(t, "baga6ea4seaqbxib4pdxs5cqdn3fmtj4rcxk6rx6ztiqmrx7fcpo3ymuxbp2rodi", pi.PieceCID.String())
 }
 
 func TestWinningPostWorker(t *testing.T) {

lib/httpreader/httpreader.go

@@ -0,0 +1,47 @@
+package httpreader
+
+import (
+	"io"
+	"net/http"
+
+	"golang.org/x/xerrors"
+)
+
+// HttpReader is a reader which will read a http resource with a simple get request.
+// Before first Read it will be passed over JsonRPC as a URL.
+type HttpReader struct {
+	URL string
+
+	reader io.ReadCloser
+}
+
+func (h *HttpReader) Close() error {
+	h.URL = ""
+	if h.reader != nil {
+		return h.reader.Close()
+	}
+	return nil
+}
+
+func (h *HttpReader) Read(p []byte) (n int, err error) {
+	if h.reader == nil {
+		res, err := http.Get(h.URL)
+		if err != nil {
+			return 0, err
+		}
+		if res.StatusCode != http.StatusOK {
+			return 0, xerrors.Errorf("unexpected http status %d", res.StatusCode)
+		}
+
+		// mark the reader as reading
+		h.URL = ""
+		h.reader = res.Body
+	}
+	if h.reader == nil {
+		return 0, xerrors.Errorf("http reader closed")
+	}
+
+	return h.reader.Read(p)
+}
+
+var _ io.ReadCloser = &HttpReader{}

lib/rpcenc/reader.go

@@ -21,7 +21,9 @@ import (
 
 	"github.com/filecoin-project/go-jsonrpc"
 	"github.com/filecoin-project/go-state-types/abi"
-	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
+
+	"github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader"
+	"github.com/filecoin-project/lotus/lib/httpreader"
 )
 
 var log = logging.Logger("rpcenc")
@@ -33,6 +35,7 @@ type StreamType string
 const (
 	Null       StreamType = "null"
 	PushStream StreamType = "push"
+	HTTP       StreamType = "http"
 	// TODO: Data transfer handoff to workers?
 )
 
@@ -101,9 +104,12 @@ func ReaderParamEncoder(addr string) jsonrpc.Option {
 	return jsonrpc.WithParamEncoder(new(io.Reader), func(value reflect.Value) (reflect.Value, error) {
 		r := value.Interface().(io.Reader)
 
-		if r, ok := r.(*sealing.NullReader); ok {
+		if r, ok := r.(*nullreader.NullReader); ok {
 			return reflect.ValueOf(ReaderStream{Type: Null, Info: fmt.Sprint(r.N)}), nil
 		}
+		if r, ok := r.(*httpreader.HttpReader); ok && r.URL != "" {
+			return reflect.ValueOf(ReaderStream{Type: HTTP, Info: r.URL}), nil
+		}
 
 		reqID := uuid.New()
 		u, err := url.Parse(addr)
@@ -412,13 +418,16 @@ func ReaderParamDecoder() (http.HandlerFunc, jsonrpc.ServerOption) {
 			return reflect.Value{}, xerrors.Errorf("unmarshaling reader id: %w", err)
 		}
 
-		if rs.Type == Null {
+		switch rs.Type {
+		case Null:
 			n, err := strconv.ParseInt(rs.Info, 10, 64)
 			if err != nil {
 				return reflect.Value{}, xerrors.Errorf("parsing null byte count: %w", err)
 			}
 
-			return reflect.ValueOf(sealing.NewNullReader(abi.UnpaddedPieceSize(n))), nil
+			return reflect.ValueOf(nullreader.NewNullReader(abi.UnpaddedPieceSize(n))), nil
+		case HTTP:
+			return reflect.ValueOf(&httpreader.HttpReader{URL: rs.Info}), nil
 		}
 
 		u, err := uuid.Parse(rs.Info)

lib/rpcenc/reader_test.go

@@ -14,7 +14,8 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-jsonrpc"
-	sealing "github.com/filecoin-project/lotus/extern/storage-sealing"
+
+	"github.com/filecoin-project/lotus/extern/storage-sealing/lib/nullreader"
 )
 
 type ReaderHandler struct {
@@ -57,7 +58,7 @@ func (h *ReaderHandler) ReadAll(ctx context.Context, r io.Reader) ([]byte, error
 }
 
 func (h *ReaderHandler) ReadNullLen(ctx context.Context, r io.Reader) (int64, error) {
-	return r.(*sealing.NullReader).N, nil
+	return r.(*nullreader.NullReader).N, nil
 }
 
 func (h *ReaderHandler) ReadUrl(ctx context.Context, u string) (string, error) {
@@ -118,7 +119,7 @@ func TestNullReaderProxy(t *testing.T) {
 
 	defer closer()
 
-	n, err := client.ReadNullLen(context.TODO(), sealing.NewNullReader(1016))
+	n, err := client.ReadNullLen(context.TODO(), nullreader.NewNullReader(1016))
 	require.NoError(t, err)
 	require.Equal(t, int64(1016), n)
 }

node/config/doc_gen.go

@@ -897,13 +897,13 @@ This is useful for forcing all deals to be assigned as snap deals to sectors mar
 			Name: "MinCommitBatch",
 			Type: "int",
 
-			Comment: `maximum batched commit size - batches will be sent immediately above this size`,
+			Comment: `minimum batched commit size - batches above this size will eventually be sent on a timeout`,
 		},
 		{
 			Name: "MaxCommitBatch",
 			Type: "int",
 
-			Comment: ``,
+			Comment: `maximum batched commit size - batches will be sent immediately above this size`,
 		},
 		{
 			Name: "CommitBatchWait",

node/config/types.go

@@ -290,8 +290,9 @@ type SealingConfig struct {
 
 	// enable / disable commit aggregation (takes effect after nv13)
 	AggregateCommits bool
-	// maximum batched commit size - batches will be sent immediately above this size
+	// minimum batched commit size - batches above this size will eventually be sent on a timeout
 	MinCommitBatch int
+	// maximum batched commit size - batches will be sent immediately above this size
 	MaxCommitBatch int
 	// how long to wait before submitting a batch after crossing the minimum batch size
 	CommitBatchWait Duration

node/hello/cbor_gen.go

@@ -26,61 +26,67 @@ func (t *HelloMessage) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufHelloMessage); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufHelloMessage); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.HeaviestTipSet ([]cid.Cid) (slice)
 	if len(t.HeaviestTipSet) > cbg.MaxLength {
 		return xerrors.Errorf("Slice value in field t.HeaviestTipSet was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.HeaviestTipSet))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.HeaviestTipSet))); err != nil {
 		return err
 	}
 	for _, v := range t.HeaviestTipSet {
-		if err := cbg.WriteCidBuf(scratch, w, v); err != nil {
+		if err := cbg.WriteCid(w, v); err != nil {
 			return xerrors.Errorf("failed writing cid field t.HeaviestTipSet: %w", err)
 		}
 	}
 
 	// t.HeaviestTipSetHeight (abi.ChainEpoch) (int64)
 	if t.HeaviestTipSetHeight >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.HeaviestTipSetHeight)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.HeaviestTipSetHeight)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.HeaviestTipSetHeight-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.HeaviestTipSetHeight-1)); err != nil {
 			return err
 		}
 	}
 
 	// t.HeaviestTipSetWeight (big.Int) (struct)
-	if err := t.HeaviestTipSetWeight.MarshalCBOR(w); err != nil {
+	if err := t.HeaviestTipSetWeight.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
 	// t.GenesisHash (cid.Cid) (struct)
 
-	if err := cbg.WriteCidBuf(scratch, w, t.GenesisHash); err != nil {
+	if err := cbg.WriteCid(cw, t.GenesisHash); err != nil {
 		return xerrors.Errorf("failed to write cid field t.GenesisHash: %w", err)
 	}
 
 	return nil
 }
 
-func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error {
+func (t *HelloMessage) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = HelloMessage{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -91,7 +97,7 @@ func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error {
 
 	// t.HeaviestTipSet ([]cid.Cid) (slice)
 
-	maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err = cr.ReadHeader()
 	if err != nil {
 		return err
 	}
@@ -110,7 +116,7 @@ func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error {
 
 	for i := 0; i < int(extra); i++ {
 
-		c, err := cbg.ReadCid(br)
+		c, err := cbg.ReadCid(cr)
 		if err != nil {
 			return xerrors.Errorf("reading cid field t.HeaviestTipSet failed: %w", err)
 		}
@@ -119,7 +125,7 @@ func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error {
 
 	// t.HeaviestTipSetHeight (abi.ChainEpoch) (int64)
 	{
-		maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err := cr.ReadHeader()
 		var extraI int64
 		if err != nil {
 			return err
@@ -146,7 +152,7 @@ func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		if err := t.HeaviestTipSetWeight.UnmarshalCBOR(br); err != nil {
+		if err := t.HeaviestTipSetWeight.UnmarshalCBOR(cr); err != nil {
 			return xerrors.Errorf("unmarshaling t.HeaviestTipSetWeight: %w", err)
 		}
 
@@ -155,7 +161,7 @@ func (t *HelloMessage) UnmarshalCBOR(r io.Reader) error {
 
 	{
 
-		c, err := cbg.ReadCid(br)
+		c, err := cbg.ReadCid(cr)
 		if err != nil {
 			return xerrors.Errorf("failed to read cid field t.GenesisHash: %w", err)
 		}
@@ -173,46 +179,52 @@ func (t *LatencyMessage) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write(lengthBufLatencyMessage); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write(lengthBufLatencyMessage); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.TArrival (int64) (int64)
 	if t.TArrival >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.TArrival)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.TArrival)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.TArrival-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.TArrival-1)); err != nil {
 			return err
 		}
 	}
 
 	// t.TSent (int64) (int64)
 	if t.TSent >= 0 {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.TSent)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.TSent)); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.TSent-1)); err != nil {
+		if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.TSent-1)); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (t *LatencyMessage) UnmarshalCBOR(r io.Reader) error {
+func (t *LatencyMessage) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = LatencyMessage{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajArray {
 		return fmt.Errorf("cbor input should be of type array")
 	}
@@ -223,7 +235,7 @@ func (t *LatencyMessage) UnmarshalCBOR(r io.Reader) error {
 
 	// t.TArrival (int64) (int64)
 	{
-		maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err := cr.ReadHeader()
 		var extraI int64
 		if err != nil {
 			return err
@@ -248,7 +260,7 @@ func (t *LatencyMessage) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.TSent (int64) (int64)
 	{
-		maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+		maj, extra, err := cr.ReadHeader()
 		var extraI int64
 		if err != nil {
 			return err

node/impl/storminer.go

@@ -37,8 +37,10 @@ import (
 	"github.com/filecoin-project/go-fil-markets/piecestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	filmktsstore "github.com/filecoin-project/go-fil-markets/stores"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/network"
+	mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore"
 
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 	"github.com/filecoin-project/lotus/extern/sector-storage/fsutil"
@@ -83,6 +85,7 @@ type StorageMinerAPI struct {
 	SectorBlocks      *sectorblocks.SectorBlocks        `optional:"true"`
 	Host              host.Host                         `optional:"true"`
 	DAGStore          *dagstore.DAGStore                `optional:"true"`
+	DAGStoreWrapper   *mktsdagstore.Wrapper             `optional:"true"`
 
 	// Miner / storage
 	Miner       *storage.Miner              `optional:"true"`
@@ -792,6 +795,35 @@ func (sm *StorageMinerAPI) DagstoreListShards(ctx context.Context) ([]api.Dagsto
 	return ret, nil
 }
 
+func (sm *StorageMinerAPI) DagstoreRegisterShard(ctx context.Context, key string) error {
+	if sm.DAGStore == nil {
+		return fmt.Errorf("dagstore not available on this node")
+	}
+
+	// First check if the shard has already been registered
+	k := shard.KeyFromString(key)
+	_, err := sm.DAGStore.GetShardInfo(k)
+	if err == nil {
+		// Shard already registered, nothing further to do
+		return nil
+	}
+	// If the shard is not registered we would expect ErrShardUnknown
+	if !errors.Is(err, dagstore.ErrShardUnknown) {
+		return fmt.Errorf("getting shard info from DAG store: %w", err)
+	}
+
+	pieceCid, err := cid.Parse(key)
+	if err != nil {
+		return fmt.Errorf("parsing shard key as piece cid: %w", err)
+	}
+
+	if err = filmktsstore.RegisterShardSync(ctx, sm.DAGStoreWrapper, pieceCid, "", true); err != nil {
+		return fmt.Errorf("failed to register shard: %w", err)
+	}
+
+	return nil
+}
+
 func (sm *StorageMinerAPI) DagstoreInitializeShard(ctx context.Context, key string) error {
 	if sm.DAGStore == nil {
 		return fmt.Errorf("dagstore not available on this node")

paychmgr/cbor_gen.go

@@ -25,25 +25,26 @@ func (t *VoucherInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{163}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{163}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.Voucher (paych.SignedVoucher) (struct)
 	if len("Voucher") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Voucher\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Voucher"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Voucher"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Voucher")); err != nil {
 		return err
 	}
 
-	if err := t.Voucher.MarshalCBOR(w); err != nil {
+	if err := t.Voucher.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -52,7 +53,7 @@ func (t *VoucherInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Proof\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Proof"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Proof"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Proof")); err != nil {
@@ -63,11 +64,11 @@ func (t *VoucherInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Byte array in field t.Proof was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Proof))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Proof))); err != nil {
 		return err
 	}
 
-	if _, err := w.Write(t.Proof[:]); err != nil {
+	if _, err := cw.Write(t.Proof[:]); err != nil {
 		return err
 	}
 
@@ -76,7 +77,7 @@ func (t *VoucherInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Submitted\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Submitted"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Submitted"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Submitted")); err != nil {
@@ -89,16 +90,21 @@ func (t *VoucherInfo) MarshalCBOR(w io.Writer) error {
 	return nil
 }
 
-func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error {
+func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = VoucherInfo{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -113,7 +119,7 @@ func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -127,16 +133,16 @@ func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				b, err := br.ReadByte()
+				b, err := cr.ReadByte()
 				if err != nil {
 					return err
 				}
 				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
+					if err := cr.UnreadByte(); err != nil {
 						return err
 					}
 					t.Voucher = new(paych.SignedVoucher)
-					if err := t.Voucher.UnmarshalCBOR(br); err != nil {
+					if err := t.Voucher.UnmarshalCBOR(cr); err != nil {
 						return xerrors.Errorf("unmarshaling t.Voucher pointer: %w", err)
 					}
 				}
@@ -145,7 +151,7 @@ func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error {
 			// t.Proof ([]uint8) (slice)
 		case "Proof":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -161,13 +167,13 @@ func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error {
 				t.Proof = make([]uint8, extra)
 			}
 
-			if _, err := io.ReadFull(br, t.Proof[:]); err != nil {
+			if _, err := io.ReadFull(cr, t.Proof[:]); err != nil {
 				return err
 			}
 			// t.Submitted (bool) (bool)
 		case "Submitted":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -196,18 +202,19 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{174}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{174}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.ChannelID (string) (string)
 	if len("ChannelID") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"ChannelID\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ChannelID"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ChannelID"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("ChannelID")); err != nil {
@@ -218,7 +225,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.ChannelID was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.ChannelID))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.ChannelID))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.ChannelID)); err != nil {
@@ -230,14 +237,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Channel\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Channel"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Channel"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Channel")); err != nil {
 		return err
 	}
 
-	if err := t.Channel.MarshalCBOR(w); err != nil {
+	if err := t.Channel.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -246,14 +253,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Control\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Control"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Control"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Control")); err != nil {
 		return err
 	}
 
-	if err := t.Control.MarshalCBOR(w); err != nil {
+	if err := t.Control.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -262,14 +269,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Target\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Target"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Target"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Target")); err != nil {
 		return err
 	}
 
-	if err := t.Target.MarshalCBOR(w); err != nil {
+	if err := t.Target.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -278,14 +285,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Direction\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Direction"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Direction"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Direction")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Direction)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Direction)); err != nil {
 		return err
 	}
 
@@ -294,7 +301,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Vouchers\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Vouchers"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Vouchers")); err != nil {
@@ -305,11 +312,11 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Slice value in field t.Vouchers was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Vouchers))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Vouchers))); err != nil {
 		return err
 	}
 	for _, v := range t.Vouchers {
-		if err := v.MarshalCBOR(w); err != nil {
+		if err := v.MarshalCBOR(cw); err != nil {
 			return err
 		}
 	}
@@ -319,14 +326,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"NextLane\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("NextLane"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("NextLane"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("NextLane")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.NextLane)); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.NextLane)); err != nil {
 		return err
 	}
 
@@ -335,14 +342,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Amount\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Amount"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Amount"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Amount")); err != nil {
 		return err
 	}
 
-	if err := t.Amount.MarshalCBOR(w); err != nil {
+	if err := t.Amount.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -351,14 +358,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"AvailableAmount\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("AvailableAmount"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("AvailableAmount"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("AvailableAmount")); err != nil {
 		return err
 	}
 
-	if err := t.AvailableAmount.MarshalCBOR(w); err != nil {
+	if err := t.AvailableAmount.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -367,14 +374,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"PendingAvailableAmount\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PendingAvailableAmount"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PendingAvailableAmount"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("PendingAvailableAmount")); err != nil {
 		return err
 	}
 
-	if err := t.PendingAvailableAmount.MarshalCBOR(w); err != nil {
+	if err := t.PendingAvailableAmount.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -383,14 +390,14 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"PendingAmount\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("PendingAmount"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PendingAmount"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("PendingAmount")); err != nil {
 		return err
 	}
 
-	if err := t.PendingAmount.MarshalCBOR(w); err != nil {
+	if err := t.PendingAmount.MarshalCBOR(cw); err != nil {
 		return err
 	}
 
@@ -399,7 +406,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"CreateMsg\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("CreateMsg"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("CreateMsg"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("CreateMsg")); err != nil {
@@ -407,11 +414,11 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.CreateMsg == nil {
-		if _, err := w.Write(cbg.CborNull); err != nil {
+		if _, err := cw.Write(cbg.CborNull); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteCidBuf(scratch, w, *t.CreateMsg); err != nil {
+		if err := cbg.WriteCid(cw, *t.CreateMsg); err != nil {
 			return xerrors.Errorf("failed to write cid field t.CreateMsg: %w", err)
 		}
 	}
@@ -421,7 +428,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"AddFundsMsg\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("AddFundsMsg"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("AddFundsMsg"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("AddFundsMsg")); err != nil {
@@ -429,11 +436,11 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 	}
 
 	if t.AddFundsMsg == nil {
-		if _, err := w.Write(cbg.CborNull); err != nil {
+		if _, err := cw.Write(cbg.CborNull); err != nil {
 			return err
 		}
 	} else {
-		if err := cbg.WriteCidBuf(scratch, w, *t.AddFundsMsg); err != nil {
+		if err := cbg.WriteCid(cw, *t.AddFundsMsg); err != nil {
 			return xerrors.Errorf("failed to write cid field t.AddFundsMsg: %w", err)
 		}
 	}
@@ -443,7 +450,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Settling\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Settling"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Settling"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Settling")); err != nil {
@@ -456,16 +463,21 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 	return nil
 }
 
-func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
+func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = ChannelInfo{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -480,7 +492,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -493,7 +505,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 		case "ChannelID":
 
 			{
-				sval, err := cbg.ReadStringBuf(br, scratch)
+				sval, err := cbg.ReadString(cr)
 				if err != nil {
 					return err
 				}
@@ -505,16 +517,16 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				b, err := br.ReadByte()
+				b, err := cr.ReadByte()
 				if err != nil {
 					return err
 				}
 				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
+					if err := cr.UnreadByte(); err != nil {
 						return err
 					}
 					t.Channel = new(address.Address)
-					if err := t.Channel.UnmarshalCBOR(br); err != nil {
+					if err := t.Channel.UnmarshalCBOR(cr); err != nil {
 						return xerrors.Errorf("unmarshaling t.Channel pointer: %w", err)
 					}
 				}
@@ -525,7 +537,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.Control.UnmarshalCBOR(br); err != nil {
+				if err := t.Control.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.Control: %w", err)
 				}
 
@@ -535,7 +547,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.Target.UnmarshalCBOR(br); err != nil {
+				if err := t.Target.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.Target: %w", err)
 				}
 
@@ -545,7 +557,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err = cr.ReadHeader()
 				if err != nil {
 					return err
 				}
@@ -558,7 +570,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 			// t.Vouchers ([]*paychmgr.VoucherInfo) (slice)
 		case "Vouchers":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -578,7 +590,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 			for i := 0; i < int(extra); i++ {
 
 				var v VoucherInfo
-				if err := v.UnmarshalCBOR(br); err != nil {
+				if err := v.UnmarshalCBOR(cr); err != nil {
 					return err
 				}
 
@@ -590,7 +602,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+				maj, extra, err = cr.ReadHeader()
 				if err != nil {
 					return err
 				}
@@ -605,7 +617,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.Amount.UnmarshalCBOR(br); err != nil {
+				if err := t.Amount.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.Amount: %w", err)
 				}
 
@@ -615,7 +627,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.AvailableAmount.UnmarshalCBOR(br); err != nil {
+				if err := t.AvailableAmount.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.AvailableAmount: %w", err)
 				}
 
@@ -625,7 +637,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.PendingAvailableAmount.UnmarshalCBOR(br); err != nil {
+				if err := t.PendingAvailableAmount.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.PendingAvailableAmount: %w", err)
 				}
 
@@ -635,7 +647,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				if err := t.PendingAmount.UnmarshalCBOR(br); err != nil {
+				if err := t.PendingAmount.UnmarshalCBOR(cr); err != nil {
 					return xerrors.Errorf("unmarshaling t.PendingAmount: %w", err)
 				}
 
@@ -645,16 +657,16 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				b, err := br.ReadByte()
+				b, err := cr.ReadByte()
 				if err != nil {
 					return err
 				}
 				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
+					if err := cr.UnreadByte(); err != nil {
 						return err
 					}
 
-					c, err := cbg.ReadCid(br)
+					c, err := cbg.ReadCid(cr)
 					if err != nil {
 						return xerrors.Errorf("failed to read cid field t.CreateMsg: %w", err)
 					}
@@ -668,16 +680,16 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				b, err := br.ReadByte()
+				b, err := cr.ReadByte()
 				if err != nil {
 					return err
 				}
 				if b != cbg.CborNull[0] {
-					if err := br.UnreadByte(); err != nil {
+					if err := cr.UnreadByte(); err != nil {
 						return err
 					}
 
-					c, err := cbg.ReadCid(br)
+					c, err := cbg.ReadCid(cr)
 					if err != nil {
 						return xerrors.Errorf("failed to read cid field t.AddFundsMsg: %w", err)
 					}
@@ -689,7 +701,7 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 			// t.Settling (bool) (bool)
 		case "Settling":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -718,18 +730,19 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{164}); err != nil {
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{164}); err != nil {
 		return err
 	}
 
-	scratch := make([]byte, 9)
-
 	// t.ChannelID (string) (string)
 	if len("ChannelID") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"ChannelID\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("ChannelID"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ChannelID"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("ChannelID")); err != nil {
@@ -740,7 +753,7 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.ChannelID was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.ChannelID))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.ChannelID))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.ChannelID)); err != nil {
@@ -752,14 +765,14 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"MsgCid\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("MsgCid"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("MsgCid"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("MsgCid")); err != nil {
 		return err
 	}
 
-	if err := cbg.WriteCidBuf(scratch, w, t.MsgCid); err != nil {
+	if err := cbg.WriteCid(cw, t.MsgCid); err != nil {
 		return xerrors.Errorf("failed to write cid field t.MsgCid: %w", err)
 	}
 
@@ -768,7 +781,7 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Received\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Received"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Received"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Received")); err != nil {
@@ -784,7 +797,7 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field \"Err\" was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Err"))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Err"))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string("Err")); err != nil {
@@ -795,7 +808,7 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("Value in field t.Err was too long")
 	}
 
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len(t.Err))); err != nil {
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Err))); err != nil {
 		return err
 	}
 	if _, err := io.WriteString(w, string(t.Err)); err != nil {
@@ -804,16 +817,21 @@ func (t *MsgInfo) MarshalCBOR(w io.Writer) error {
 	return nil
 }
 
-func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error {
+func (t *MsgInfo) UnmarshalCBOR(r io.Reader) (err error) {
 	*t = MsgInfo{}
 
-	br := cbg.GetPeeker(r)
-	scratch := make([]byte, 8)
+	cr := cbg.NewCborReader(r)
 
-	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	maj, extra, err := cr.ReadHeader()
 	if err != nil {
 		return err
 	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
 	if maj != cbg.MajMap {
 		return fmt.Errorf("cbor input should be of type map")
 	}
@@ -828,7 +846,7 @@ func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error {
 	for i := uint64(0); i < n; i++ {
 
 		{
-			sval, err := cbg.ReadStringBuf(br, scratch)
+			sval, err := cbg.ReadString(cr)
 			if err != nil {
 				return err
 			}
@@ -841,7 +859,7 @@ func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error {
 		case "ChannelID":
 
 			{
-				sval, err := cbg.ReadStringBuf(br, scratch)
+				sval, err := cbg.ReadString(cr)
 				if err != nil {
 					return err
 				}
@@ -853,7 +871,7 @@ func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error {
 
 			{
 
-				c, err := cbg.ReadCid(br)
+				c, err := cbg.ReadCid(cr)
 				if err != nil {
 					return xerrors.Errorf("failed to read cid field t.MsgCid: %w", err)
 				}
@@ -864,7 +882,7 @@ func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error {
 			// t.Received (bool) (bool)
 		case "Received":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
+			maj, extra, err = cr.ReadHeader()
 			if err != nil {
 				return err
 			}
@@ -883,7 +901,7 @@ func (t *MsgInfo) UnmarshalCBOR(r io.Reader) error {
 		case "Err":
 
 			{
-				sval, err := cbg.ReadStringBuf(br, scratch)
+				sval, err := cbg.ReadString(cr)
 				if err != nil {
 					return err
 				}