sectorstorage: calltracker: work around cbor-gen bytearray len limit

extern/sector-storage/cbor_gen.go

@@ -78,7 +78,7 @@ func (t *Call) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.Result ([]uint8) (slice)
+	// t.Result (sectorstorage.ManyBytes) (struct)
 	if len("Result") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"Result\" was too long")
 	}
@@ -90,15 +90,7 @@ func (t *Call) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	if len(t.Result) > cbg.ByteArrayMaxLen {
-		return xerrors.Errorf("Byte array in field t.Result was too long")
-	}
-
-	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Result))); err != nil {
-		return err
-	}
-
-	if _, err := w.Write(t.Result[:]); err != nil {
+	if err := t.Result.MarshalCBOR(w); err != nil {
 		return err
 	}
 	return nil
@@ -173,27 +165,25 @@ func (t *Call) UnmarshalCBOR(r io.Reader) error {
 				t.State = CallState(extra)
 
 			}
-			// t.Result ([]uint8) (slice)
+			// t.Result (sectorstorage.ManyBytes) (struct)
 		case "Result":
 
-			maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
-			if err != nil {
-				return err
-			}
+			{
 
-			if extra > cbg.ByteArrayMaxLen {
-				return fmt.Errorf("t.Result: byte array too large (%d)", extra)
-			}
-			if maj != cbg.MajByteString {
-				return fmt.Errorf("expected byte array")
-			}
+				b, err := br.ReadByte()
+				if err != nil {
+					return err
+				}
+				if b != cbg.CborNull[0] {
+					if err := br.UnreadByte(); err != nil {
+						return err
+					}
+					t.Result = new(ManyBytes)
+					if err := t.Result.UnmarshalCBOR(br); err != nil {
+						return xerrors.Errorf("unmarshaling t.Result pointer: %w", err)
+					}
+				}
 
-			if extra > 0 {
-				t.Result = make([]uint8, extra)
-			}
-
-			if _, err := io.ReadFull(br, t.Result[:]); err != nil {
-				return err
 			}
 
 		default:

extern/sector-storage/worker_calltracker.go

@@ -1,7 +1,12 @@
 package sectorstorage
 
 import (
+	"fmt"
+	"io"
+
 	"github.com/filecoin-project/go-statestore"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
 )
@@ -24,7 +29,7 @@ type Call struct {
 
 	State CallState
 
-	Result []byte // json bytes
+	Result *ManyBytes // json bytes
 }
 
 func (wt *workerCallTracker) onStart(ci storiface.CallID, rt ReturnType) error {
@@ -39,7 +44,7 @@ func (wt *workerCallTracker) onDone(ci storiface.CallID, ret []byte) error {
 	st := wt.st.Get(ci)
 	return st.Mutate(func(cs *Call) error {
 		cs.State = CallDone
-		cs.Result = ret
+		cs.Result = &ManyBytes{ret}
 		return nil
 	})
 }
@@ -53,3 +58,60 @@ func (wt *workerCallTracker) unfinished() ([]Call, error) {
 	var out []Call
 	return out, wt.st.List(&out)
 }
+
+// Ideally this would be a tag on the struct field telling cbor-gen to enforce higher max-len
+type ManyBytes struct {
+	b []byte
+}
+
+const many = 100 << 20
+
+func (t *ManyBytes) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		t = &ManyBytes{}
+	}
+
+	if len(t.b) > many {
+		return xerrors.Errorf("byte array in field t.Result was too long")
+	}
+
+	scratch := make([]byte, 9)
+
+	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.b))); err != nil {
+		return err
+	}
+
+	if _, err := w.Write(t.b[:]); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *ManyBytes) UnmarshalCBOR(r io.Reader) error {
+	*t = ManyBytes{}
+
+	br := cbg.GetPeeker(r)
+	scratch := make([]byte, 9)
+
+	maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
+	if err != nil {
+		return err
+	}
+
+	if extra > many {
+		return fmt.Errorf("byte array too large (%d)", extra)
+	}
+	if maj != cbg.MajByteString {
+		return fmt.Errorf("expected byte array")
+	}
+
+	if extra > 0 {
+		t.b = make([]uint8, extra)
+	}
+
+	if _, err := io.ReadFull(br, t.b[:]); err != nil {
+		return err
+	}
+
+	return nil
+}