sector import: Update type-gen

documentation/en/api-v0-methods-miner.md

@@ -3272,29 +3272,32 @@ Inputs:
     "DataUnsealed": {
       "Local": true,
       "URL": "string value",
-      "Headers": {
-        "Authorization": [
-          "Bearer ey.."
-        ]
+      "Headers": [
+        {
+          "Key": "string value",
+          "Value": "string value"
         }
+      ]
     },
     "DataSealed": {
       "Local": true,
       "URL": "string value",
-      "Headers": {
-        "Authorization": [
-          "Bearer ey.."
-        ]
+      "Headers": [
+        {
+          "Key": "string value",
+          "Value": "string value"
         }
+      ]
     },
     "DataCache": {
       "Local": true,
       "URL": "string value",
-      "Headers": {
-        "Authorization": [
-          "Bearer ey.."
-        ]
+      "Headers": [
+        {
+          "Key": "string value",
+          "Value": "string value"
         }
+      ]
     }
   }
 ]

gen/main.go

@@ -103,6 +103,8 @@ func main() {
 
 	err = gen.WriteMapEncodersToFile("./storage/sealer/storiface/cbor_gen.go", "storiface",
 		storiface.CallID{},
+		storiface.SecDataHttpHeader{},
+		storiface.SectorData{},
 	)
 	if err != nil {
 		fmt.Println(err)

storage/pipeline/cbor_gen.go

@@ -15,6 +15,7 @@ import (
 	abi "github.com/filecoin-project/go-state-types/abi"
 
 	api "github.com/filecoin-project/lotus/api"
+	storiface "github.com/filecoin-project/lotus/storage/sealer/storiface"
 )
 
 var _ = xerrors.Errorf
@@ -30,7 +31,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
 
 	cw := cbg.NewCborWriter(w)
 
-	if _, err := cw.Write([]byte{184, 31}); err != nil {
+	if _, err := cw.Write([]byte{184, 36}); err != nil {
 		return err
 	}
 
@@ -654,6 +655,70 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
+	// t.RemoteDataUnsealed (storiface.SectorData) (struct)
+	if len("RemoteDataUnsealed") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"RemoteDataUnsealed\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RemoteDataUnsealed"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("RemoteDataUnsealed")); err != nil {
+		return err
+	}
+
+	if err := t.RemoteDataUnsealed.MarshalCBOR(cw); err != nil {
+		return err
+	}
+
+	// t.RemoteDataSealed (storiface.SectorData) (struct)
+	if len("RemoteDataSealed") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"RemoteDataSealed\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RemoteDataSealed"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("RemoteDataSealed")); err != nil {
+		return err
+	}
+
+	if err := t.RemoteDataSealed.MarshalCBOR(cw); err != nil {
+		return err
+	}
+
+	// t.RemoteDataCache (storiface.SectorData) (struct)
+	if len("RemoteDataCache") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"RemoteDataCache\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RemoteDataCache"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("RemoteDataCache")); err != nil {
+		return err
+	}
+
+	if err := t.RemoteDataCache.MarshalCBOR(cw); err != nil {
+		return err
+	}
+
+	// t.RemoteDataFinalized (bool) (bool)
+	if len("RemoteDataFinalized") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"RemoteDataFinalized\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RemoteDataFinalized"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("RemoteDataFinalized")); err != nil {
+		return err
+	}
+
+	if err := cbg.WriteBool(w, t.RemoteDataFinalized); err != nil {
+		return err
+	}
+
 	// t.LastErr (string) (string)
 	if len("LastErr") > cbg.MaxLength {
 		return xerrors.Errorf("Value in field \"LastErr\" was too long")
@@ -1354,6 +1419,84 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) (err error) {
 
 				t.TerminatedAt = abi.ChainEpoch(extraI)
 			}
+			// t.RemoteDataUnsealed (storiface.SectorData) (struct)
+		case "RemoteDataUnsealed":
+
+			{
+
+				b, err := cr.ReadByte()
+				if err != nil {
+					return err
+				}
+				if b != cbg.CborNull[0] {
+					if err := cr.UnreadByte(); err != nil {
+						return err
+					}
+					t.RemoteDataUnsealed = new(storiface.SectorData)
+					if err := t.RemoteDataUnsealed.UnmarshalCBOR(cr); err != nil {
+						return xerrors.Errorf("unmarshaling t.RemoteDataUnsealed pointer: %w", err)
+					}
+				}
+
+			}
+			// t.RemoteDataSealed (storiface.SectorData) (struct)
+		case "RemoteDataSealed":
+
+			{
+
+				b, err := cr.ReadByte()
+				if err != nil {
+					return err
+				}
+				if b != cbg.CborNull[0] {
+					if err := cr.UnreadByte(); err != nil {
+						return err
+					}
+					t.RemoteDataSealed = new(storiface.SectorData)
+					if err := t.RemoteDataSealed.UnmarshalCBOR(cr); err != nil {
+						return xerrors.Errorf("unmarshaling t.RemoteDataSealed pointer: %w", err)
+					}
+				}
+
+			}
+			// t.RemoteDataCache (storiface.SectorData) (struct)
+		case "RemoteDataCache":
+
+			{
+
+				b, err := cr.ReadByte()
+				if err != nil {
+					return err
+				}
+				if b != cbg.CborNull[0] {
+					if err := cr.UnreadByte(); err != nil {
+						return err
+					}
+					t.RemoteDataCache = new(storiface.SectorData)
+					if err := t.RemoteDataCache.UnmarshalCBOR(cr); err != nil {
+						return xerrors.Errorf("unmarshaling t.RemoteDataCache pointer: %w", err)
+					}
+				}
+
+			}
+			// t.RemoteDataFinalized (bool) (bool)
+		case "RemoteDataFinalized":
+
+			maj, extra, err = cr.ReadHeader()
+			if err != nil {
+				return err
+			}
+			if maj != cbg.MajOther {
+				return fmt.Errorf("booleans must be major type 7")
+			}
+			switch extra {
+			case 20:
+				t.RemoteDataFinalized = false
+			case 21:
+				t.RemoteDataFinalized = true
+			default:
+				return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra)
+			}
 			// t.LastErr (string) (string)
 		case "LastErr":
 

storage/sealer/ffiwrapper/sealer_cgo.go

@@ -1152,7 +1152,7 @@ func (sb *Sealer) DownloadSectorData(ctx context.Context, sector storiface.Secto
 			return xerrors.Errorf("sector(%v) with local data (%#v) requested in DownloadSectorData", sector, data)
 		}
 
-		_, err := spaths.FetchWithTemp(ctx, []string{data.URL}, out, data.Headers)
+		_, err := spaths.FetchWithTemp(ctx, []string{data.URL}, out, data.HttpHeaders())
 		if err != nil {
 			return xerrors.Errorf("downloading sector data: %w", err)
 		}

storage/sealer/storiface/cbor_gen.go

@@ -153,3 +153,315 @@ func (t *CallID) UnmarshalCBOR(r io.Reader) (err error) {
 
 	return nil
 }
+func (t *SecDataHttpHeader) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{162}); err != nil {
+		return err
+	}
+
+	// t.Key (string) (string)
+	if len("Key") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Key\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Key"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("Key")); err != nil {
+		return err
+	}
+
+	if len(t.Key) > cbg.MaxLength {
+		return xerrors.Errorf("Value in field t.Key was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Key))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string(t.Key)); err != nil {
+		return err
+	}
+
+	// t.Value (string) (string)
+	if len("Value") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Value\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Value"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("Value")); err != nil {
+		return err
+	}
+
+	if len(t.Value) > cbg.MaxLength {
+		return xerrors.Errorf("Value in field t.Value was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Value))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string(t.Value)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *SecDataHttpHeader) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = SecDataHttpHeader{}
+
+	cr := cbg.NewCborReader(r)
+
+	maj, extra, err := cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
+	if maj != cbg.MajMap {
+		return fmt.Errorf("cbor input should be of type map")
+	}
+
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("SecDataHttpHeader: map struct too large (%d)", extra)
+	}
+
+	var name string
+	n := extra
+
+	for i := uint64(0); i < n; i++ {
+
+		{
+			sval, err := cbg.ReadString(cr)
+			if err != nil {
+				return err
+			}
+
+			name = string(sval)
+		}
+
+		switch name {
+		// t.Key (string) (string)
+		case "Key":
+
+			{
+				sval, err := cbg.ReadString(cr)
+				if err != nil {
+					return err
+				}
+
+				t.Key = string(sval)
+			}
+			// t.Value (string) (string)
+		case "Value":
+
+			{
+				sval, err := cbg.ReadString(cr)
+				if err != nil {
+					return err
+				}
+
+				t.Value = string(sval)
+			}
+
+		default:
+			// Field doesn't exist on this type, so ignore it
+			cbg.ScanForLinks(r, func(cid.Cid) {})
+		}
+	}
+
+	return nil
+}
+func (t *SectorData) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+
+	cw := cbg.NewCborWriter(w)
+
+	if _, err := cw.Write([]byte{163}); err != nil {
+		return err
+	}
+
+	// t.Local (bool) (bool)
+	if len("Local") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Local\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Local"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("Local")); err != nil {
+		return err
+	}
+
+	if err := cbg.WriteBool(w, t.Local); err != nil {
+		return err
+	}
+
+	// t.URL (string) (string)
+	if len("URL") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"URL\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("URL"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("URL")); err != nil {
+		return err
+	}
+
+	if len(t.URL) > cbg.MaxLength {
+		return xerrors.Errorf("Value in field t.URL was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.URL))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string(t.URL)); err != nil {
+		return err
+	}
+
+	// t.Headers ([]storiface.SecDataHttpHeader) (slice)
+	if len("Headers") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Headers\" was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Headers"))); err != nil {
+		return err
+	}
+	if _, err := io.WriteString(w, string("Headers")); err != nil {
+		return err
+	}
+
+	if len(t.Headers) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.Headers was too long")
+	}
+
+	if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Headers))); err != nil {
+		return err
+	}
+	for _, v := range t.Headers {
+		if err := v.MarshalCBOR(cw); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (t *SectorData) UnmarshalCBOR(r io.Reader) (err error) {
+	*t = SectorData{}
+
+	cr := cbg.NewCborReader(r)
+
+	maj, extra, err := cr.ReadHeader()
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+	}()
+
+	if maj != cbg.MajMap {
+		return fmt.Errorf("cbor input should be of type map")
+	}
+
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("SectorData: map struct too large (%d)", extra)
+	}
+
+	var name string
+	n := extra
+
+	for i := uint64(0); i < n; i++ {
+
+		{
+			sval, err := cbg.ReadString(cr)
+			if err != nil {
+				return err
+			}
+
+			name = string(sval)
+		}
+
+		switch name {
+		// t.Local (bool) (bool)
+		case "Local":
+
+			maj, extra, err = cr.ReadHeader()
+			if err != nil {
+				return err
+			}
+			if maj != cbg.MajOther {
+				return fmt.Errorf("booleans must be major type 7")
+			}
+			switch extra {
+			case 20:
+				t.Local = false
+			case 21:
+				t.Local = true
+			default:
+				return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra)
+			}
+			// t.URL (string) (string)
+		case "URL":
+
+			{
+				sval, err := cbg.ReadString(cr)
+				if err != nil {
+					return err
+				}
+
+				t.URL = string(sval)
+			}
+			// t.Headers ([]storiface.SecDataHttpHeader) (slice)
+		case "Headers":
+
+			maj, extra, err = cr.ReadHeader()
+			if err != nil {
+				return err
+			}
+
+			if extra > cbg.MaxLength {
+				return fmt.Errorf("t.Headers: array too large (%d)", extra)
+			}
+
+			if maj != cbg.MajArray {
+				return fmt.Errorf("expected cbor array")
+			}
+
+			if extra > 0 {
+				t.Headers = make([]SecDataHttpHeader, extra)
+			}
+
+			for i := 0; i < int(extra); i++ {
+
+				var v SecDataHttpHeader
+				if err := v.UnmarshalCBOR(cr); err != nil {
+					return err
+				}
+
+				t.Headers[i] = v
+			}
+
+		default:
+			// Field doesn't exist on this type, so ignore it
+			cbg.ScanForLinks(r, func(cid.Cid) {})
+		}
+	}
+
+	return nil
+}

storage/sealer/storiface/storage.go

@@ -137,5 +137,19 @@ type SectorData struct {
 	URL string
 
 	// optional http headers to use when requesting sector data
-	Headers http.Header
+	Headers []SecDataHttpHeader
+}
+
+func (sd *SectorData) HttpHeaders() http.Header {
+	out := http.Header{}
+	for _, header := range sd.Headers {
+		out[header.Key] = append(out[header.Key], header.Value)
+	}
+	return out
+}
+
+// note: we can't use http.Header as that's backed by a go map, which is all kinds of messy
+type SecDataHttpHeader struct {
+	Key   string
+	Value string
 }