diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index 30cd20191..550389d52 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 76d71cb4c..f60f1fc3a 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index 4034c4407..e28ce4846 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -3272,29 +3272,32 @@ Inputs: "DataUnsealed": { "Local": true, "URL": "string value", - "Headers": { - "Authorization": [ - "Bearer ey.." - ] - } + "Headers": [ + { + "Key": "string value", + "Value": "string value" + } + ] }, "DataSealed": { "Local": true, "URL": "string value", - "Headers": { - "Authorization": [ - "Bearer ey.." - ] - } + "Headers": [ + { + "Key": "string value", + "Value": "string value" + } + ] }, "DataCache": { "Local": true, "URL": "string value", - "Headers": { - "Authorization": [ - "Bearer ey.." - ] - } + "Headers": [ + { + "Key": "string value", + "Value": "string value" + } + ] } } ] diff --git a/gen/main.go b/gen/main.go index 77f340813..b7aaa0a9a 100644 --- a/gen/main.go +++ b/gen/main.go @@ -103,6 +103,8 @@ func main() { err = gen.WriteMapEncodersToFile("./storage/sealer/storiface/cbor_gen.go", "storiface", storiface.CallID{}, + storiface.SecDataHttpHeader{}, + storiface.SectorData{}, ) if err != nil { fmt.Println(err) diff --git a/storage/pipeline/cbor_gen.go b/storage/pipeline/cbor_gen.go index 0d9d1a918..a86e5f768 100644 --- a/storage/pipeline/cbor_gen.go +++ b/storage/pipeline/cbor_gen.go @@ -15,6 +15,7 @@ import ( abi "github.com/filecoin-project/go-state-types/abi" api "github.com/filecoin-project/lotus/api" + storiface "github.com/filecoin-project/lotus/storage/sealer/storiface" ) var _ = xerrors.Errorf @@ -30,7 +31,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{184, 31}); err != nil { + if _, err := cw.Write([]byte{184, 36}); err != nil { return err } @@ -654,6 +655,70 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } } + // t.RemoteDataUnsealed (storiface.SectorData) (struct) + if len("RemoteDataUnsealed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"RemoteDataUnsealed\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RemoteDataUnsealed"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("RemoteDataUnsealed")); err != nil { + return err + } + + if err := t.RemoteDataUnsealed.MarshalCBOR(cw); err != nil { + return err + } + + // t.RemoteDataSealed (storiface.SectorData) (struct) + if len("RemoteDataSealed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"RemoteDataSealed\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RemoteDataSealed"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("RemoteDataSealed")); err != nil { + return err + } + + if err := t.RemoteDataSealed.MarshalCBOR(cw); err != nil { + return err + } + + // t.RemoteDataCache (storiface.SectorData) (struct) + if len("RemoteDataCache") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"RemoteDataCache\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RemoteDataCache"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("RemoteDataCache")); err != nil { + return err + } + + if err := t.RemoteDataCache.MarshalCBOR(cw); err != nil { + return err + } + + // t.RemoteDataFinalized (bool) (bool) + if len("RemoteDataFinalized") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"RemoteDataFinalized\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RemoteDataFinalized"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("RemoteDataFinalized")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.RemoteDataFinalized); err != nil { + return err + } + // t.LastErr (string) (string) if len("LastErr") > cbg.MaxLength { return xerrors.Errorf("Value in field \"LastErr\" was too long") @@ -1354,6 +1419,84 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) (err error) { t.TerminatedAt = abi.ChainEpoch(extraI) } + // t.RemoteDataUnsealed (storiface.SectorData) (struct) + case "RemoteDataUnsealed": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.RemoteDataUnsealed = new(storiface.SectorData) + if err := t.RemoteDataUnsealed.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.RemoteDataUnsealed pointer: %w", err) + } + } + + } + // t.RemoteDataSealed (storiface.SectorData) (struct) + case "RemoteDataSealed": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.RemoteDataSealed = new(storiface.SectorData) + if err := t.RemoteDataSealed.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.RemoteDataSealed pointer: %w", err) + } + } + + } + // t.RemoteDataCache (storiface.SectorData) (struct) + case "RemoteDataCache": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.RemoteDataCache = new(storiface.SectorData) + if err := t.RemoteDataCache.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.RemoteDataCache pointer: %w", err) + } + } + + } + // t.RemoteDataFinalized (bool) (bool) + case "RemoteDataFinalized": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.RemoteDataFinalized = false + case 21: + t.RemoteDataFinalized = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } // t.LastErr (string) (string) case "LastErr": diff --git a/storage/sealer/ffiwrapper/sealer_cgo.go b/storage/sealer/ffiwrapper/sealer_cgo.go index 0ed53183c..cc137151f 100644 --- a/storage/sealer/ffiwrapper/sealer_cgo.go +++ b/storage/sealer/ffiwrapper/sealer_cgo.go @@ -1152,7 +1152,7 @@ func (sb *Sealer) DownloadSectorData(ctx context.Context, sector storiface.Secto return xerrors.Errorf("sector(%v) with local data (%#v) requested in DownloadSectorData", sector, data) } - _, err := spaths.FetchWithTemp(ctx, []string{data.URL}, out, data.Headers) + _, err := spaths.FetchWithTemp(ctx, []string{data.URL}, out, data.HttpHeaders()) if err != nil { return xerrors.Errorf("downloading sector data: %w", err) } diff --git a/storage/sealer/storiface/cbor_gen.go b/storage/sealer/storiface/cbor_gen.go index 5b4623175..6fec11557 100644 --- a/storage/sealer/storiface/cbor_gen.go +++ b/storage/sealer/storiface/cbor_gen.go @@ -153,3 +153,315 @@ func (t *CallID) UnmarshalCBOR(r io.Reader) (err error) { return nil } +func (t *SecDataHttpHeader) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Key (string) (string) + if len("Key") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Key\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Key"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Key")); err != nil { + return err + } + + if len(t.Key) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Key was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Key))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Key)); err != nil { + return err + } + + // t.Value (string) (string) + if len("Value") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Value\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Value"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Value")); err != nil { + return err + } + + if len(t.Value) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Value was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Value))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Value)); err != nil { + return err + } + return nil +} + +func (t *SecDataHttpHeader) UnmarshalCBOR(r io.Reader) (err error) { + *t = SecDataHttpHeader{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SecDataHttpHeader: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Key (string) (string) + case "Key": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.Key = string(sval) + } + // t.Value (string) (string) + case "Value": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.Value = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *SectorData) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Local (bool) (bool) + if len("Local") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Local\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Local"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Local")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.Local); err != nil { + return err + } + + // t.URL (string) (string) + if len("URL") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"URL\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("URL"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("URL")); err != nil { + return err + } + + if len(t.URL) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.URL was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.URL))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.URL)); err != nil { + return err + } + + // t.Headers ([]storiface.SecDataHttpHeader) (slice) + if len("Headers") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"Headers\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Headers"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("Headers")); err != nil { + return err + } + + if len(t.Headers) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Headers was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Headers))); err != nil { + return err + } + for _, v := range t.Headers { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + } + return nil +} + +func (t *SectorData) UnmarshalCBOR(r io.Reader) (err error) { + *t = SectorData{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SectorData: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Local (bool) (bool) + case "Local": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.Local = false + case 21: + t.Local = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + // t.URL (string) (string) + case "URL": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.URL = string(sval) + } + // t.Headers ([]storiface.SecDataHttpHeader) (slice) + case "Headers": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Headers: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Headers = make([]SecDataHttpHeader, extra) + } + + for i := 0; i < int(extra); i++ { + + var v SecDataHttpHeader + if err := v.UnmarshalCBOR(cr); err != nil { + return err + } + + t.Headers[i] = v + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/storage/sealer/storiface/storage.go b/storage/sealer/storiface/storage.go index 4248cd71e..975011a0b 100644 --- a/storage/sealer/storiface/storage.go +++ b/storage/sealer/storiface/storage.go @@ -137,5 +137,19 @@ type SectorData struct { URL string // optional http headers to use when requesting sector data - Headers http.Header + Headers []SecDataHttpHeader +} + +func (sd *SectorData) HttpHeaders() http.Header { + out := http.Header{} + for _, header := range sd.Headers { + out[header.Key] = append(out[header.Key], header.Value) + } + return out +} + +// note: we can't use http.Header as that's backed by a go map, which is all kinds of messy +type SecDataHttpHeader struct { + Key string + Value string }