From 2076cb106d3d896d8b6278a057f5f46aa23d69d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 21 Jan 2020 17:05:10 +0100 Subject: [PATCH 1/2] cobr-gen: Soft struct-map unmarshaling --- api/cbor_gen.go | 338 ++++++------- api/test/deals.go | 1 - gen/main.go | 2 +- go.mod | 2 +- go.sum | 4 +- lib/statemachine/cbor_gen.go | 147 +++--- node/node_test.go | 6 + storage/sealing/cbor_gen.go | 885 ++++++++++++++--------------------- storage/sealing/fsm.go | 4 + 9 files changed, 590 insertions(+), 799 deletions(-) diff --git a/api/cbor_gen.go b/api/cbor_gen.go index a65f97aaf..3b1bea712 100644 --- a/api/cbor_gen.go +++ b/api/cbor_gen.go @@ -98,114 +98,94 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type map") } - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("PaymentInfo: map struct too large (%d)", extra) } var name string + n := extra - // t.Channel (address.Address) (struct) + for i := uint64(0); i < n; i++ { - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "Channel" { - return fmt.Errorf("expected struct map entry %s to be Channel", name) - } - - { - - if err := t.Channel.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.ChannelMessage (cid.Cid) (struct) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "ChannelMessage" { - return fmt.Errorf("expected struct map entry %s to be ChannelMessage", name) - } - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { + { + sval, err := cbg.ReadString(br) + if err != nil { return err } - } else { - c, err := cbg.ReadCid(br) + name = string(sval) + } + + switch name { + // t.Channel (address.Address) (struct) + case "Channel": + + { + + if err := t.Channel.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.ChannelMessage (cid.Cid) (struct) + case "ChannelMessage": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ChannelMessage: %w", err) + } + + t.ChannelMessage = &c + } + + } + // t.Vouchers ([]*types.SignedVoucher) (slice) + case "Vouchers": + + maj, extra, err = cbg.CborReadHeader(br) if err != nil { - return xerrors.Errorf("failed to read cid field t.ChannelMessage: %w", err) + return err } - t.ChannelMessage = &c + if extra > cbg.MaxLength { + return fmt.Errorf("t.Vouchers: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Vouchers = make([]*types.SignedVoucher, extra) + } + for i := 0; i < int(extra); i++ { + + var v types.SignedVoucher + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Vouchers[i] = &v + } + + default: } - - } - // t.Vouchers ([]*types.SignedVoucher) (slice) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "Vouchers" { - return fmt.Errorf("expected struct map entry %s to be Vouchers", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.MaxLength { - return fmt.Errorf("t.Vouchers: array too large (%d)", extra) - } - - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } - if extra > 0 { - t.Vouchers = make([]*types.SignedVoucher, extra) - } - for i := 0; i < int(extra); i++ { - - var v types.SignedVoucher - if err := v.UnmarshalCBOR(br); err != nil { - return err - } - - t.Vouchers[i] = &v } return nil } - func (t *SealedRef) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -276,84 +256,65 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type map") } - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("SealedRef: map struct too large (%d)", extra) } var name string + n := extra - // t.SectorID (uint64) (uint64) + for i := uint64(0); i < n; i++ { - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) } - name = string(sval) - } + switch name { + // t.SectorID (uint64) (uint64) + case "SectorID": - if name != "SectorID" { - return fmt.Errorf("expected struct map entry %s to be SectorID", name) - } + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = uint64(extra) + // t.Offset (uint64) (uint64) + case "Offset": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = uint64(extra) - // t.Offset (uint64) (uint64) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Offset = uint64(extra) + // t.Size (uint64) (uint64) + case "Size": - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = uint64(extra) + + default: } - - name = string(sval) } - if name != "Offset" { - return fmt.Errorf("expected struct map entry %s to be Offset", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Offset = uint64(extra) - // t.Size (uint64) (uint64) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "Size" { - return fmt.Errorf("expected struct map entry %s to be Size", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Size = uint64(extra) return nil } - func (t *SealedRefs) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -401,50 +362,55 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type map") } - if extra != 1 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("SealedRefs: map struct too large (%d)", extra) } var name string + n := extra - // t.Refs ([]api.SealedRef) (slice) + for i := uint64(0); i < n; i++ { - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) } - name = string(sval) - } + switch name { + // t.Refs ([]api.SealedRef) (slice) + case "Refs": - if name != "Refs" { - return fmt.Errorf("expected struct map entry %s to be Refs", name) - } + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } + if extra > cbg.MaxLength { + return fmt.Errorf("t.Refs: array too large (%d)", extra) + } - if extra > cbg.MaxLength { - return fmt.Errorf("t.Refs: array too large (%d)", extra) - } + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Refs = make([]SealedRef, extra) + } + for i := 0; i < int(extra); i++ { - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } - if extra > 0 { - t.Refs = make([]SealedRef, extra) - } - for i := 0; i < int(extra); i++ { + var v SealedRef + if err := v.UnmarshalCBOR(br); err != nil { + return err + } - var v SealedRef - if err := v.UnmarshalCBOR(br); err != nil { - return err + t.Refs[i] = v + } + + default: } - - t.Refs[i] = v } return nil diff --git a/api/test/deals.go b/api/test/deals.go index 4c2fb7439..203acb7d6 100644 --- a/api/test/deals.go +++ b/api/test/deals.go @@ -65,7 +65,6 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) { defer close(done) for mine { time.Sleep(blocktime) - fmt.Println("mining a block now") if err := sn[0].MineOne(ctx); err != nil { t.Error(err) } diff --git a/gen/main.go b/gen/main.go index 7c5a0e1e4..c0698ca47 100644 --- a/gen/main.go +++ b/gen/main.go @@ -122,7 +122,7 @@ func main() { os.Exit(1) } - err = gen.WriteMapEncodersToFile("./storage/sectors/cbor_gen.go", "sectors", + err = gen.WriteMapEncodersToFile("./storage/sealing/cbor_gen.go", "sealing", sealing.SealTicket{}, sealing.SealSeed{}, sealing.Piece{}, diff --git a/go.mod b/go.mod index bb60aa910..cc9f1d9c1 100644 --- a/go.mod +++ b/go.mod @@ -86,7 +86,7 @@ require ( github.com/prometheus/common v0.2.0 github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe + github.com/whyrusleeping/cbor-gen v0.0.0-20200121160201-1834514787d2 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d go.opencensus.io v0.22.2 diff --git a/go.sum b/go.sum index 72721c8a8..1d36c637c 100644 --- a/go.sum +++ b/go.sum @@ -711,8 +711,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:x github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= -github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe h1:n7En1uyDtknjLRDXebWlPGJoHvwL8AkNcSQzuOoYYYQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20200121160201-1834514787d2 h1:VwJlCmEiu+/sl5KUq5w4i5VrhTIAhI++GVHGWf+vGnc= +github.com/whyrusleeping/cbor-gen v0.0.0-20200121160201-1834514787d2/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= diff --git a/lib/statemachine/cbor_gen.go b/lib/statemachine/cbor_gen.go index bf3830df7..e32e81697 100644 --- a/lib/statemachine/cbor_gen.go +++ b/lib/statemachine/cbor_gen.go @@ -66,61 +66,54 @@ func (t *TestState) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type map") } - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("TestState: map struct too large (%d)", extra) } var name string + n := extra - // t.A (uint64) (uint64) + for i := uint64(0); i < n; i++ { - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) } - name = string(sval) - } + switch name { + // t.A (uint64) (uint64) + case "A": - if name != "A" { - return fmt.Errorf("expected struct map entry %s to be A", name) - } + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.A = uint64(extra) + // t.B (uint64) (uint64) + case "B": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.A = uint64(extra) - // t.B (uint64) (uint64) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.B = uint64(extra) - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + default: } - - name = string(sval) } - if name != "B" { - return fmt.Errorf("expected struct map entry %s to be B", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.B = uint64(extra) return nil } - func (t *TestEvent) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -182,57 +175,51 @@ func (t *TestEvent) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type map") } - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("TestEvent: map struct too large (%d)", extra) } var name string + n := extra - // t.A (string) (string) + for i := uint64(0); i < n; i++ { - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) } - name = string(sval) - } + switch name { + // t.A (string) (string) + case "A": - if name != "A" { - return fmt.Errorf("expected struct map entry %s to be A", name) - } + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + t.A = string(sval) + } + // t.Val (uint64) (uint64) + case "Val": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Val = uint64(extra) + + default: } - - t.A = string(sval) - } - // t.Val (uint64) (uint64) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) } - if name != "Val" { - return fmt.Errorf("expected struct map entry %s to be Val", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Val = uint64(extra) return nil } diff --git a/node/node_test.go b/node/node_test.go index 078f0ba14..b9c1f778e 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -393,6 +393,12 @@ func TestAPIRPC(t *testing.T) { } func TestAPIDealFlow(t *testing.T) { + logging.SetLogLevel("miner", "ERROR") + logging.SetLogLevel("chainstore", "ERROR") + logging.SetLogLevel("chain", "ERROR") + logging.SetLogLevel("sub", "ERROR") + logging.SetLogLevel("storageminer", "ERROR") + test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond) } diff --git a/storage/sealing/cbor_gen.go b/storage/sealing/cbor_gen.go index 62c6241dc..61ad5b606 100644 --- a/storage/sealing/cbor_gen.go +++ b/storage/sealing/cbor_gen.go @@ -73,68 +73,61 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type map") } - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("SealTicket: map struct too large (%d)", extra) } var name string + n := extra - // t.BlockHeight (uint64) (uint64) + for i := uint64(0); i < n; i++ { - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) } - name = string(sval) - } + switch name { + // t.BlockHeight (uint64) (uint64) + case "BlockHeight": - if name != "BlockHeight" { - return fmt.Errorf("expected struct map entry %s to be BlockHeight", name) - } + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.BlockHeight = uint64(extra) + // t.TicketBytes ([]uint8) (slice) + case "TicketBytes": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.BlockHeight = uint64(extra) - // t.TicketBytes ([]uint8) (slice) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.TicketBytes: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.TicketBytes = make([]byte, extra) + if _, err := io.ReadFull(br, t.TicketBytes); err != nil { + return err + } + + default: } - - name = string(sval) } - if name != "TicketBytes" { - return fmt.Errorf("expected struct map entry %s to be TicketBytes", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.TicketBytes: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.TicketBytes = make([]byte, extra) - if _, err := io.ReadFull(br, t.TicketBytes); err != nil { - return err - } return nil } - func (t *SealSeed) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -196,68 +189,61 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type map") } - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("SealSeed: map struct too large (%d)", extra) } var name string + n := extra - // t.BlockHeight (uint64) (uint64) + for i := uint64(0); i < n; i++ { - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) } - name = string(sval) - } + switch name { + // t.BlockHeight (uint64) (uint64) + case "BlockHeight": - if name != "BlockHeight" { - return fmt.Errorf("expected struct map entry %s to be BlockHeight", name) - } + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.BlockHeight = uint64(extra) + // t.TicketBytes ([]uint8) (slice) + case "TicketBytes": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.BlockHeight = uint64(extra) - // t.TicketBytes ([]uint8) (slice) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.TicketBytes: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.TicketBytes = make([]byte, extra) + if _, err := io.ReadFull(br, t.TicketBytes); err != nil { + return err + } + + default: } - - name = string(sval) } - if name != "TicketBytes" { - return fmt.Errorf("expected struct map entry %s to be TicketBytes", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.TicketBytes: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.TicketBytes = make([]byte, extra) - if _, err := io.ReadFull(br, t.TicketBytes); err != nil { - return err - } return nil } - func (t *Piece) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -335,91 +321,72 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type map") } - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("Piece: map struct too large (%d)", extra) } var name string + n := extra - // t.DealID (uint64) (uint64) + for i := uint64(0); i < n; i++ { - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) } - name = string(sval) - } + switch name { + // t.DealID (uint64) (uint64) + case "DealID": - if name != "DealID" { - return fmt.Errorf("expected struct map entry %s to be DealID", name) - } + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = uint64(extra) + // t.Size (uint64) (uint64) + case "Size": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = uint64(extra) - // t.Size (uint64) (uint64) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = uint64(extra) + // t.CommP ([]uint8) (slice) + case "CommP": - { - sval, err := cbg.ReadString(br) - if err != nil { - return err + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.CommP: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommP = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommP); err != nil { + return err + } + + default: } - - name = string(sval) } - if name != "Size" { - return fmt.Errorf("expected struct map entry %s to be Size", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Size = uint64(extra) - // t.CommP ([]uint8) (slice) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "CommP" { - return fmt.Errorf("expected struct map entry %s to be CommP", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.CommP: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.CommP = make([]byte, extra) - if _, err := io.ReadFull(br, t.CommP); err != nil { - return err - } return nil } - func (t *SectorInfo) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -477,7 +444,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } - // t.Pieces ([]storage.Piece) (slice) + // t.Pieces ([]sealing.Piece) (slice) if len("Pieces") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Pieces\" was too long") } @@ -571,7 +538,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { return err } - // t.Ticket (storage.SealTicket) (struct) + // t.Ticket (sealing.SealTicket) (struct) if len("Ticket") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Ticket\" was too long") } @@ -609,7 +576,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } } - // t.Seed (storage.SealSeed) (struct) + // t.Seed (sealing.SealSeed) (struct) if len("Seed") > cbg.MaxLength { return xerrors.Errorf("Value in field \"Seed\" was too long") } @@ -705,388 +672,250 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type map") } - if extra != 13 { - return fmt.Errorf("cbor input had wrong number of fields") + if extra > cbg.MaxLength { + return fmt.Errorf("SectorInfo: map struct too large (%d)", extra) } var name string + n := extra - // t.State (uint64) (uint64) + for i := uint64(0); i < n; i++ { - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "State" { - return fmt.Errorf("expected struct map entry %s to be State", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) - // t.SectorID (uint64) (uint64) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "SectorID" { - return fmt.Errorf("expected struct map entry %s to be SectorID", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = uint64(extra) - // t.Nonce (uint64) (uint64) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "Nonce" { - return fmt.Errorf("expected struct map entry %s to be Nonce", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Nonce = uint64(extra) - // t.Pieces ([]storage.Piece) (slice) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "Pieces" { - return fmt.Errorf("expected struct map entry %s to be Pieces", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.MaxLength { - return fmt.Errorf("t.Pieces: array too large (%d)", extra) - } - - if maj != cbg.MajArray { - return fmt.Errorf("expected cbor array") - } - if extra > 0 { - t.Pieces = make([]Piece, extra) - } - for i := 0; i < int(extra); i++ { - - var v Piece - if err := v.UnmarshalCBOR(br); err != nil { - return err - } - - t.Pieces[i] = v - } - - // t.CommD ([]uint8) (slice) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "CommD" { - return fmt.Errorf("expected struct map entry %s to be CommD", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.CommD: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.CommD = make([]byte, extra) - if _, err := io.ReadFull(br, t.CommD); err != nil { - return err - } - // t.CommR ([]uint8) (slice) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "CommR" { - return fmt.Errorf("expected struct map entry %s to be CommR", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.CommR: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.CommR = make([]byte, extra) - if _, err := io.ReadFull(br, t.CommR); err != nil { - return err - } - // t.Proof ([]uint8) (slice) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "Proof" { - return fmt.Errorf("expected struct map entry %s to be Proof", name) - } - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Proof: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Proof = make([]byte, extra) - if _, err := io.ReadFull(br, t.Proof); err != nil { - return err - } - // t.Ticket (storage.SealTicket) (struct) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "Ticket" { - return fmt.Errorf("expected struct map entry %s to be Ticket", name) - } - - { - - if err := t.Ticket.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.PreCommitMessage (cid.Cid) (struct) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "PreCommitMessage" { - return fmt.Errorf("expected struct map entry %s to be PreCommitMessage", name) - } - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { + { + sval, err := cbg.ReadString(br) + if err != nil { return err } - } else { - c, err := cbg.ReadCid(br) + name = string(sval) + } + + switch name { + // t.State (uint64) (uint64) + case "State": + + maj, extra, err = cbg.CborReadHeader(br) if err != nil { - return xerrors.Errorf("failed to read cid field t.PreCommitMessage: %w", err) - } - - t.PreCommitMessage = &c - } - - } - // t.Seed (storage.SealSeed) (struct) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "Seed" { - return fmt.Errorf("expected struct map entry %s to be Seed", name) - } - - { - - if err := t.Seed.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.CommitMessage (cid.Cid) (struct) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "CommitMessage" { - return fmt.Errorf("expected struct map entry %s to be CommitMessage", name) - } - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { return err } - } else { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err) + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") } + t.State = uint64(extra) + // t.SectorID (uint64) (uint64) + case "SectorID": - t.CommitMessage = &c - } - - } - // t.FaultReportMsg (cid.Cid) (struct) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) - } - - if name != "FaultReportMsg" { - return fmt.Errorf("expected struct map entry %s to be FaultReportMsg", name) - } - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { return err } - } else { + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = uint64(extra) + // t.Nonce (uint64) (uint64) + case "Nonce": - c, err := cbg.ReadCid(br) + maj, extra, err = cbg.CborReadHeader(br) if err != nil { - return xerrors.Errorf("failed to read cid field t.FaultReportMsg: %w", err) + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Nonce = uint64(extra) + // t.Pieces ([]sealing.Piece) (slice) + case "Pieces": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err } - t.FaultReportMsg = &c + if extra > cbg.MaxLength { + return fmt.Errorf("t.Pieces: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + if extra > 0 { + t.Pieces = make([]Piece, extra) + } + for i := 0; i < int(extra); i++ { + + var v Piece + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Pieces[i] = v + } + + // t.CommD ([]uint8) (slice) + case "CommD": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.CommD: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommD = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommD); err != nil { + return err + } + // t.CommR ([]uint8) (slice) + case "CommR": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.CommR: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.CommR = make([]byte, extra) + if _, err := io.ReadFull(br, t.CommR); err != nil { + return err + } + // t.Proof ([]uint8) (slice) + case "Proof": + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Proof: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.Proof = make([]byte, extra) + if _, err := io.ReadFull(br, t.Proof); err != nil { + return err + } + // t.Ticket (sealing.SealTicket) (struct) + case "Ticket": + + { + + if err := t.Ticket.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.PreCommitMessage (cid.Cid) (struct) + case "PreCommitMessage": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PreCommitMessage: %w", err) + } + + t.PreCommitMessage = &c + } + + } + // t.Seed (sealing.SealSeed) (struct) + case "Seed": + + { + + if err := t.Seed.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.CommitMessage (cid.Cid) (struct) + case "CommitMessage": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.CommitMessage: %w", err) + } + + t.CommitMessage = &c + } + + } + // t.FaultReportMsg (cid.Cid) (struct) + case "FaultReportMsg": + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.FaultReportMsg: %w", err) + } + + t.FaultReportMsg = &c + } + + } + // t.LastErr (string) (string) + case "LastErr": + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.LastErr = string(sval) + } + + default: } - - } - // t.LastErr (string) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - name = string(sval) } - if name != "LastErr" { - return fmt.Errorf("expected struct map entry %s to be LastErr", name) - } - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.LastErr = string(sval) - } return nil } diff --git a/storage/sealing/fsm.go b/storage/sealing/fsm.go index e470290a9..8c5f04c2f 100644 --- a/storage/sealing/fsm.go +++ b/storage/sealing/fsm.go @@ -236,6 +236,10 @@ func planOne(ts ...func() (mut mutator, next api.SectorState)) func(events []sta continue } + if err, iserr := events[0].User.(error); iserr { + log.Warnf("sector %d got error event %T: %+v", state.SectorID, events[0].User, err) + } + events[0].User.(mutator).apply(state) state.State = next return nil From d2401bc3393981ffc66c0e75f6567c6499e47ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 21 Jan 2020 17:28:55 +0100 Subject: [PATCH 2/2] Update cbor-gen, error on unknown fields --- api/cbor_gen.go | 3 +++ go.mod | 2 +- go.sum | 4 ++-- lib/statemachine/cbor_gen.go | 2 ++ storage/sealing/cbor_gen.go | 4 ++++ 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/api/cbor_gen.go b/api/cbor_gen.go index 3b1bea712..fd244cdb5 100644 --- a/api/cbor_gen.go +++ b/api/cbor_gen.go @@ -181,6 +181,7 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { } default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) } } @@ -310,6 +311,7 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error { t.Size = uint64(extra) default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) } } @@ -410,6 +412,7 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error { } default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) } } diff --git a/go.mod b/go.mod index cc9f1d9c1..914691b7f 100644 --- a/go.mod +++ b/go.mod @@ -86,7 +86,7 @@ require ( github.com/prometheus/common v0.2.0 github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20200121160201-1834514787d2 + github.com/whyrusleeping/cbor-gen v0.0.0-20200121162646-b63bacf5eaf8 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d go.opencensus.io v0.22.2 diff --git a/go.sum b/go.sum index 1d36c637c..47f2659a5 100644 --- a/go.sum +++ b/go.sum @@ -711,8 +711,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:x github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= -github.com/whyrusleeping/cbor-gen v0.0.0-20200121160201-1834514787d2 h1:VwJlCmEiu+/sl5KUq5w4i5VrhTIAhI++GVHGWf+vGnc= -github.com/whyrusleeping/cbor-gen v0.0.0-20200121160201-1834514787d2/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20200121162646-b63bacf5eaf8 h1:PHZv8Nu+95MBVNu3qSgg3ncxIv8hy4DzGAOBR9xYQRc= +github.com/whyrusleeping/cbor-gen v0.0.0-20200121162646-b63bacf5eaf8/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= diff --git a/lib/statemachine/cbor_gen.go b/lib/statemachine/cbor_gen.go index e32e81697..394be1e72 100644 --- a/lib/statemachine/cbor_gen.go +++ b/lib/statemachine/cbor_gen.go @@ -109,6 +109,7 @@ func (t *TestState) UnmarshalCBOR(r io.Reader) error { t.B = uint64(extra) default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) } } @@ -218,6 +219,7 @@ func (t *TestEvent) UnmarshalCBOR(r io.Reader) error { t.Val = uint64(extra) default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) } } diff --git a/storage/sealing/cbor_gen.go b/storage/sealing/cbor_gen.go index 61ad5b606..1e48bc7d5 100644 --- a/storage/sealing/cbor_gen.go +++ b/storage/sealing/cbor_gen.go @@ -123,6 +123,7 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { } default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) } } @@ -239,6 +240,7 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error { } default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) } } @@ -382,6 +384,7 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { } default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) } } @@ -914,6 +917,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { } default: + return fmt.Errorf("unknown struct field %d: '%s'", i, name) } }