diff --git a/api/cbor_gen.go b/api/cbor_gen.go index ddb4e8416..71c551ebe 100644 --- a/api/cbor_gen.go +++ b/api/cbor_gen.go @@ -244,6 +244,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error { if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil { return err } + return nil } @@ -280,36 +281,48 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error { // t.SectorID (abi.SectorNumber) (uint64) case "SectorID": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = abi.SectorNumber(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = abi.SectorNumber(extra) // t.Offset (uint64) (uint64) case "Offset": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Offset = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Offset = uint64(extra) // t.Size (abi.UnpaddedPieceSize) (uint64) case "Size": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = abi.UnpaddedPieceSize(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Size = abi.UnpaddedPieceSize(extra) default: return fmt.Errorf("unknown struct field %d: '%s'", i, name) diff --git a/chain/blocksync/cbor_gen.go b/chain/blocksync/cbor_gen.go index 9e76e8f64..6668eb3a6 100644 --- a/chain/blocksync/cbor_gen.go +++ b/chain/blocksync/cbor_gen.go @@ -38,14 +38,17 @@ func (t *BlockSyncRequest) MarshalCBOR(w io.Writer) error { } // t.RequestLength (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.RequestLength))); err != nil { return err } // t.Options (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Options))); err != nil { return err } + return nil } @@ -92,24 +95,32 @@ func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error { // t.RequestLength (uint64) (uint64) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.RequestLength = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.RequestLength = uint64(extra) // t.Options (uint64) (uint64) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Options = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Options = uint64(extra) return nil } @@ -137,6 +148,7 @@ func (t *BlockSyncResponse) MarshalCBOR(w io.Writer) error { } // t.Status (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil { return err } @@ -199,14 +211,18 @@ func (t *BlockSyncResponse) UnmarshalCBOR(r io.Reader) error { // t.Status (uint64) (uint64) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Status = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Status = uint64(extra) // t.Message (string) (string) { diff --git a/chain/sync.go b/chain/sync.go index 9b19013b4..5a9fc9e92 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -512,7 +512,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err if h.Timestamp < baseTs.MinTimestamp()+(build.BlockDelay*uint64(h.Height-baseTs.Height())) { log.Warn("timestamp funtimes: ", h.Timestamp, baseTs.MinTimestamp(), h.Height, baseTs.Height()) - diff := (baseTs.MinTimestamp()+(build.BlockDelay*uint64(h.Height-baseTs.Height()))) - h.Timestamp + diff := (baseTs.MinTimestamp() + (build.BlockDelay * uint64(h.Height-baseTs.Height()))) - h.Timestamp return xerrors.Errorf("block was generated too soon (h.ts:%d < base.mints:%d + BLOCK_DELAY:%d * deltaH:%d; diff %d)", h.Timestamp, baseTs.MinTimestamp(), build.BlockDelay, h.Height-baseTs.Height(), diff) } diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index 1bf5937fa..05183a904 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -94,6 +94,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { } // t.Timestamp (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Timestamp))); err != nil { return err } @@ -104,9 +105,11 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error { } // t.ForkSignaling (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ForkSignaling))); err != nil { return err } + return nil } @@ -271,14 +274,18 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { } // t.Timestamp (uint64) (uint64) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Timestamp = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Timestamp = uint64(extra) // t.BlockSig (crypto.Signature) (struct) { @@ -302,14 +309,18 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { } // t.ForkSignaling (uint64) (uint64) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.ForkSignaling = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.ForkSignaling = uint64(extra) return nil } @@ -521,14 +532,17 @@ func (t *EPostTicket) MarshalCBOR(w io.Writer) error { } // t.SectorID (abi.SectorNumber) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil { return err } // t.ChallengeIndex (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ChallengeIndex))); err != nil { return err } + return nil } @@ -566,24 +580,32 @@ func (t *EPostTicket) UnmarshalCBOR(r io.Reader) error { } // t.SectorID (abi.SectorNumber) (uint64) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = abi.SectorNumber(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = abi.SectorNumber(extra) // t.ChallengeIndex (uint64) (uint64) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.ChallengeIndex = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.ChallengeIndex = uint64(extra) return nil } @@ -607,6 +629,7 @@ func (t *Message) MarshalCBOR(w io.Writer) error { } // t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } @@ -627,6 +650,7 @@ func (t *Message) MarshalCBOR(w io.Writer) error { } // t.Method (abi.MethodNum) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil { return err } @@ -680,14 +704,18 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } // t.Nonce (uint64) (uint64) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Nonce = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Nonce = uint64(extra) // t.Value (big.Int) (struct) { @@ -717,14 +745,18 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { } // t.Method (abi.MethodNum) (uint64) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Method = abi.MethodNum(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Method = abi.MethodNum(extra) // t.Params ([]uint8) (slice) maj, extra, err = cbg.CborReadHeader(br) @@ -890,6 +922,7 @@ func (t *Actor) MarshalCBOR(w io.Writer) error { } // t.Nonce (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil { return err } @@ -942,14 +975,18 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error { } // t.Nonce (uint64) (uint64) - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Nonce = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Nonce = uint64(extra) // t.Balance (big.Int) (struct) { diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go index 6bf85568d..03f94d7b5 100644 --- a/chain/vm/syscalls.go +++ b/chain/vm/syscalls.go @@ -3,13 +3,16 @@ package vm import ( ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-address" + commcid "github.com/filecoin-project/go-fil-commcid" "github.com/filecoin-project/go-sectorbuilder" + "github.com/filecoin-project/lotus/lib/zerocomm" "github.com/filecoin-project/specs-actors/actors/abi" "github.com/filecoin-project/specs-actors/actors/crypto" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" "golang.org/x/xerrors" + "math/bits" ) func init() { @@ -29,11 +32,30 @@ type syscallShim struct { func (ss *syscallShim) ComputeUnsealedSectorCID(ssize abi.SectorSize, pieces []abi.PieceInfo) (cid.Cid, error) { // TODO: does this pull in unwanted dependencies? var ffipieces []ffi.PublicPieceInfo + var sum abi.PaddedPieceSize for _, p := range pieces { ffipieces = append(ffipieces, ffi.PublicPieceInfo{ Size: p.Size.Unpadded(), CommP: cidToCommD(p.PieceCID), }) + sum += p.Size + } + + { + // pad remaining space with 0 CommPs + toFill := uint64(abi.PaddedPieceSize(ssize) - sum) + n := bits.OnesCount64(toFill) + for i := 0; i < n; i++ { + next := bits.TrailingZeros64(toFill) + psize := uint64(1) << next + toFill ^= psize + + unpadded := abi.PaddedPieceSize(psize).Unpadded() + ffipieces = append(ffipieces, ffi.PublicPieceInfo{ + Size: unpadded, + CommP: zerocomm.ForSize(unpadded), + }) + } } commd, err := sectorbuilder.GenerateDataCommitment(ssize, ffipieces) @@ -42,14 +64,7 @@ func (ss *syscallShim) ComputeUnsealedSectorCID(ssize abi.SectorSize, pieces []a return cid.Undef, err } - // TODO: pulling these numbers kinda out of an unmerged PR - hb, err := mh.Encode(commd[:], 0xf104) - if err != nil { - log.Errorf("mh encode failed: %s", err) - return cid.Undef, err - } - - return cid.NewCidV1(0xf101, mh.Multihash(hb)), nil + return commcid.DataCommitmentV1ToCID(commd[:]), nil } func (ss *syscallShim) HashBlake2b(data []byte) [32]byte { diff --git a/cli/state.go b/cli/state.go index c28df9528..da3cdfbea 100644 --- a/cli/state.go +++ b/cli/state.go @@ -274,10 +274,8 @@ var stateReplaySetCmd = &cli.Command{ return err } - } - res, err := api.StateReplay(ctx, ts, mcid) if err != nil { return xerrors.Errorf("replay call failed: %w", err) diff --git a/cmd/lotus-seed/genesis.go b/cmd/lotus-seed/genesis.go index 039fc5dfb..187f5b499 100644 --- a/cmd/lotus-seed/genesis.go +++ b/cmd/lotus-seed/genesis.go @@ -39,15 +39,15 @@ var genesisNewCmd = &cli.Command{ } out := genesis.Template{ - Accounts: []genesis.Actor{}, - Miners: []genesis.Miner{}, + Accounts: []genesis.Actor{}, + Miners: []genesis.Miner{}, NetworkName: cctx.String("network-name"), } if out.NetworkName == "" { out.NetworkName = "localnet-" + uuid.New().String() } - genb, err := json.MarshalIndent(&out, ""," ") + genb, err := json.MarshalIndent(&out, "", " ") if err != nil { return err } @@ -68,7 +68,7 @@ var genesisNewCmd = &cli.Command{ var genesisAddMinerCmd = &cli.Command{ Name: "add-miner", Description: "add genesis miner", - Flags: []cli.Flag{}, + Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { if cctx.Args().Len() != 2 { return xerrors.New("seed genesis add-miner [genesis.json] [preseal.json]") @@ -128,7 +128,7 @@ var genesisAddMinerCmd = &cli.Command{ }) } - genb, err = json.MarshalIndent(&template, ""," ") + genb, err = json.MarshalIndent(&template, "", " ") if err != nil { return err } diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index 26e605add..5ee3bd345 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -250,7 +250,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin SectorID: sector.SectorID, Pieces: []sealing.Piece{ { - DealID: dealID, + DealID: &dealID, Size: abi.PaddedPieceSize(meta.SectorSize).Unpadded(), CommP: sector.CommD[:], }, diff --git a/go.mod b/go.mod index cf536ea9a..09c349a72 100644 --- a/go.mod +++ b/go.mod @@ -88,7 +88,7 @@ require ( github.com/prometheus/common v0.4.0 github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20200213013405-80352c7ae952 + github.com/whyrusleeping/cbor-gen v0.0.0-20200222160900-51052a1e8191 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d go.opencensus.io v0.22.2 diff --git a/go.sum b/go.sum index 6fc4f8c29..4c6c0a40b 100644 --- a/go.sum +++ b/go.sum @@ -762,6 +762,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66 h1:LolR9FiE github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200213013405-80352c7ae952 h1:YGoOfwBeeb6A8t3eSe2nmPtMsyYPa6WzBmTbSPLRNK8= github.com/whyrusleeping/cbor-gen v0.0.0-20200213013405-80352c7ae952/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/cbor-gen v0.0.0-20200222160900-51052a1e8191 h1:TeuxLwKwQy612jEhfVhGJTqLsM2EwMi1eJE052ug+NY= +github.com/whyrusleeping/cbor-gen v0.0.0-20200222160900-51052a1e8191/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= diff --git a/lib/statemachine/cbor_gen.go b/lib/statemachine/cbor_gen.go index bf187e209..4bebe01d0 100644 --- a/lib/statemachine/cbor_gen.go +++ b/lib/statemachine/cbor_gen.go @@ -52,6 +52,7 @@ func (t *TestState) MarshalCBOR(w io.Writer) error { if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.B))); err != nil { return err } + return nil } @@ -88,25 +89,33 @@ func (t *TestState) UnmarshalCBOR(r io.Reader) error { // t.A (uint64) (uint64) case "A": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.A = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.A = uint64(extra) // t.B (uint64) (uint64) case "B": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.B = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.B = uint64(extra) default: return fmt.Errorf("unknown struct field %d: '%s'", i, name) @@ -162,6 +171,7 @@ func (t *TestEvent) MarshalCBOR(w io.Writer) error { if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Val))); err != nil { return err } + return nil } @@ -209,14 +219,18 @@ func (t *TestEvent) UnmarshalCBOR(r io.Reader) error { // t.Val (uint64) (uint64) case "Val": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Val = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Val = uint64(extra) default: return fmt.Errorf("unknown struct field %d: '%s'", i, name) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index f45ec98ae..512db2335 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -185,7 +185,10 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb deals := make([]abi.DealID, len(info.Pieces)) for i, piece := range info.Pieces { - deals[i] = piece.DealID + if piece.DealID == nil { + continue + } + deals[i] = *piece.DealID } log := make([]api.SectorLog, len(info.Log)) diff --git a/paychmgr/cbor_gen.go b/paychmgr/cbor_gen.go index 002ee2f08..158fd5912 100644 --- a/paychmgr/cbor_gen.go +++ b/paychmgr/cbor_gen.go @@ -254,6 +254,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error { if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.NextLane))); err != nil { return err } + return nil } @@ -320,14 +321,18 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { // t.Direction (uint64) (uint64) case "Direction": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Direction = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Direction = uint64(extra) // t.Vouchers ([]*paychmgr.VoucherInfo) (slice) case "Vouchers": @@ -359,14 +364,18 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { // t.NextLane (uint64) (uint64) case "NextLane": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.NextLane = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.NextLane = uint64(extra) default: return fmt.Errorf("unknown struct field %d: '%s'", i, name) diff --git a/storage/sealing/cbor_gen.go b/storage/sealing/cbor_gen.go index fe25daa2d..3cd9b6451 100644 --- a/storage/sealing/cbor_gen.go +++ b/storage/sealing/cbor_gen.go @@ -310,8 +310,14 @@ func (t *Piece) MarshalCBOR(w io.Writer) error { return err } - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { - return err + if t.DealID == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(*t.DealID))); err != nil { + return err + } } // t.Size (abi.UnpaddedPieceSize) (uint64) @@ -388,25 +394,45 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { // t.DealID (abi.DealID) (uint64) case "DealID": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + typed := abi.DealID(extra) + t.DealID = &typed + } + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.DealID = abi.DealID(extra) // t.Size (abi.UnpaddedPieceSize) (uint64) case "Size": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Size = abi.UnpaddedPieceSize(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Size = abi.UnpaddedPieceSize(extra) // t.CommP ([]uint8) (slice) case "CommP": @@ -765,36 +791,48 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { // t.State (uint64) (uint64) case "State": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.State = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.State = uint64(extra) // t.SectorID (abi.SectorNumber) (uint64) case "SectorID": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorID = abi.SectorNumber(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.SectorID = abi.SectorNumber(extra) // t.Nonce (uint64) (uint64) case "Nonce": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Nonce = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Nonce = uint64(extra) // t.Pieces ([]sealing.Piece) (slice) case "Pieces": @@ -1147,14 +1185,18 @@ func (t *Log) UnmarshalCBOR(r io.Reader) error { // t.Timestamp (uint64) (uint64) case "Timestamp": - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err + { + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Timestamp = uint64(extra) + } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.Timestamp = uint64(extra) // t.Trace (string) (string) case "Trace": diff --git a/storage/sealing/checks.go b/storage/sealing/checks.go index 6cfddee26..1c5df91bc 100644 --- a/storage/sealing/checks.go +++ b/storage/sealing/checks.go @@ -1,7 +1,12 @@ package sealing import ( + "bytes" "context" + commcid "github.com/filecoin-project/go-fil-commcid" + "github.com/filecoin-project/lotus/lib/zerocomm" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/multiformats/go-multihash" @@ -19,6 +24,7 @@ import ( type ErrApi struct{ error } type ErrInvalidDeals struct{ error } +type ErrInvalidPiece struct{ error } type ErrExpiredDeals struct{ error } type ErrBadCommD struct{ error } @@ -36,7 +42,14 @@ func checkPieces(ctx context.Context, si SectorInfo, api sealingApi) error { } for i, piece := range si.Pieces { - deal, err := api.StateMarketStorageDeal(ctx, piece.DealID, nil) + if piece.DealID == nil { + exp := zerocomm.ForSize(piece.Size) + if string(piece.CommP) != string(exp[:]) { + return &ErrInvalidPiece{xerrors.Errorf("deal %d piece %d had non-zero CommP %+v", piece.DealID, i, piece.CommP)} + } + continue + } + deal, err := api.StateMarketStorageDeal(ctx, *piece.DealID, nil) if err != nil { return &ErrApi{xerrors.Errorf("getting deal %d for piece %d: %w", piece.DealID, i, err)} } @@ -99,7 +112,17 @@ func checkSeal(ctx context.Context, maddr address.Address, si SectorInfo, api se if r.ExitCode != 0 { return &ErrBadCommD{xerrors.Errorf("receipt for ComputeDataCommitment had exit code %d", r.ExitCode)} } - if string(r.Return) != string(si.CommD) { + + var c cbg.CborCid + if err := c.UnmarshalCBOR(bytes.NewReader(r.Return)); err != nil { + return err + } + cd, err := commcid.CIDToDataCommitmentV1(cid.Cid(c)) + if err != nil { + return err + } + + if string(cd) != string(si.CommD) { return &ErrBadCommD{xerrors.Errorf("on chain CommD differs from sector: %x != %x", r.Return, si.CommD)} } diff --git a/storage/sealing/fsm.go b/storage/sealing/fsm.go index 59449303b..06ba11354 100644 --- a/storage/sealing/fsm.go +++ b/storage/sealing/fsm.go @@ -228,7 +228,7 @@ func (m *Sealing) restartSectors(ctx context.Context) error { } for _, sector := range trackedSectors { - if err := m.sectors.Send(sector.SectorID, SectorRestart{}); err != nil { + if err := m.sectors.Send(uint64(sector.SectorID), SectorRestart{}); err != nil { log.Errorf("restarting sector %d: %+v", sector.SectorID, err) } } diff --git a/storage/sealing/garbage.go b/storage/sealing/garbage.go index cb68eda48..ecf0a1179 100644 --- a/storage/sealing/garbage.go +++ b/storage/sealing/garbage.go @@ -1,37 +1,25 @@ package sealing import ( - "bytes" "context" "io" - "math" - "math/bits" - "math/rand" - commcid "github.com/filecoin-project/go-fil-commcid" - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/market" "golang.org/x/xerrors" - "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/abi" ) -func (m *Sealing) pledgeReader(size abi.UnpaddedPieceSize, parts uint64) io.Reader { - parts = 1 << bits.Len64(parts) // round down to nearest power of 2 - if uint64(size)/parts < 127 { - parts = uint64(size) / 127 +type nullReader struct {} + +func (nullReader) Read(out []byte) (int, error) { + for i := range out { + out[i] = 0 } + return len(out), nil +} - piece := abi.PaddedPieceSize(uint64(size.Padded()) / parts).Unpadded() - - readers := make([]io.Reader, parts) - for i := range readers { - readers[i] = io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece)) - } - - return io.MultiReader(readers...) +func (m *Sealing) pledgeReader(size abi.UnpaddedPieceSize) io.Reader { + return io.LimitReader(&nullReader{}, int64(size)) } func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, existingPieceSizes []abi.UnpaddedPieceSize, sizes ...abi.UnpaddedPieceSize) ([]Piece, error) { @@ -41,70 +29,9 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, e log.Infof("Pledge %d, contains %+v", sectorID, existingPieceSizes) - deals := make([]market.ClientDealProposal, len(sizes)) - for i, size := range sizes { - commP, err := m.fastPledgeCommitment(size, uint64(1)) - if err != nil { - return nil, err - } - - sdp := market.DealProposal{ - PieceCID: commcid.PieceCommitmentV1ToCID(commP[:]), - PieceSize: size.Padded(), - Client: m.worker, - Provider: m.maddr, - StartEpoch: math.MaxInt64, - EndEpoch: math.MaxInt64, - StoragePricePerEpoch: types.NewInt(0), - ProviderCollateral: types.NewInt(0), - } - - deals[i] = market.ClientDealProposal{ - Proposal: sdp, - } - } - - log.Infof("Publishing deals for %d", sectorID) - - params, aerr := actors.SerializeParams(&market.PublishStorageDealsParams{ - Deals: deals, - }) - if aerr != nil { - return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", aerr) - } - - smsg, err := m.api.MpoolPushMessage(ctx, &types.Message{ - To: actors.StorageMarketAddress, - From: m.worker, - Value: types.NewInt(0), - GasPrice: types.NewInt(0), - GasLimit: types.NewInt(1000000), - Method: builtin.MethodsMarket.PublishStorageDeals, - Params: params, - }) - if err != nil { - return nil, err - } - r, err := m.api.StateWaitMsg(ctx, smsg.Cid()) // TODO: more finality - if err != nil { - return nil, err - } - if r.Receipt.ExitCode != 0 { - log.Error(xerrors.Errorf("publishing deal (ts %s) %s failed: exit %d", r.TipSet.Key(), smsg.Cid(), r.Receipt.ExitCode)) - } - var resp actors.PublishStorageDealResponse - if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil { - return nil, err - } - if len(resp.IDs) != len(sizes) { - return nil, xerrors.New("got unexpected number of DealIDs from PublishStorageDeals") - } - - log.Infof("Deals for sector %d: %+v", sectorID, resp.IDs) - out := make([]Piece, len(sizes)) for i, size := range sizes { - ppi, err := m.sb.AddPiece(ctx, size, sectorID, m.pledgeReader(size, uint64(1)), existingPieceSizes) + ppi, err := m.sb.AddPiece(ctx, size, sectorID, m.pledgeReader(size), existingPieceSizes) if err != nil { return nil, xerrors.Errorf("add piece: %w", err) } @@ -112,9 +39,8 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, e existingPieceSizes = append(existingPieceSizes, size) out[i] = Piece{ - DealID: resp.IDs[i], - Size: abi.UnpaddedPieceSize(ppi.Size), - CommP: ppi.CommP[:], + Size: ppi.Size, + CommP: ppi.CommP[:], } } @@ -141,7 +67,7 @@ func (m *Sealing) PledgeSector() error { return } - if err := m.newSector(context.TODO(), sid, pieces[0].DealID, pieces[0].ppi()); err != nil { + if err := m.newSector(sid, pieces); err != nil { log.Errorf("%+v", err) return } diff --git a/storage/sealing/sealing.go b/storage/sealing/sealing.go index 9f230d578..1ad733935 100644 --- a/storage/sealing/sealing.go +++ b/storage/sealing/sealing.go @@ -120,20 +120,20 @@ func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r i return xerrors.Errorf("adding piece to sector: %w", err) } - return m.newSector(ctx, sectorID, dealID, ppi) + return m.newSector(sectorID, []Piece{ + { + DealID: &dealID, + + Size: ppi.Size, + CommP: ppi.CommP[:], + }, + },) } -func (m *Sealing) newSector(ctx context.Context, sid abi.SectorNumber, dealID abi.DealID, ppi sectorbuilder.PublicPieceInfo) error { +func (m *Sealing) newSector(sid abi.SectorNumber, pieces []Piece) error { log.Infof("Start sealing %d", sid) - return m.sectors.Send(sid, SectorStart{ + return m.sectors.Send(uint64(sid), SectorStart{ id: sid, - pieces: []Piece{ - { - DealID: dealID, - - Size: abi.UnpaddedPieceSize(ppi.Size), - CommP: ppi.CommP[:], - }, - }, + pieces: pieces, }) } diff --git a/storage/sealing/states.go b/storage/sealing/states.go index 4cd6549a8..273f9fd4f 100644 --- a/storage/sealing/states.go +++ b/storage/sealing/states.go @@ -99,8 +99,9 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf } params := &miner.SectorPreCommitInfo{ - Expiration: 0, + Expiration: 10000000, // TODO: implement SectorNumber: sector.SectorID, + RegisteredProof: abi.RegisteredProof_StackedDRG32GiBSeal, SealedCID: commcid.ReplicaCommitmentV1ToCID(sector.CommR), SealRandEpoch: sector.Ticket.BlockHeight, diff --git a/storage/sealing/types.go b/storage/sealing/types.go index ccfe50427..e2a2c049d 100644 --- a/storage/sealing/types.go +++ b/storage/sealing/types.go @@ -34,7 +34,7 @@ func (t *SealSeed) Equals(o *SealSeed) bool { } type Piece struct { - DealID abi.DealID + DealID *abi.DealID Size abi.UnpaddedPieceSize CommP []byte @@ -97,9 +97,12 @@ func (t *SectorInfo) pieceInfos() []sectorbuilder.PublicPieceInfo { } func (t *SectorInfo) deals() []abi.DealID { - out := make([]abi.DealID, len(t.Pieces)) - for i, piece := range t.Pieces { - out[i] = piece.DealID + out := make([]abi.DealID, 0, len(t.Pieces)) + for _, piece := range t.Pieces { + if piece.DealID == nil { + continue + } + out = append(out, *piece.DealID) } return out } diff --git a/storage/sealing/utils.go b/storage/sealing/utils.go index 0bfbb30fe..cfc17734c 100644 --- a/storage/sealing/utils.go +++ b/storage/sealing/utils.go @@ -1,15 +1,8 @@ package sealing import ( - "io" - "math/bits" - "math/rand" - "sync" - "github.com/filecoin-project/specs-actors/actors/abi" - "github.com/hashicorp/go-multierror" - - sectorbuilder "github.com/filecoin-project/go-sectorbuilder" + "math/bits" ) func fillersFromRem(in abi.UnpaddedPieceSize) ([]abi.UnpaddedPieceSize, error) { @@ -48,44 +41,6 @@ func fillersFromRem(in abi.UnpaddedPieceSize) ([]abi.UnpaddedPieceSize, error) { return out, nil } -func (m *Sealing) fastPledgeCommitment(size abi.UnpaddedPieceSize, parts uint64) (commP [sectorbuilder.CommLen]byte, err error) { - parts = 1 << bits.Len64(parts) // round down to nearest power of 2 - if uint64(size)/parts < 127 { - parts = uint64(size) / 127 - } - - piece := abi.PaddedPieceSize(uint64(size.Padded()) / parts).Unpadded() - out := make([]sectorbuilder.PublicPieceInfo, parts) - var lk sync.Mutex - - var wg sync.WaitGroup - wg.Add(int(parts)) - for i := uint64(0); i < parts; i++ { - go func(i uint64) { - defer wg.Done() - - commP, perr := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece)), piece) - - lk.Lock() - if perr != nil { - err = multierror.Append(err, perr) - } - out[i] = sectorbuilder.PublicPieceInfo{ - Size: piece, - CommP: commP, - } - lk.Unlock() - }(i) - } - wg.Wait() - - if err != nil { - return [32]byte{}, err - } - - return sectorbuilder.GenerateDataCommitment(m.sb.SectorSize(), out) -} - func (m *Sealing) ListSectors() ([]SectorInfo, error) { var sectors []SectorInfo if err := m.sectors.List(§ors); err != nil { @@ -96,6 +51,6 @@ func (m *Sealing) ListSectors() ([]SectorInfo, error) { func (m *Sealing) GetSectorInfo(sid abi.SectorNumber) (SectorInfo, error) { var out SectorInfo - err := m.sectors.Get(sid).Get(&out) + err := m.sectors.Get(uint64(sid)).Get(&out) return out, err }