Implement committed capacity sectors

api/cbor_gen.go

@@ -244,6 +244,7 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Size))); err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -280,36 +281,48 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 		// t.SectorID (abi.SectorNumber) (uint64)
 		case "SectorID":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.SectorID = abi.SectorNumber(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.SectorID = abi.SectorNumber(extra)
 			// t.Offset (uint64) (uint64)
 		case "Offset":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.Offset = uint64(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.Offset = uint64(extra)
 			// t.Size (abi.UnpaddedPieceSize) (uint64)
 		case "Size":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.Size = abi.UnpaddedPieceSize(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.Size = abi.UnpaddedPieceSize(extra)
 
 		default:
 			return fmt.Errorf("unknown struct field %d: '%s'", i, name)

chain/blocksync/cbor_gen.go

@@ -38,14 +38,17 @@ func (t *BlockSyncRequest) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.RequestLength (uint64) (uint64)
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.RequestLength))); err != nil {
 		return err
 	}
 
 	// t.Options (uint64) (uint64)
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Options))); err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -92,24 +95,32 @@ func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error {
 
 	// t.RequestLength (uint64) (uint64)
 
-	maj, extra, err = cbg.CborReadHeader(br)
+	{
-	if err != nil {
+
-		return err
+		maj, extra, err = cbg.CborReadHeader(br)
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.RequestLength = uint64(extra)
+
 	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.RequestLength = uint64(extra)
 	// t.Options (uint64) (uint64)
 
-	maj, extra, err = cbg.CborReadHeader(br)
+	{
-	if err != nil {
+
-		return err
+		maj, extra, err = cbg.CborReadHeader(br)
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.Options = uint64(extra)
+
 	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.Options = uint64(extra)
 	return nil
 }
 
@@ -137,6 +148,7 @@ func (t *BlockSyncResponse) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Status (uint64) (uint64)
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Status))); err != nil {
 		return err
 	}
@@ -199,14 +211,18 @@ func (t *BlockSyncResponse) UnmarshalCBOR(r io.Reader) error {
 
 	// t.Status (uint64) (uint64)
 
-	maj, extra, err = cbg.CborReadHeader(br)
+	{
-	if err != nil {
+
-		return err
+		maj, extra, err = cbg.CborReadHeader(br)
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.Status = uint64(extra)
+
 	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.Status = uint64(extra)
 	// t.Message (string) (string)
 
 	{

chain/sync.go

@@ -512,7 +512,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
 
 	if h.Timestamp < baseTs.MinTimestamp()+(build.BlockDelay*uint64(h.Height-baseTs.Height())) {
 		log.Warn("timestamp funtimes: ", h.Timestamp, baseTs.MinTimestamp(), h.Height, baseTs.Height())
-		diff := (baseTs.MinTimestamp()+(build.BlockDelay*uint64(h.Height-baseTs.Height()))) - h.Timestamp
+		diff := (baseTs.MinTimestamp() + (build.BlockDelay * uint64(h.Height-baseTs.Height()))) - h.Timestamp
 
 		return xerrors.Errorf("block was generated too soon (h.ts:%d < base.mints:%d + BLOCK_DELAY:%d * deltaH:%d; diff %d)", h.Timestamp, baseTs.MinTimestamp(), build.BlockDelay, h.Height-baseTs.Height(), diff)
 	}

chain/types/cbor_gen.go

@@ -94,6 +94,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Timestamp (uint64) (uint64)
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Timestamp))); err != nil {
 		return err
 	}
@@ -104,9 +105,11 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.ForkSignaling (uint64) (uint64)
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ForkSignaling))); err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -271,14 +274,18 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.Timestamp (uint64) (uint64)
 
-	maj, extra, err = cbg.CborReadHeader(br)
+	{
-	if err != nil {
+
-		return err
+		maj, extra, err = cbg.CborReadHeader(br)
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.Timestamp = uint64(extra)
+
 	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.Timestamp = uint64(extra)
 	// t.BlockSig (crypto.Signature) (struct)
 
 	{
@@ -302,14 +309,18 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.ForkSignaling (uint64) (uint64)
 
-	maj, extra, err = cbg.CborReadHeader(br)
+	{
-	if err != nil {
+
-		return err
+		maj, extra, err = cbg.CborReadHeader(br)
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.ForkSignaling = uint64(extra)
+
 	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.ForkSignaling = uint64(extra)
 	return nil
 }
 
@@ -521,14 +532,17 @@ func (t *EPostTicket) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.SectorID (abi.SectorNumber) (uint64)
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil {
 		return err
 	}
 
 	// t.ChallengeIndex (uint64) (uint64)
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ChallengeIndex))); err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -566,24 +580,32 @@ func (t *EPostTicket) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.SectorID (abi.SectorNumber) (uint64)
 
-	maj, extra, err = cbg.CborReadHeader(br)
+	{
-	if err != nil {
+
-		return err
+		maj, extra, err = cbg.CborReadHeader(br)
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.SectorID = abi.SectorNumber(extra)
+
 	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.SectorID = abi.SectorNumber(extra)
 	// t.ChallengeIndex (uint64) (uint64)
 
-	maj, extra, err = cbg.CborReadHeader(br)
+	{
-	if err != nil {
+
-		return err
+		maj, extra, err = cbg.CborReadHeader(br)
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.ChallengeIndex = uint64(extra)
+
 	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.ChallengeIndex = uint64(extra)
 	return nil
 }
 
@@ -607,6 +629,7 @@ func (t *Message) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Nonce (uint64) (uint64)
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil {
 		return err
 	}
@@ -627,6 +650,7 @@ func (t *Message) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Method (abi.MethodNum) (uint64)
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Method))); err != nil {
 		return err
 	}
@@ -680,14 +704,18 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.Nonce (uint64) (uint64)
 
-	maj, extra, err = cbg.CborReadHeader(br)
+	{
-	if err != nil {
+
-		return err
+		maj, extra, err = cbg.CborReadHeader(br)
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.Nonce = uint64(extra)
+
 	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.Nonce = uint64(extra)
 	// t.Value (big.Int) (struct)
 
 	{
@@ -717,14 +745,18 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.Method (abi.MethodNum) (uint64)
 
-	maj, extra, err = cbg.CborReadHeader(br)
+	{
-	if err != nil {
+
-		return err
+		maj, extra, err = cbg.CborReadHeader(br)
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.Method = abi.MethodNum(extra)
+
 	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.Method = abi.MethodNum(extra)
 	// t.Params ([]uint8) (slice)
 
 	maj, extra, err = cbg.CborReadHeader(br)
@@ -890,6 +922,7 @@ func (t *Actor) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Nonce (uint64) (uint64)
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Nonce))); err != nil {
 		return err
 	}
@@ -942,14 +975,18 @@ func (t *Actor) UnmarshalCBOR(r io.Reader) error {
 	}
 	// t.Nonce (uint64) (uint64)
 
-	maj, extra, err = cbg.CborReadHeader(br)
+	{
-	if err != nil {
+
-		return err
+		maj, extra, err = cbg.CborReadHeader(br)
+		if err != nil {
+			return err
+		}
+		if maj != cbg.MajUnsignedInt {
+			return fmt.Errorf("wrong type for uint64 field")
+		}
+		t.Nonce = uint64(extra)
+
 	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.Nonce = uint64(extra)
 	// t.Balance (big.Int) (struct)
 
 	{

chain/vm/syscalls.go

@@ -3,13 +3,16 @@ package vm
 import (
 	ffi "github.com/filecoin-project/filecoin-ffi"
 	"github.com/filecoin-project/go-address"
+	commcid "github.com/filecoin-project/go-fil-commcid"
 	"github.com/filecoin-project/go-sectorbuilder"
+	"github.com/filecoin-project/lotus/lib/zerocomm"
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/crypto"
 	"github.com/filecoin-project/specs-actors/actors/runtime"
 	"github.com/ipfs/go-cid"
 	mh "github.com/multiformats/go-multihash"
 	"golang.org/x/xerrors"
+	"math/bits"
 )
 
 func init() {
@@ -29,11 +32,30 @@ type syscallShim struct {
 func (ss *syscallShim) ComputeUnsealedSectorCID(ssize abi.SectorSize, pieces []abi.PieceInfo) (cid.Cid, error) {
 	// TODO: does this pull in unwanted dependencies?
 	var ffipieces []ffi.PublicPieceInfo
+	var sum abi.PaddedPieceSize
 	for _, p := range pieces {
 		ffipieces = append(ffipieces, ffi.PublicPieceInfo{
 			Size:  p.Size.Unpadded(),
 			CommP: cidToCommD(p.PieceCID),
 		})
+		sum += p.Size
+	}
+
+	{
+		// pad remaining space with 0 CommPs
+		toFill := uint64(abi.PaddedPieceSize(ssize) - sum)
+		n := bits.OnesCount64(toFill)
+		for i := 0; i < n; i++ {
+			next := bits.TrailingZeros64(toFill)
+			psize := uint64(1) << next
+			toFill ^= psize
+
+			unpadded := abi.PaddedPieceSize(psize).Unpadded()
+			ffipieces = append(ffipieces, ffi.PublicPieceInfo{
+				Size:  unpadded,
+				CommP: zerocomm.ForSize(unpadded),
+			})
+		}
 	}
 
 	commd, err := sectorbuilder.GenerateDataCommitment(ssize, ffipieces)
@@ -42,14 +64,7 @@ func (ss *syscallShim) ComputeUnsealedSectorCID(ssize abi.SectorSize, pieces []a
 		return cid.Undef, err
 	}
 
-	// TODO: pulling these numbers kinda out of an unmerged PR
+	return commcid.DataCommitmentV1ToCID(commd[:]), nil
-	hb, err := mh.Encode(commd[:], 0xf104)
-	if err != nil {
-		log.Errorf("mh encode failed: %s", err)
-		return cid.Undef, err
-	}
-
-	return cid.NewCidV1(0xf101, mh.Multihash(hb)), nil
 }
 
 func (ss *syscallShim) HashBlake2b(data []byte) [32]byte {

cli/state.go

@@ -274,10 +274,8 @@ var stateReplaySetCmd = &cli.Command{
 				return err
 			}
 
-
 		}
 
-
 		res, err := api.StateReplay(ctx, ts, mcid)
 		if err != nil {
 			return xerrors.Errorf("replay call failed: %w", err)

cmd/lotus-seed/genesis.go

@@ -39,15 +39,15 @@ var genesisNewCmd = &cli.Command{
 		}
 
 		out := genesis.Template{
-			Accounts: []genesis.Actor{},
+			Accounts:    []genesis.Actor{},
-			Miners: []genesis.Miner{},
+			Miners:      []genesis.Miner{},
 			NetworkName: cctx.String("network-name"),
 		}
 		if out.NetworkName == "" {
 			out.NetworkName = "localnet-" + uuid.New().String()
 		}
 
-		genb, err := json.MarshalIndent(&out, "","  ")
+		genb, err := json.MarshalIndent(&out, "", "  ")
 		if err != nil {
 			return err
 		}
@@ -68,7 +68,7 @@ var genesisNewCmd = &cli.Command{
 var genesisAddMinerCmd = &cli.Command{
 	Name:        "add-miner",
 	Description: "add genesis miner",
-	Flags: []cli.Flag{},
+	Flags:       []cli.Flag{},
 	Action: func(cctx *cli.Context) error {
 		if cctx.Args().Len() != 2 {
 			return xerrors.New("seed genesis add-miner [genesis.json] [preseal.json]")
@@ -128,7 +128,7 @@ var genesisAddMinerCmd = &cli.Command{
 			})
 		}
 
-		genb, err = json.MarshalIndent(&template, "","  ")
+		genb, err = json.MarshalIndent(&template, "", "  ")
 		if err != nil {
 			return err
 		}

cmd/lotus-storage-miner/init.go

@@ -250,7 +250,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin
 			SectorID: sector.SectorID,
 			Pieces: []sealing.Piece{
 				{
-					DealID: dealID,
+					DealID: &dealID,
 					Size:   abi.PaddedPieceSize(meta.SectorSize).Unpadded(),
 					CommP:  sector.CommD[:],
 				},

go.mod

@@ -88,7 +88,7 @@ require (
 	github.com/prometheus/common v0.4.0
 	github.com/stretchr/testify v1.4.0
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
-	github.com/whyrusleeping/cbor-gen v0.0.0-20200213013405-80352c7ae952
+	github.com/whyrusleeping/cbor-gen v0.0.0-20200222160900-51052a1e8191
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
 	github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d
 	go.opencensus.io v0.22.2

go.sum

@@ -762,6 +762,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66 h1:LolR9FiE
 github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200213013405-80352c7ae952 h1:YGoOfwBeeb6A8t3eSe2nmPtMsyYPa6WzBmTbSPLRNK8=
 github.com/whyrusleeping/cbor-gen v0.0.0-20200213013405-80352c7ae952/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
+github.com/whyrusleeping/cbor-gen v0.0.0-20200222160900-51052a1e8191 h1:TeuxLwKwQy612jEhfVhGJTqLsM2EwMi1eJE052ug+NY=
+github.com/whyrusleeping/cbor-gen v0.0.0-20200222160900-51052a1e8191/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=

lib/statemachine/cbor_gen.go

@@ -52,6 +52,7 @@ func (t *TestState) MarshalCBOR(w io.Writer) error {
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.B))); err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -88,25 +89,33 @@ func (t *TestState) UnmarshalCBOR(r io.Reader) error {
 		// t.A (uint64) (uint64)
 		case "A":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.A = uint64(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.A = uint64(extra)
 			// t.B (uint64) (uint64)
 		case "B":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.B = uint64(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.B = uint64(extra)
 
 		default:
 			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
@@ -162,6 +171,7 @@ func (t *TestEvent) MarshalCBOR(w io.Writer) error {
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Val))); err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -209,14 +219,18 @@ func (t *TestEvent) UnmarshalCBOR(r io.Reader) error {
 			// t.Val (uint64) (uint64)
 		case "Val":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.Val = uint64(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.Val = uint64(extra)
 
 		default:
 			return fmt.Errorf("unknown struct field %d: '%s'", i, name)

node/impl/storminer.go

@@ -185,7 +185,10 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid abi.SectorNumb
 
 	deals := make([]abi.DealID, len(info.Pieces))
 	for i, piece := range info.Pieces {
-		deals[i] = piece.DealID
+		if piece.DealID == nil {
+			continue
+		}
+		deals[i] = *piece.DealID
 	}
 
 	log := make([]api.SectorLog, len(info.Log))

paychmgr/cbor_gen.go

@@ -254,6 +254,7 @@ func (t *ChannelInfo) MarshalCBOR(w io.Writer) error {
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.NextLane))); err != nil {
 		return err
 	}
+
 	return nil
 }
 
@@ -320,14 +321,18 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 			// t.Direction (uint64) (uint64)
 		case "Direction":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.Direction = uint64(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.Direction = uint64(extra)
 			// t.Vouchers ([]*paychmgr.VoucherInfo) (slice)
 		case "Vouchers":
 
@@ -359,14 +364,18 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error {
 			// t.NextLane (uint64) (uint64)
 		case "NextLane":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.NextLane = uint64(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.NextLane = uint64(extra)
 
 		default:
 			return fmt.Errorf("unknown struct field %d: '%s'", i, name)

storage/sealing/cbor_gen.go

@@ -310,8 +310,14 @@ func (t *Piece) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
+	if t.DealID == nil {
-		return err
+		if _, err := w.Write(cbg.CborNull); err != nil {
+			return err
+		}
+	} else {
+		if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(*t.DealID))); err != nil {
+			return err
+		}
 	}
 
 	// t.Size (abi.UnpaddedPieceSize) (uint64)
@@ -388,25 +394,45 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error {
 		// t.DealID (abi.DealID) (uint64)
 		case "DealID":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				pb, err := br.PeekByte()
+				if err != nil {
+					return err
+				}
+				if pb == cbg.CborNull[0] {
+					var nbuf [1]byte
+					if _, err := br.Read(nbuf[:]); err != nil {
+						return err
+					}
+				} else {
+					maj, extra, err = cbg.CborReadHeader(br)
+					if err != nil {
+						return err
+					}
+					if maj != cbg.MajUnsignedInt {
+						return fmt.Errorf("wrong type for uint64 field")
+					}
+					typed := abi.DealID(extra)
+					t.DealID = &typed
+				}
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.DealID = abi.DealID(extra)
 			// t.Size (abi.UnpaddedPieceSize) (uint64)
 		case "Size":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.Size = abi.UnpaddedPieceSize(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.Size = abi.UnpaddedPieceSize(extra)
 			// t.CommP ([]uint8) (slice)
 		case "CommP":
 
@@ -765,36 +791,48 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
 		// t.State (uint64) (uint64)
 		case "State":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.State = uint64(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.State = uint64(extra)
 			// t.SectorID (abi.SectorNumber) (uint64)
 		case "SectorID":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.SectorID = abi.SectorNumber(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.SectorID = abi.SectorNumber(extra)
 			// t.Nonce (uint64) (uint64)
 		case "Nonce":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.Nonce = uint64(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.Nonce = uint64(extra)
 			// t.Pieces ([]sealing.Piece) (slice)
 		case "Pieces":
 
@@ -1147,14 +1185,18 @@ func (t *Log) UnmarshalCBOR(r io.Reader) error {
 		// t.Timestamp (uint64) (uint64)
 		case "Timestamp":
 
-			maj, extra, err = cbg.CborReadHeader(br)
+			{
-			if err != nil {
+
-				return err
+				maj, extra, err = cbg.CborReadHeader(br)
+				if err != nil {
+					return err
+				}
+				if maj != cbg.MajUnsignedInt {
+					return fmt.Errorf("wrong type for uint64 field")
+				}
+				t.Timestamp = uint64(extra)
+
 			}
-			if maj != cbg.MajUnsignedInt {
-				return fmt.Errorf("wrong type for uint64 field")
-			}
-			t.Timestamp = uint64(extra)
 			// t.Trace (string) (string)
 		case "Trace":
 

storage/sealing/checks.go

@@ -1,7 +1,12 @@
 package sealing
 
 import (
+	"bytes"
 	"context"
+	commcid "github.com/filecoin-project/go-fil-commcid"
+	"github.com/filecoin-project/lotus/lib/zerocomm"
+	"github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
 
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/multiformats/go-multihash"
@@ -19,6 +24,7 @@ import (
 type ErrApi struct{ error }
 
 type ErrInvalidDeals struct{ error }
+type ErrInvalidPiece struct{ error }
 type ErrExpiredDeals struct{ error }
 
 type ErrBadCommD struct{ error }
@@ -36,7 +42,14 @@ func checkPieces(ctx context.Context, si SectorInfo, api sealingApi) error {
 	}
 
 	for i, piece := range si.Pieces {
-		deal, err := api.StateMarketStorageDeal(ctx, piece.DealID, nil)
+		if piece.DealID == nil {
+			exp := zerocomm.ForSize(piece.Size)
+			if string(piece.CommP) != string(exp[:]) {
+				return &ErrInvalidPiece{xerrors.Errorf("deal %d piece %d had non-zero CommP %+v", piece.DealID, i, piece.CommP)}
+			}
+			continue
+		}
+		deal, err := api.StateMarketStorageDeal(ctx, *piece.DealID, nil)
 		if err != nil {
 			return &ErrApi{xerrors.Errorf("getting deal %d for piece %d: %w", piece.DealID, i, err)}
 		}
@@ -99,7 +112,17 @@ func checkSeal(ctx context.Context, maddr address.Address, si SectorInfo, api se
 	if r.ExitCode != 0 {
 		return &ErrBadCommD{xerrors.Errorf("receipt for ComputeDataCommitment had exit code %d", r.ExitCode)}
 	}
-	if string(r.Return) != string(si.CommD) {
+
+	var c cbg.CborCid
+	if err := c.UnmarshalCBOR(bytes.NewReader(r.Return)); err != nil {
+		return err
+	}
+	cd, err := commcid.CIDToDataCommitmentV1(cid.Cid(c))
+	if err != nil {
+		return err
+	}
+
+	if string(cd) != string(si.CommD) {
 		return &ErrBadCommD{xerrors.Errorf("on chain CommD differs from sector: %x != %x", r.Return, si.CommD)}
 	}
 

storage/sealing/fsm.go

@@ -228,7 +228,7 @@ func (m *Sealing) restartSectors(ctx context.Context) error {
 	}
 
 	for _, sector := range trackedSectors {
-		if err := m.sectors.Send(sector.SectorID, SectorRestart{}); err != nil {
+		if err := m.sectors.Send(uint64(sector.SectorID), SectorRestart{}); err != nil {
 			log.Errorf("restarting sector %d: %+v", sector.SectorID, err)
 		}
 	}

storage/sealing/garbage.go

@@ -1,37 +1,25 @@
 package sealing
 
 import (
-	"bytes"
 	"context"
 	"io"
-	"math"
-	"math/bits"
-	"math/rand"
 
-	commcid "github.com/filecoin-project/go-fil-commcid"
-	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/builtin"
-	"github.com/filecoin-project/specs-actors/actors/builtin/market"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/lotus/chain/types"
 )
 
-func (m *Sealing) pledgeReader(size abi.UnpaddedPieceSize, parts uint64) io.Reader {
+type nullReader struct {}
-	parts = 1 << bits.Len64(parts) // round down to nearest power of 2
+
-	if uint64(size)/parts < 127 {
+func (nullReader) Read(out []byte) (int, error) {
-		parts = uint64(size) / 127
+	for i := range out {
+		out[i] = 0
 	}
+	return len(out), nil
+}
 
-	piece := abi.PaddedPieceSize(uint64(size.Padded()) / parts).Unpadded()
+func (m *Sealing) pledgeReader(size abi.UnpaddedPieceSize) io.Reader {
-
+	return io.LimitReader(&nullReader{}, int64(size))
-	readers := make([]io.Reader, parts)
-	for i := range readers {
-		readers[i] = io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece))
-	}
-
-	return io.MultiReader(readers...)
 }
 
 func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, existingPieceSizes []abi.UnpaddedPieceSize, sizes ...abi.UnpaddedPieceSize) ([]Piece, error) {
@@ -41,70 +29,9 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, e
 
 	log.Infof("Pledge %d, contains %+v", sectorID, existingPieceSizes)
 
-	deals := make([]market.ClientDealProposal, len(sizes))
-	for i, size := range sizes {
-		commP, err := m.fastPledgeCommitment(size, uint64(1))
-		if err != nil {
-			return nil, err
-		}
-
-		sdp := market.DealProposal{
-			PieceCID:             commcid.PieceCommitmentV1ToCID(commP[:]),
-			PieceSize:            size.Padded(),
-			Client:               m.worker,
-			Provider:             m.maddr,
-			StartEpoch:           math.MaxInt64,
-			EndEpoch:             math.MaxInt64,
-			StoragePricePerEpoch: types.NewInt(0),
-			ProviderCollateral:   types.NewInt(0),
-		}
-
-		deals[i] = market.ClientDealProposal{
-			Proposal: sdp,
-		}
-	}
-
-	log.Infof("Publishing deals for %d", sectorID)
-
-	params, aerr := actors.SerializeParams(&market.PublishStorageDealsParams{
-		Deals: deals,
-	})
-	if aerr != nil {
-		return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", aerr)
-	}
-
-	smsg, err := m.api.MpoolPushMessage(ctx, &types.Message{
-		To:       actors.StorageMarketAddress,
-		From:     m.worker,
-		Value:    types.NewInt(0),
-		GasPrice: types.NewInt(0),
-		GasLimit: types.NewInt(1000000),
-		Method:   builtin.MethodsMarket.PublishStorageDeals,
-		Params:   params,
-	})
-	if err != nil {
-		return nil, err
-	}
-	r, err := m.api.StateWaitMsg(ctx, smsg.Cid()) // TODO: more finality
-	if err != nil {
-		return nil, err
-	}
-	if r.Receipt.ExitCode != 0 {
-		log.Error(xerrors.Errorf("publishing deal (ts %s) %s failed: exit %d", r.TipSet.Key(), smsg.Cid(), r.Receipt.ExitCode))
-	}
-	var resp actors.PublishStorageDealResponse
-	if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil {
-		return nil, err
-	}
-	if len(resp.IDs) != len(sizes) {
-		return nil, xerrors.New("got unexpected number of DealIDs from PublishStorageDeals")
-	}
-
-	log.Infof("Deals for sector %d: %+v", sectorID, resp.IDs)
-
 	out := make([]Piece, len(sizes))
 	for i, size := range sizes {
-		ppi, err := m.sb.AddPiece(ctx, size, sectorID, m.pledgeReader(size, uint64(1)), existingPieceSizes)
+		ppi, err := m.sb.AddPiece(ctx, size, sectorID, m.pledgeReader(size), existingPieceSizes)
 		if err != nil {
 			return nil, xerrors.Errorf("add piece: %w", err)
 		}
@@ -112,9 +39,8 @@ func (m *Sealing) pledgeSector(ctx context.Context, sectorID abi.SectorNumber, e
 		existingPieceSizes = append(existingPieceSizes, size)
 
 		out[i] = Piece{
-			DealID: resp.IDs[i],
+			Size:  ppi.Size,
-			Size:   abi.UnpaddedPieceSize(ppi.Size),
+			CommP: ppi.CommP[:],
-			CommP:  ppi.CommP[:],
 		}
 	}
 
@@ -141,7 +67,7 @@ func (m *Sealing) PledgeSector() error {
 			return
 		}
 
-		if err := m.newSector(context.TODO(), sid, pieces[0].DealID, pieces[0].ppi()); err != nil {
+		if err := m.newSector(sid, pieces); err != nil {
 			log.Errorf("%+v", err)
 			return
 		}

storage/sealing/sealing.go

@@ -120,20 +120,20 @@ func (m *Sealing) SealPiece(ctx context.Context, size abi.UnpaddedPieceSize, r i
 		return xerrors.Errorf("adding piece to sector: %w", err)
 	}
 
-	return m.newSector(ctx, sectorID, dealID, ppi)
+	return m.newSector(sectorID, []Piece{
+		{
+			DealID: &dealID,
+
+			Size:  ppi.Size,
+			CommP: ppi.CommP[:],
+		},
+	},)
 }
 
-func (m *Sealing) newSector(ctx context.Context, sid abi.SectorNumber, dealID abi.DealID, ppi sectorbuilder.PublicPieceInfo) error {
+func (m *Sealing) newSector(sid abi.SectorNumber, pieces []Piece) error {
 	log.Infof("Start sealing %d", sid)
-	return m.sectors.Send(sid, SectorStart{
+	return m.sectors.Send(uint64(sid), SectorStart{
 		id: sid,
-		pieces: []Piece{
+		pieces: pieces,
-			{
-				DealID: dealID,
-
-				Size:  abi.UnpaddedPieceSize(ppi.Size),
-				CommP: ppi.CommP[:],
-			},
-		},
 	})
 }

storage/sealing/states.go

@@ -99,8 +99,9 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
 	}
 
 	params := &miner.SectorPreCommitInfo{
-		Expiration:   0,
+		Expiration:   10000000, // TODO: implement
 		SectorNumber: sector.SectorID,
+		RegisteredProof: abi.RegisteredProof_StackedDRG32GiBSeal,
 
 		SealedCID:     commcid.ReplicaCommitmentV1ToCID(sector.CommR),
 		SealRandEpoch: sector.Ticket.BlockHeight,

storage/sealing/types.go

@@ -34,7 +34,7 @@ func (t *SealSeed) Equals(o *SealSeed) bool {
 }
 
 type Piece struct {
-	DealID abi.DealID
+	DealID *abi.DealID
 
 	Size  abi.UnpaddedPieceSize
 	CommP []byte
@@ -97,9 +97,12 @@ func (t *SectorInfo) pieceInfos() []sectorbuilder.PublicPieceInfo {
 }
 
 func (t *SectorInfo) deals() []abi.DealID {
-	out := make([]abi.DealID, len(t.Pieces))
+	out := make([]abi.DealID, 0, len(t.Pieces))
-	for i, piece := range t.Pieces {
+	for _, piece := range t.Pieces {
-		out[i] = piece.DealID
+		if piece.DealID == nil {
+			continue
+		}
+		out = append(out, *piece.DealID)
 	}
 	return out
 }

storage/sealing/utils.go

@@ -1,15 +1,8 @@
 package sealing
 
 import (
-	"io"
-	"math/bits"
-	"math/rand"
-	"sync"
-
 	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/hashicorp/go-multierror"
+	"math/bits"
-
-	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 )
 
 func fillersFromRem(in abi.UnpaddedPieceSize) ([]abi.UnpaddedPieceSize, error) {
@@ -48,44 +41,6 @@ func fillersFromRem(in abi.UnpaddedPieceSize) ([]abi.UnpaddedPieceSize, error) {
 	return out, nil
 }
 
-func (m *Sealing) fastPledgeCommitment(size abi.UnpaddedPieceSize, parts uint64) (commP [sectorbuilder.CommLen]byte, err error) {
-	parts = 1 << bits.Len64(parts) // round down to nearest power of 2
-	if uint64(size)/parts < 127 {
-		parts = uint64(size) / 127
-	}
-
-	piece := abi.PaddedPieceSize(uint64(size.Padded()) / parts).Unpadded()
-	out := make([]sectorbuilder.PublicPieceInfo, parts)
-	var lk sync.Mutex
-
-	var wg sync.WaitGroup
-	wg.Add(int(parts))
-	for i := uint64(0); i < parts; i++ {
-		go func(i uint64) {
-			defer wg.Done()
-
-			commP, perr := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42+int64(i))), int64(piece)), piece)
-
-			lk.Lock()
-			if perr != nil {
-				err = multierror.Append(err, perr)
-			}
-			out[i] = sectorbuilder.PublicPieceInfo{
-				Size:  piece,
-				CommP: commP,
-			}
-			lk.Unlock()
-		}(i)
-	}
-	wg.Wait()
-
-	if err != nil {
-		return [32]byte{}, err
-	}
-
-	return sectorbuilder.GenerateDataCommitment(m.sb.SectorSize(), out)
-}
-
 func (m *Sealing) ListSectors() ([]SectorInfo, error) {
 	var sectors []SectorInfo
 	if err := m.sectors.List(&sectors); err != nil {
@@ -96,6 +51,6 @@ func (m *Sealing) ListSectors() ([]SectorInfo, error) {
 
 func (m *Sealing) GetSectorInfo(sid abi.SectorNumber) (SectorInfo, error) {
 	var out SectorInfo
-	err := m.sectors.Get(sid).Get(&out)
+	err := m.sectors.Get(uint64(sid)).Get(&out)
 	return out, err
 }