Merge remote-tracking branch 'origin/testnet-staging' into feat/cleanup-sectorstate-structs

.circleci/config.yml

@@ -76,7 +76,7 @@ jobs:
       - store_artifacts:
           path: lotus-storage-miner
 
-  test:
+  test: &test
     description: |
       Run tests with gotestsum.
     parameters:
@@ -154,6 +154,9 @@ jobs:
             - "~/go/src/github.com"
             - "~/go/src/golang.org"
 
+  test-short: 
+    <<: *test
+
   build_macos:
     description: build darwin lotus binary
     macos:
@@ -254,8 +257,9 @@ workflows:
     jobs:
       - lint-changes:
           args: "--new-from-rev origin/master"
-      - test:
+      - test
-          codecov-upload: true
+      - test-short:
+          go-test-flags: "--timeout 10m --short"
       - mod-tidy-check
       - build-all
       - build_macos:

api/api_common.go

@@ -10,6 +10,8 @@ import (
 	"github.com/filecoin-project/lotus/build"
 )
 
+type Permission = string
+
 type Common interface {
 	// Auth
 	AuthVerify(ctx context.Context, token string) ([]Permission, error)

api/api_storage.go

@@ -29,6 +29,10 @@ const (
 	CommitFailed
 
 	FailedUnrecoverable
+
+	Faulty        // sector is corrupted or gone for some reason
+	FaultReported // sector has been declared as a fault on chain
+	FaultedFinal  // fault declared on chain
 )
 
 var SectorStates = []string{
@@ -39,6 +43,7 @@ var SectorStates = []string{
 	PreCommitting:        "PreCommitting",
 	PreCommitted:         "PreCommitted",
 	Committing:           "Committing",
+	CommitWait:           "CommitWait",
 	Proving:              "Proving",
 
 	SealFailed:       "SealFailed",
@@ -47,6 +52,10 @@ var SectorStates = []string{
 	CommitFailed:     "CommitFailed",
 
 	FailedUnrecoverable: "FailedUnrecoverable",
+
+	Faulty:        "Faulty",
+	FaultReported: "FaultReported",
+	FaultedFinal:  "FaultedFinal",
 }
 
 // StorageMiner is a low-level interface to the Filecoin network storage miner node

api/apistruct/permissioned.go

@@ -1,50 +1,50 @@
-package api
+package apistruct
 
 import (
 	"context"
 	"reflect"
 
 	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/api"
 )
 
 type permKey int
 
 var permCtxKey permKey
 
-type Permission = string
-
 const (
 	// When changing these, update docs/API.md too
 
-	PermRead  Permission = "read" // default
+	PermRead  api.Permission = "read" // default
-	PermWrite Permission = "write"
+	PermWrite api.Permission = "write"
-	PermSign  Permission = "sign"  // Use wallet keys for signing
+	PermSign  api.Permission = "sign"  // Use wallet keys for signing
-	PermAdmin Permission = "admin" // Manage permissions
+	PermAdmin api.Permission = "admin" // Manage permissions
 )
 
-var AllPermissions = []Permission{PermRead, PermWrite, PermSign, PermAdmin}
+var AllPermissions = []api.Permission{PermRead, PermWrite, PermSign, PermAdmin}
-var defaultPerms = []Permission{PermRead}
+var defaultPerms = []api.Permission{PermRead}
 
-func WithPerm(ctx context.Context, perms []Permission) context.Context {
+func WithPerm(ctx context.Context, perms []api.Permission) context.Context {
 	return context.WithValue(ctx, permCtxKey, perms)
 }
 
-func PermissionedStorMinerAPI(a StorageMiner) StorageMiner {
+func PermissionedStorMinerAPI(a api.StorageMiner) api.StorageMiner {
 	var out StorageMinerStruct
 	permissionedAny(a, &out.Internal)
 	permissionedAny(a, &out.CommonStruct.Internal)
 	return &out
 }
 
-func PermissionedFullAPI(a FullNode) FullNode {
+func PermissionedFullAPI(a api.FullNode) api.FullNode {
 	var out FullNodeStruct
 	permissionedAny(a, &out.Internal)
 	permissionedAny(a, &out.CommonStruct.Internal)
 	return &out
 }
 
-func HasPerm(ctx context.Context, perm Permission) bool {
+func HasPerm(ctx context.Context, perm api.Permission) bool {
-	callerPerms, ok := ctx.Value(permCtxKey).([]Permission)
+	callerPerms, ok := ctx.Value(permCtxKey).([]api.Permission)
 	if !ok {
 		callerPerms = defaultPerms
 	}
@@ -63,7 +63,7 @@ func permissionedAny(in interface{}, out interface{}) {
 
 	for f := 0; f < rint.NumField(); f++ {
 		field := rint.Type().Field(f)
-		requiredPerm := Permission(field.Tag.Get("perm"))
+		requiredPerm := api.Permission(field.Tag.Get("perm"))
 		if requiredPerm == "" {
 			panic("missing 'perm' tag on " + field.Name) // ok
 		}

api/apistruct/struct.go

@@ -1,4 +1,4 @@
-package api
+package apistruct
 
 import (
 	"context"
@@ -9,6 +9,7 @@ import (
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 
+	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/address"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -20,8 +21,8 @@ var _ = AllPermissions
 
 type CommonStruct struct {
 	Internal struct {
-		AuthVerify func(ctx context.Context, token string) ([]Permission, error) `perm:"read"`
+		AuthVerify func(ctx context.Context, token string) ([]api.Permission, error) `perm:"read"`
-		AuthNew    func(ctx context.Context, perms []Permission) ([]byte, error) `perm:"admin"`
+		AuthNew    func(ctx context.Context, perms []api.Permission) ([]byte, error) `perm:"admin"`
 
 		NetConnectedness func(context.Context, peer.ID) (network.Connectedness, error) `perm:"read"`
 		NetPeers         func(context.Context) ([]peer.AddrInfo, error)                `perm:"read"`
@@ -29,8 +30,8 @@ type CommonStruct struct {
 		NetAddrsListen   func(context.Context) (peer.AddrInfo, error)                  `perm:"read"`
 		NetDisconnect    func(context.Context, peer.ID) error                          `perm:"write"`
 
-		ID      func(context.Context) (peer.ID, error) `perm:"read"`
+		ID      func(context.Context) (peer.ID, error)     `perm:"read"`
-		Version func(context.Context) (Version, error) `perm:"read"`
+		Version func(context.Context) (api.Version, error) `perm:"read"`
 	}
 }
 
@@ -44,16 +45,16 @@ type FullNodeStruct struct {
 		ChainGetRandomness     func(context.Context, types.TipSetKey, int64) ([]byte, error)       `perm:"read"`
 		ChainGetBlock          func(context.Context, cid.Cid) (*types.BlockHeader, error)          `perm:"read"`
 		ChainGetTipSet         func(context.Context, types.TipSetKey) (*types.TipSet, error)       `perm:"read"`
-		ChainGetBlockMessages  func(context.Context, cid.Cid) (*BlockMessages, error)              `perm:"read"`
+		ChainGetBlockMessages  func(context.Context, cid.Cid) (*api.BlockMessages, error)          `perm:"read"`
 		ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error)     `perm:"read"`
-		ChainGetParentMessages func(context.Context, cid.Cid) ([]Message, error)                   `perm:"read"`
+		ChainGetParentMessages func(context.Context, cid.Cid) ([]api.Message, error)               `perm:"read"`
 		ChainGetTipSetByHeight func(context.Context, uint64, *types.TipSet) (*types.TipSet, error) `perm:"read"`
 		ChainReadObj           func(context.Context, cid.Cid) ([]byte, error)                      `perm:"read"`
 		ChainSetHead           func(context.Context, *types.TipSet) error                          `perm:"admin"`
 		ChainGetGenesis        func(context.Context) (*types.TipSet, error)                        `perm:"read"`
 		ChainTipSetWeight      func(context.Context, *types.TipSet) (types.BigInt, error)          `perm:"read"`
 
-		SyncState          func(context.Context) (*SyncState, error)                    `perm:"read"`
+		SyncState          func(context.Context) (*api.SyncState, error)                `perm:"read"`
 		SyncSubmitBlock    func(ctx context.Context, blk *types.BlockMsg) error         `perm:"write"`
 		SyncIncomingBlocks func(ctx context.Context) (<-chan *types.BlockHeader, error) `perm:"read"`
 
@@ -61,7 +62,7 @@ type FullNodeStruct struct {
 		MpoolPush        func(context.Context, *types.SignedMessage) error                    `perm:"write"`
 		MpoolPushMessage func(context.Context, *types.Message) (*types.SignedMessage, error)  `perm:"sign"`
 		MpoolGetNonce    func(context.Context, address.Address) (uint64, error)               `perm:"read"`
-		MpoolSub         func(context.Context) (<-chan MpoolUpdate, error)                    `perm:"read"`
+		MpoolSub         func(context.Context) (<-chan api.MpoolUpdate, error)                `perm:"read"`
 
 		MinerCreateBlock func(context.Context, address.Address, *types.TipSet, *types.Ticket, *types.EPostProof, []*types.SignedMessage, uint64, uint64) (*types.BlockMsg, error) `perm:"write"`
 
@@ -77,28 +78,28 @@ type FullNodeStruct struct {
 		WalletImport         func(context.Context, *types.KeyInfo) (address.Address, error)                       `perm:"admin"`
 
 		ClientImport      func(ctx context.Context, path string) (cid.Cid, error)                                                                     `perm:"admin"`
-		ClientListImports func(ctx context.Context) ([]Import, error)                                                                                 `perm:"write"`
+		ClientListImports func(ctx context.Context) ([]api.Import, error)                                                                             `perm:"write"`
 		ClientHasLocal    func(ctx context.Context, root cid.Cid) (bool, error)                                                                       `perm:"write"`
-		ClientFindData    func(ctx context.Context, root cid.Cid) ([]QueryOffer, error)                                                               `perm:"read"`
+		ClientFindData    func(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error)                                                           `perm:"read"`
 		ClientStartDeal   func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"`
-		ClientGetDealInfo func(context.Context, cid.Cid) (*DealInfo, error)                                                                           `perm:"read"`
+		ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error)                                                                       `perm:"read"`
-		ClientListDeals   func(ctx context.Context) ([]DealInfo, error)                                                                               `perm:"write"`
+		ClientListDeals   func(ctx context.Context) ([]api.DealInfo, error)                                                                           `perm:"write"`
-		ClientRetrieve    func(ctx context.Context, order RetrievalOrder, path string) error                                                          `perm:"admin"`
+		ClientRetrieve    func(ctx context.Context, order api.RetrievalOrder, path string) error                                                      `perm:"admin"`
 		ClientQueryAsk    func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error)                                `perm:"read"`
 
-		StateMinerSectors             func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error)               `perm:"read"`
+		StateMinerSectors             func(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error)           `perm:"read"`
-		StateMinerProvingSet          func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error)               `perm:"read"`
+		StateMinerProvingSet          func(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error)           `perm:"read"`
-		StateMinerPower               func(context.Context, address.Address, *types.TipSet) (MinerPower, error)                       `perm:"read"`
+		StateMinerPower               func(context.Context, address.Address, *types.TipSet) (api.MinerPower, error)                   `perm:"read"`
 		StateMinerWorker              func(context.Context, address.Address, *types.TipSet) (address.Address, error)                  `perm:"read"`
 		StateMinerPeerID              func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error)                 `perm:"read"`
 		StateMinerElectionPeriodStart func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error)              `perm:"read"`
 		StateMinerSectorSize          func(context.Context, address.Address, *types.TipSet) (uint64, error)                           `perm:"read"`
 		StateCall                     func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error)             `perm:"read"`
-		StateReplay                   func(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error)                           `perm:"read"`
+		StateReplay                   func(context.Context, *types.TipSet, cid.Cid) (*api.ReplayResults, error)                       `perm:"read"`
 		StateGetActor                 func(context.Context, address.Address, *types.TipSet) (*types.Actor, error)                     `perm:"read"`
-		StateReadState                func(context.Context, *types.Actor, *types.TipSet) (*ActorState, error)                         `perm:"read"`
+		StateReadState                func(context.Context, *types.Actor, *types.TipSet) (*api.ActorState, error)                     `perm:"read"`
 		StatePledgeCollateral         func(context.Context, *types.TipSet) (types.BigInt, error)                                      `perm:"read"`
-		StateWaitMsg                  func(context.Context, cid.Cid) (*MsgWait, error)                                                `perm:"read"`
+		StateWaitMsg                  func(context.Context, cid.Cid) (*api.MsgWait, error)                                            `perm:"read"`
 		StateListMiners               func(context.Context, *types.TipSet) ([]address.Address, error)                                 `perm:"read"`
 		StateListActors               func(context.Context, *types.TipSet) ([]address.Address, error)                                 `perm:"read"`
 		StateMarketBalance            func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"`
@@ -111,19 +112,19 @@ type FullNodeStruct struct {
 
 		MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"`
 
-		PaychGet                   func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error)      `perm:"sign"`
+		PaychGet                   func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error)   `perm:"sign"`
-		PaychList                  func(context.Context) ([]address.Address, error)                                                         `perm:"read"`
+		PaychList                  func(context.Context) ([]address.Address, error)                                                          `perm:"read"`
-		PaychStatus                func(context.Context, address.Address) (*PaychStatus, error)                                             `perm:"read"`
+		PaychStatus                func(context.Context, address.Address) (*api.PaychStatus, error)                                          `perm:"read"`
-		PaychClose                 func(context.Context, address.Address) (cid.Cid, error)                                                  `perm:"sign"`
+		PaychClose                 func(context.Context, address.Address) (cid.Cid, error)                                                   `perm:"sign"`
-		PaychAllocateLane          func(context.Context, address.Address) (uint64, error)                                                   `perm:"sign"`
+		PaychAllocateLane          func(context.Context, address.Address) (uint64, error)                                                    `perm:"sign"`
-		PaychNewPayment            func(ctx context.Context, from, to address.Address, vouchers []VoucherSpec) (*PaymentInfo, error)        `perm:"sign"`
+		PaychNewPayment            func(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) `perm:"sign"`
-		PaychVoucherCheck          func(context.Context, *types.SignedVoucher) error                                                        `perm:"read"`
+		PaychVoucherCheck          func(context.Context, *types.SignedVoucher) error                                                         `perm:"read"`
-		PaychVoucherCheckValid     func(context.Context, address.Address, *types.SignedVoucher) error                                       `perm:"read"`
+		PaychVoucherCheckValid     func(context.Context, address.Address, *types.SignedVoucher) error                                        `perm:"read"`
-		PaychVoucherCheckSpendable func(context.Context, address.Address, *types.SignedVoucher, []byte, []byte) (bool, error)               `perm:"read"`
+		PaychVoucherCheckSpendable func(context.Context, address.Address, *types.SignedVoucher, []byte, []byte) (bool, error)                `perm:"read"`
-		PaychVoucherAdd            func(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) `perm:"write"`
+		PaychVoucherAdd            func(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error)  `perm:"write"`
 		PaychVoucherCreate         func(context.Context, address.Address, types.BigInt, uint64) (*types.SignedVoucher, error)               `perm:"sign"`
 		PaychVoucherList           func(context.Context, address.Address) ([]*types.SignedVoucher, error)                                   `perm:"write"`
-		PaychVoucherSubmit         func(context.Context, address.Address, *types.SignedVoucher) (cid.Cid, error)                            `perm:"sign"`
+		PaychVoucherSubmit         func(context.Context, address.Address, *types.SignedVoucher) (cid.Cid, error)                             `perm:"sign"`
 	}
 }
 
@@ -136,10 +137,10 @@ type StorageMinerStruct struct {
 
 		PledgeSector func(context.Context) error `perm:"write"`
 
-		SectorsStatus func(context.Context, uint64) (SectorInfo, error)     `perm:"read"`
+		SectorsStatus func(context.Context, uint64) (api.SectorInfo, error)     `perm:"read"`
-		SectorsList   func(context.Context) ([]uint64, error)               `perm:"read"`
+		SectorsList   func(context.Context) ([]uint64, error)                   `perm:"read"`
-		SectorsRefs   func(context.Context) (map[string][]SealedRef, error) `perm:"read"`
+		SectorsRefs   func(context.Context) (map[string][]api.SealedRef, error) `perm:"read"`
-		SectorsUpdate func(context.Context, uint64, SectorState) error      `perm:"write"`
+		SectorsUpdate func(context.Context, uint64, api.SectorState) error      `perm:"write"`
 
 		WorkerStats func(context.Context) (sectorbuilder.WorkerStats, error) `perm:"read"`
 
@@ -148,11 +149,11 @@ type StorageMinerStruct struct {
 	}
 }
 
-func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]Permission, error) {
+func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]api.Permission, error) {
 	return c.Internal.AuthVerify(ctx, token)
 }
 
-func (c *CommonStruct) AuthNew(ctx context.Context, perms []Permission) ([]byte, error) {
+func (c *CommonStruct) AuthNew(ctx context.Context, perms []api.Permission) ([]byte, error) {
 	return c.Internal.AuthNew(ctx, perms)
 }
 
@@ -182,11 +183,11 @@ func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
 }
 
 // Version implements API.Version
-func (c *CommonStruct) Version(ctx context.Context) (Version, error) {
+func (c *CommonStruct) Version(ctx context.Context) (api.Version, error) {
 	return c.Internal.Version(ctx)
 }
 
-func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]Import, error) {
+func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]api.Import, error) {
 	return c.Internal.ClientListImports(ctx)
 }
 
@@ -198,22 +199,22 @@ func (c *FullNodeStruct) ClientHasLocal(ctx context.Context, root cid.Cid) (bool
 	return c.Internal.ClientHasLocal(ctx, root)
 }
 
-func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) {
+func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) {
 	return c.Internal.ClientFindData(ctx, root)
 }
 
 func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) {
 	return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration)
 }
-func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*DealInfo, error) {
+func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*api.DealInfo, error) {
 	return c.Internal.ClientGetDealInfo(ctx, deal)
 }
 
-func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]DealInfo, error) {
+func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
 	return c.Internal.ClientListDeals(ctx)
 }
 
-func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error {
+func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path string) error {
 	return c.Internal.ClientRetrieve(ctx, order, path)
 }
 
@@ -233,7 +234,7 @@ func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Messag
 	return c.Internal.MpoolPushMessage(ctx, msg)
 }
 
-func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan MpoolUpdate, error) {
+func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan api.MpoolUpdate, error) {
 	return c.Internal.MpoolSub(ctx)
 }
 
@@ -305,7 +306,7 @@ func (c *FullNodeStruct) ChainGetTipSet(ctx context.Context, key types.TipSetKey
 	return c.Internal.ChainGetTipSet(ctx, key)
 }
 
-func (c *FullNodeStruct) ChainGetBlockMessages(ctx context.Context, b cid.Cid) (*BlockMessages, error) {
+func (c *FullNodeStruct) ChainGetBlockMessages(ctx context.Context, b cid.Cid) (*api.BlockMessages, error) {
 	return c.Internal.ChainGetBlockMessages(ctx, b)
 }
 
@@ -313,7 +314,7 @@ func (c *FullNodeStruct) ChainGetParentReceipts(ctx context.Context, b cid.Cid)
 	return c.Internal.ChainGetParentReceipts(ctx, b)
 }
 
-func (c *FullNodeStruct) ChainGetParentMessages(ctx context.Context, b cid.Cid) ([]Message, error) {
+func (c *FullNodeStruct) ChainGetParentMessages(ctx context.Context, b cid.Cid) ([]api.Message, error) {
 	return c.Internal.ChainGetParentMessages(ctx, b)
 }
 
@@ -337,7 +338,7 @@ func (c *FullNodeStruct) ChainTipSetWeight(ctx context.Context, ts *types.TipSet
 	return c.Internal.ChainTipSetWeight(ctx, ts)
 }
 
-func (c *FullNodeStruct) SyncState(ctx context.Context) (*SyncState, error) {
+func (c *FullNodeStruct) SyncState(ctx context.Context) (*api.SyncState, error) {
 	return c.Internal.SyncState(ctx)
 }
 
@@ -349,15 +350,15 @@ func (c *FullNodeStruct) SyncIncomingBlocks(ctx context.Context) (<-chan *types.
 	return c.Internal.SyncIncomingBlocks(ctx)
 }
 
-func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) {
+func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) {
 	return c.Internal.StateMinerSectors(ctx, addr, ts)
 }
 
-func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) {
+func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) {
 	return c.Internal.StateMinerProvingSet(ctx, addr, ts)
 }
 
-func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, ts *types.TipSet) (MinerPower, error) {
+func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, ts *types.TipSet) (api.MinerPower, error) {
 	return c.Internal.StateMinerPower(ctx, a, ts)
 }
 
@@ -381,7 +382,7 @@ func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, ts *
 	return c.Internal.StateCall(ctx, msg, ts)
 }
 
-func (c *FullNodeStruct) StateReplay(ctx context.Context, ts *types.TipSet, mc cid.Cid) (*ReplayResults, error) {
+func (c *FullNodeStruct) StateReplay(ctx context.Context, ts *types.TipSet, mc cid.Cid) (*api.ReplayResults, error) {
 	return c.Internal.StateReplay(ctx, ts, mc)
 }
 
@@ -389,7 +390,7 @@ func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Addres
 	return c.Internal.StateGetActor(ctx, actor, ts)
 }
 
-func (c *FullNodeStruct) StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*ActorState, error) {
+func (c *FullNodeStruct) StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*api.ActorState, error) {
 	return c.Internal.StateReadState(ctx, act, ts)
 }
 
@@ -397,7 +398,7 @@ func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, ts *types.Ti
 	return c.Internal.StatePledgeCollateral(ctx, ts)
 }
 
-func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid) (*MsgWait, error) {
+func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid) (*api.MsgWait, error) {
 	return c.Internal.StateWaitMsg(ctx, msgc)
 }
 func (c *FullNodeStruct) StateListMiners(ctx context.Context, ts *types.TipSet) ([]address.Address, error) {
@@ -440,7 +441,7 @@ func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr address
 	return c.Internal.MarketEnsureAvailable(ctx, addr, amt)
 }
 
-func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) {
+func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error) {
 	return c.Internal.PaychGet(ctx, from, to, ensureFunds)
 }
 
@@ -448,7 +449,7 @@ func (c *FullNodeStruct) PaychList(ctx context.Context) ([]address.Address, erro
 	return c.Internal.PaychList(ctx)
 }
 
-func (c *FullNodeStruct) PaychStatus(ctx context.Context, pch address.Address) (*PaychStatus, error) {
+func (c *FullNodeStruct) PaychStatus(ctx context.Context, pch address.Address) (*api.PaychStatus, error) {
 	return c.Internal.PaychStatus(ctx, pch)
 }
 
@@ -480,7 +481,7 @@ func (c *FullNodeStruct) PaychAllocateLane(ctx context.Context, ch address.Addre
 	return c.Internal.PaychAllocateLane(ctx, ch)
 }
 
-func (c *FullNodeStruct) PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []VoucherSpec) (*PaymentInfo, error) {
+func (c *FullNodeStruct) PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) {
 	return c.Internal.PaychNewPayment(ctx, from, to, vouchers)
 }
 
@@ -501,7 +502,7 @@ func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error {
 }
 
 // Get the status of a given sector by ID
-func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (SectorInfo, error) {
+func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (api.SectorInfo, error) {
 	return c.Internal.SectorsStatus(ctx, sid)
 }
 
@@ -510,11 +511,11 @@ func (c *StorageMinerStruct) SectorsList(ctx context.Context) ([]uint64, error)
 	return c.Internal.SectorsList(ctx)
 }
 
-func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]SealedRef, error) {
+func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]api.SealedRef, error) {
 	return c.Internal.SectorsRefs(ctx)
 }
 
-func (c *StorageMinerStruct) SectorsUpdate(ctx context.Context, id uint64, state SectorState) error {
+func (c *StorageMinerStruct) SectorsUpdate(ctx context.Context, id uint64, state api.SectorState) error {
 	return c.Internal.SectorsUpdate(ctx, id, state)
 }
 
@@ -530,6 +531,6 @@ func (c *StorageMinerStruct) WorkerDone(ctx context.Context, task uint64, res se
 	return c.Internal.WorkerDone(ctx, task, res)
 }
 
-var _ Common = &CommonStruct{}
+var _ api.Common = &CommonStruct{}
-var _ FullNode = &FullNodeStruct{}
+var _ api.FullNode = &FullNodeStruct{}
-var _ StorageMiner = &StorageMinerStruct{}
+var _ api.StorageMiner = &StorageMinerStruct{}

api/client/client.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"github.com/filecoin-project/lotus/api/apistruct"
 	"net/http"
 
 	"github.com/filecoin-project/lotus/api"
@@ -9,7 +10,7 @@ import (
 
 // NewCommonRPC creates a new http jsonrpc client.
 func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) {
-	var res api.CommonStruct
+	var res apistruct.CommonStruct
 	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.Internal,
@@ -20,7 +21,7 @@ func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.C
 
 // NewFullNodeRPC creates a new http jsonrpc client.
 func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) {
-	var res api.FullNodeStruct
+	var res apistruct.FullNodeStruct
 	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,
@@ -32,7 +33,7 @@ func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonr
 
 // NewStorageMinerRPC creates a new http jsonrpc client for storage miner
 func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) {
-	var res api.StorageMinerStruct
+	var res apistruct.StorageMinerStruct
 	closer, err := jsonrpc.NewMergeClient(addr, "Filecoin",
 		[]interface{}{
 			&res.CommonStruct.Internal,

chain/actors/actor_miner.go

@@ -53,14 +53,10 @@ type StorageMinerActorState struct {
 	// Contains mostly static info about this miner
 	Info cid.Cid
 
-	// Faulty sectors reported since last SubmitPost,
+	// Faulty sectors reported since last SubmitPost
-	// up to the current proving period's challenge time.
+	FaultSet types.BitField
-	CurrentFaultSet types.BitField
 
-	// Faults submitted after the current proving period's challenge time,
+	LastFaultSubmission uint64
-	// but before the PoSt for that period is submitted.
-	// These become the currentFaultSet when a PoSt is submitted.
-	NextFaultSet types.BitField
 
 	// Amount of power this miner has.
 	Power types.BigInt
@@ -340,7 +336,7 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC
 		return nil, aerrors.Wrapf(err, "failed to compute data commitment (sector %d, deals: %v)", params.SectorID, params.DealIDs)
 	}
 
-	if ok, err := ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil {
+	if ok, err := vmctx.Sys().ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil {
 		return nil, err
 	} else if !ok {
 		return nil, aerrors.Newf(2, "porep proof was invalid (t:%x; s:%x(%d); p:%s)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, truncateHexPrint(params.Proof))
@@ -463,8 +459,17 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
 		return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
 	}
 
+	faults, nerr := self.FaultSet.AllMap()
+	if nerr != nil {
+		return nil, aerrors.Absorb(err, 5, "RLE+ invalid")
+	}
+
 	var sectorInfos []ffi.PublicSectorInfo
 	if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error {
+		if faults[id] {
+			return nil
+		}
+
 		var comms [][]byte
 		if err := cbor.DecodeInto(v.Raw, &comms); err != nil {
 			return xerrors.New("could not decode comms")
@@ -485,12 +490,6 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
 		return nil, aerrors.Absorb(err, 3, "could not decode sectorset")
 	}
 
-	faults, nerr := self.CurrentFaultSet.All()
-	if nerr != nil {
-		return nil, aerrors.Absorb(err, 5, "RLE+ invalid")
-	}
-	_ = faults
-
 	proverID := vmctx.Message().To // TODO: normalize to ID address
 
 	var candidates []sectorbuilder.EPostCandidate
@@ -598,6 +597,25 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID
 	return true, comms[0], comms[1], nil
 }
 
+func RemoveFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, ids []uint64) (cid.Cid, aerrors.ActorError) {
+
+	ssr, err := amt.LoadAMT(types.WrapStorage(s), ss)
+	if err != nil {
+		return cid.Undef, aerrors.HandleExternalError(err, "could not load sector set node")
+	}
+
+	if err := ssr.BatchDelete(ids); err != nil {
+		return cid.Undef, aerrors.HandleExternalError(err, "failed to delete from sector set")
+	}
+
+	ncid, err := ssr.Flush()
+	if err != nil {
+		return cid.Undef, aerrors.HandleExternalError(err, "failed to flush sector set")
+	}
+
+	return ncid, nil
+}
+
 func ValidatePoRep(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, ActorError) {
 	_, span := trace.StartSpan(ctx, "ValidatePoRep")
 	defer span.End()
@@ -787,34 +805,28 @@ type DeclareFaultsParams struct {
 }
 
 func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) {
-	/*
+	oldstate, self, aerr := loadState(vmctx)
-		oldstate, self, aerr := loadState(vmctx)
+	if aerr != nil {
-		if aerr != nil {
+		return nil, aerr
-			return nil, aerr
+	}
-		}
 
-		challengeHeight := self.ProvingPeriodEnd - build.PoStChallangeTime
+	nfaults, err := types.MergeBitFields(params.Faults, self.FaultSet)
+	if err != nil {
+		return nil, aerrors.Absorb(err, 1, "failed to merge bitfields")
+	}
 
-		if vmctx.BlockHeight() < challengeHeight {
+	self.FaultSet = nfaults
-			// TODO: optimized bitfield methods
-			for _, v := range params.Faults.All() {
-				self.CurrentFaultSet.Set(v)
-			}
-		} else {
-			for _, v := range params.Faults.All() {
-				self.NextFaultSet.Set(v)
-			}
-		}
 
-		nstate, err := vmctx.Storage().Put(self)
+	self.LastFaultSubmission = vmctx.BlockHeight()
-		if err != nil {
+
-			return nil, err
+	nstate, aerr := vmctx.Storage().Put(self)
-		}
+	if err != nil {
-		if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
+		return nil, aerr
-			return nil, err
+	}
-		}
+	if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
+		return nil, err
+	}
 
-	*/
 	return nil, nil
 }
 
@@ -906,10 +918,12 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerro
 		return aerrors.HandleExternalError(nerr, "failed to load proving set")
 	}
 
-	self.CurrentFaultSet = self.NextFaultSet
+	faults, nerr := self.FaultSet.All()
-	self.NextFaultSet = types.NewBitField()
+	if nerr != nil {
+		return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)")
+	}
 
-	faults := []uint64{} // TODO
+	self.FaultSet = types.NewBitField()
 
 	oldPower := self.Power
 	self.Power = types.BigMul(types.NewInt(pss.Count-uint64(len(faults))),
@@ -941,7 +955,13 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerro
 		return err
 	}
 
-	self.ProvingSet = self.Sectors
+	ncid, err := RemoveFromSectorSet(vmctx.Context(), vmctx.Storage(), self.Sectors, faults)
+	if err != nil {
+		return err
+	}
+
+	self.Sectors = ncid
+	self.ProvingSet = ncid
 	self.ElectionPeriodStart = vmctx.BlockHeight()
 	return nil
 }

chain/actors/actor_miner_test.go

@@ -0,0 +1,157 @@
+package actors_test
+
+import (
+	"bytes"
+	"context"
+	"math/rand"
+	"testing"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/aerrors"
+	"github.com/filecoin-project/lotus/chain/address"
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/lib/sectorbuilder"
+	hamt "github.com/ipfs/go-hamt-ipld"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	cbg "github.com/whyrusleeping/cbor-gen"
+)
+
+func TestMinerCommitSectors(t *testing.T) {
+	var worker, client address.Address
+	var minerAddr address.Address
+	opts := []HarnessOpt{
+		HarnessAddr(&worker, 1000000),
+		HarnessAddr(&client, 1000000),
+		HarnessActor(&minerAddr, &worker, actors.StorageMinerCodeCid,
+			func() cbg.CBORMarshaler {
+				return &actors.StorageMinerConstructorParams{
+					Owner:      worker,
+					Worker:     worker,
+					SectorSize: 1024,
+					PeerID:     "fakepeerid",
+				}
+			}),
+	}
+
+	h := NewHarness(t, opts...)
+	h.vm.Syscalls.ValidatePoRep = func(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, aerrors.ActorError) {
+		// all proofs are valid
+		return true, nil
+	}
+
+	ret, _ := h.SendFunds(t, worker, minerAddr, types.NewInt(100000))
+	ApplyOK(t, ret)
+
+	ret, _ = h.InvokeWithValue(t, client, actors.StorageMarketAddress, actors.SMAMethods.AddBalance, types.NewInt(2000), nil)
+	ApplyOK(t, ret)
+
+	addSectorToMiner(h, t, minerAddr, worker, client, 1)
+
+	assertSectorIDs(h, t, minerAddr, []uint64{1})
+}
+
+func addSectorToMiner(h *Harness, t *testing.T, minerAddr, worker, client address.Address, sid uint64) {
+	t.Helper()
+	s := sectorbuilder.UserBytesForSectorSize(1024)
+	deal := h.makeFakeDeal(t, minerAddr, worker, client, s)
+	ret, _ := h.Invoke(t, worker, actors.StorageMarketAddress, actors.SMAMethods.PublishStorageDeals,
+		&actors.PublishStorageDealsParams{
+			Deals: []actors.StorageDealProposal{*deal},
+		})
+	ApplyOK(t, ret)
+	var dealIds actors.PublishStorageDealResponse
+	if err := dealIds.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil {
+		t.Fatal(err)
+	}
+
+	dealid := dealIds.DealIDs[0]
+
+	ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.PreCommitSector,
+		&actors.SectorPreCommitInfo{
+			SectorNumber: sid,
+			CommR:        []byte("cats"),
+			SealEpoch:    10,
+			DealIDs:      []uint64{dealid},
+		})
+	ApplyOK(t, ret)
+
+	h.BlockHeight += 100
+	ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.ProveCommitSector,
+		&actors.SectorProveCommitInfo{
+			Proof:    []byte("prooofy"),
+			SectorID: sid,
+			DealIDs:  []uint64{dealid}, // TODO: weird that i have to pass this again
+		})
+	ApplyOK(t, ret)
+}
+
+func assertSectorIDs(h *Harness, t *testing.T, maddr address.Address, ids []uint64) {
+	t.Helper()
+	sectors, err := getMinerSectorSet(context.TODO(), h.vm.StateTree(), h.bs, maddr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(sectors) != len(ids) {
+		t.Fatal("miner has wrong number of sectors in their sector set")
+	}
+
+	all := make(map[uint64]bool)
+	for _, s := range sectors {
+		all[s.SectorID] = true
+	}
+
+	for _, id := range ids {
+		if !all[id] {
+			t.Fatal("expected to find sector ID: ", id)
+		}
+	}
+}
+
+func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) {
+	mact, err := st.GetActor(maddr)
+	if err != nil {
+		return nil, err
+	}
+
+	cst := hamt.CSTFromBstore(bs)
+
+	var mstate actors.StorageMinerActorState
+	if err := cst.Get(ctx, mact.Head, &mstate); err != nil {
+		return nil, err
+	}
+
+	return stmgr.LoadSectorsFromSet(ctx, bs, mstate.Sectors)
+}
+
+func (h *Harness) makeFakeDeal(t *testing.T, miner, worker, client address.Address, size uint64) *actors.StorageDealProposal {
+	data := make([]byte, size)
+	rand.Read(data)
+	commP, err := sectorbuilder.GeneratePieceCommitment(bytes.NewReader(data), size)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	prop := actors.StorageDealProposal{
+		PieceRef:  commP[:],
+		PieceSize: size,
+		//PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid
+
+		Client:   client,
+		Provider: miner,
+
+		ProposalExpiration: 10000,
+		Duration:           150,
+
+		StoragePricePerEpoch: types.NewInt(1),
+		StorageCollateral:    types.NewInt(0),
+	}
+
+	if err := api.SignWith(context.TODO(), h.w.Sign, client, &prop); err != nil {
+		t.Fatal(err)
+	}
+
+	return &prop
+}

chain/actors/actor_paych_test.go

@@ -83,7 +83,7 @@ func TestPaychUpdate(t *testing.T) {
 	ApplyOK(t, ret)
 
 	// now we have to 'wait' for the chain to advance.
-	h.vm.SetBlockHeight(1000)
+	h.BlockHeight = 1000
 
 	ret, _ = h.Invoke(t, targetAddr, pch, actors.PCAMethods.Collect, nil)
 	ApplyOK(t, ret)

chain/actors/actor_storagemarket.go

@@ -3,6 +3,8 @@ package actors
 import (
 	"bytes"
 	"context"
+	"sort"
+
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-amt-ipld"
@@ -73,9 +75,8 @@ const (
 )
 
 type StorageDealProposal struct {
-	PieceRef           []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea
+	PieceRef  []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea
-	PieceSize          uint64
+	PieceSize uint64
-	PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid
 
 	Client   address.Address
 	Provider address.Address
@@ -131,36 +132,19 @@ func (sdp *StorageDealProposal) Verify() error {
 	return sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes())
 }
 
-func (d *StorageDeal) Sign(ctx context.Context, sign SignFunc) error {
-	var buf bytes.Buffer
-	if err := d.Proposal.MarshalCBOR(&buf); err != nil {
-		return err
-	}
-	sig, err := sign(ctx, buf.Bytes())
-	if err != nil {
-		return err
-	}
-	d.CounterSignature = sig
-	return nil
-}
-
-func (d *StorageDeal) Verify(proposerWorker address.Address) error {
-	var buf bytes.Buffer
-	if err := d.Proposal.MarshalCBOR(&buf); err != nil {
-		return err
-	}
-
-	return d.CounterSignature.Verify(proposerWorker, buf.Bytes())
-}
-
-type StorageDeal struct {
-	Proposal         StorageDealProposal
-	CounterSignature *types.Signature
-}
-
 type OnChainDeal struct {
-	Deal            StorageDeal
+	PieceRef  []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea
-	ActivationEpoch uint64 // 0 = inactive
+	PieceSize uint64
+
+	Client   address.Address
+	Provider address.Address
+
+	ProposalExpiration uint64
+	Duration           uint64 // TODO: spec
+
+	StoragePricePerEpoch types.BigInt
+	StorageCollateral    types.BigInt
+	ActivationEpoch      uint64 // 0 = inactive
 }
 
 type WithdrawBalanceParams struct {
@@ -245,8 +229,15 @@ func (sma StorageMarketActor) AddBalance(act *types.Actor, vmctx types.VMContext
 }
 
 func setMarketBalances(vmctx types.VMContext, nd *hamt.Node, set map[address.Address]StorageParticipantBalance) (cid.Cid, ActorError) {
-	for addr, b := range set {
+	keys := make([]address.Address, 0, len(set))
-		balance := b // to stop linter complaining
+	for k := range set {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		return bytes.Compare(keys[i].Bytes(), keys[j].Bytes()) < 0
+	})
+	for _, addr := range keys {
+		balance := set[addr]
 		if err := nd.Set(vmctx.Context(), string(addr.Bytes()), &balance); err != nil {
 			return cid.Undef, aerrors.HandleExternalError(err, "setting new balance")
 		}
@@ -297,7 +288,7 @@ func (sma StorageMarketActor) CheckLockedBalance(act *types.Actor, vmctx types.V
 */
 
 type PublishStorageDealsParams struct {
-	Deals []StorageDeal
+	Deals []StorageDealProposal
 }
 
 type PublishStorageDealResponse struct {
@@ -326,7 +317,7 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types.
 		DealIDs: make([]uint64, len(params.Deals)),
 	}
 
-	workerBytes, aerr := vmctx.Send(params.Deals[0].Proposal.Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil)
+	workerBytes, aerr := vmctx.Send(params.Deals[0].Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil)
 	if aerr != nil {
 		return nil, aerr
 	}
@@ -342,7 +333,20 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types.
 			return nil, err
 		}
 
-		err := deals.Set(self.NextDealID, &OnChainDeal{Deal: deal})
+		err := deals.Set(self.NextDealID, &OnChainDeal{
+			PieceRef:  deal.PieceRef,
+			PieceSize: deal.PieceSize,
+
+			Client:   deal.Client,
+			Provider: deal.Provider,
+
+			ProposalExpiration: deal.ProposalExpiration,
+			Duration:           deal.Duration,
+
+			StoragePricePerEpoch: deal.StoragePricePerEpoch,
+			StorageCollateral:    deal.StorageCollateral,
+			ActivationEpoch:      0,
+		})
 		if err != nil {
 			return nil, aerrors.HandleExternalError(err, "setting deal in deal AMT")
 		}
@@ -376,34 +380,28 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types.
 	return outBuf.Bytes(), nil
 }
 
-func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDeal, providerWorker address.Address) aerrors.ActorError {
+func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDealProposal, providerWorker address.Address) aerrors.ActorError {
-	if vmctx.BlockHeight() > deal.Proposal.ProposalExpiration {
+	if vmctx.BlockHeight() > deal.ProposalExpiration {
 		return aerrors.New(1, "deal proposal already expired")
 	}
 
-	if err := deal.Proposal.Verify(); err != nil {
+	if vmctx.Message().From != providerWorker {
-		return aerrors.Absorb(err, 2, "verifying proposer signature")
+		return aerrors.New(2, "Deals must be submitted by the miner worker")
 	}
 
-	err := deal.Verify(providerWorker)
+	if err := deal.Verify(); err != nil {
-	if err != nil {
+		return aerrors.Absorb(err, 3, "verifying proposer signature")
-		return aerrors.Absorb(err, 2, "verifying provider signature")
-	}
-
-	// TODO: maybe this is actually fine
-	if vmctx.Message().From != providerWorker && vmctx.Message().From != deal.Proposal.Client {
-		return aerrors.New(4, "message not sent by deal participant")
 	}
 
 	// TODO: do some caching (changes gas so needs to be in spec too)
-	b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), st.Balances, deal.Proposal.Client, providerWorker)
+	b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), st.Balances, deal.Client, providerWorker)
 	if aerr != nil {
 		return aerrors.Wrap(aerr, "getting client, and provider balances")
 	}
 	clientBalance := b[0]
 	providerBalance := b[1]
 
-	totalPrice := deal.Proposal.TotalStoragePrice()
+	totalPrice := deal.TotalStoragePrice()
 
 	if clientBalance.Available.LessThan(totalPrice) {
 		return aerrors.Newf(5, "client doesn't have enough available funds to cover storage price; %d < %d", clientBalance.Available, totalPrice)
@@ -412,17 +410,17 @@ func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDe
 	clientBalance = lockFunds(clientBalance, totalPrice)
 
 	// TODO: REVIEW: Not clear who pays for this
-	if providerBalance.Available.LessThan(deal.Proposal.StorageCollateral) {
+	if providerBalance.Available.LessThan(deal.StorageCollateral) {
-		return aerrors.Newf(6, "provider doesn't have enough available funds to cover StorageCollateral; %d < %d", providerBalance.Available, deal.Proposal.StorageCollateral)
+		return aerrors.Newf(6, "provider doesn't have enough available funds to cover StorageCollateral; %d < %d", providerBalance.Available, deal.StorageCollateral)
 	}
 
-	providerBalance = lockFunds(providerBalance, deal.Proposal.StorageCollateral)
+	providerBalance = lockFunds(providerBalance, deal.StorageCollateral)
 
 	// TODO: piece checks (e.g. size > sectorSize)?
 
 	bcid, aerr := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{
-		deal.Proposal.Client: clientBalance,
+		deal.Client:    clientBalance,
-		providerWorker:       providerBalance,
+		providerWorker: providerBalance,
 	})
 	if aerr != nil {
 		return aerr
@@ -458,11 +456,11 @@ func (sma StorageMarketActor) ActivateStorageDeals(act *types.Actor, vmctx types
 			return nil, aerrors.HandleExternalError(err, "getting deal info failed")
 		}
 
-		if vmctx.Message().From != dealInfo.Deal.Proposal.Provider {
+		if vmctx.Message().From != dealInfo.Provider {
 			return nil, aerrors.New(1, "ActivateStorageDeals can only be called by the deal provider")
 		}
 
-		if vmctx.BlockHeight() > dealInfo.Deal.Proposal.ProposalExpiration {
+		if vmctx.BlockHeight() > dealInfo.ProposalExpiration {
 			return nil, aerrors.New(2, "deal cannot be activated: proposal expired")
 		}
 
@@ -533,7 +531,7 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx
 			return nil, aerrors.HandleExternalError(err, "getting deal info failed")
 		}
 
-		if dealInfo.Deal.Proposal.Provider != vmctx.Message().From {
+		if dealInfo.Provider != vmctx.Message().From {
 			return nil, aerrors.New(3, "ProcessStorageDealsPayment can only be called by deal provider")
 		}
 
@@ -542,15 +540,15 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx
 			return nil, aerrors.New(4, "ActivationEpoch lower than block height")
 		}
 
-		if vmctx.BlockHeight() > dealInfo.ActivationEpoch+dealInfo.Deal.Proposal.Duration {
+		if vmctx.BlockHeight() > dealInfo.ActivationEpoch+dealInfo.Duration {
 			// Deal expired, miner should drop it
 			// TODO: process payment for the remainder of last proving period
 			return nil, nil
 		}
 
-		toPay := types.BigMul(dealInfo.Deal.Proposal.StoragePricePerEpoch, types.NewInt(build.SlashablePowerDelay))
+		toPay := types.BigMul(dealInfo.StoragePricePerEpoch, types.NewInt(build.SlashablePowerDelay))
 
-		b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Deal.Proposal.Client, providerWorker)
+		b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Client, providerWorker)
 		if aerr != nil {
 			return nil, aerr
 		}
@@ -561,8 +559,8 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx
 
 		// TODO: call set once
 		bcid, aerr := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{
-			dealInfo.Deal.Proposal.Client: clientBal,
+			dealInfo.Client: clientBal,
-			providerWorker:                providerBal,
+			providerWorker:  providerBal,
 		})
 		if aerr != nil {
 			return nil, aerr
@@ -625,15 +623,15 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type
 			return nil, aerrors.HandleExternalError(err, "getting deal info failed")
 		}
 
-		if dealInfo.Deal.Proposal.Provider != vmctx.Message().From {
+		if dealInfo.Provider != vmctx.Message().From {
 			return nil, aerrors.New(5, "referenced deal was not from caller")
 		}
 
 		var commP [32]byte
-		copy(commP[:], dealInfo.Deal.Proposal.PieceRef)
+		copy(commP[:], dealInfo.PieceRef)
 
 		pieces = append(pieces, sectorbuilder.PublicPieceInfo{
-			Size:  dealInfo.Deal.Proposal.PieceSize,
+			Size:  dealInfo.PieceSize,
 			CommP: commP,
 		})
 	}

chain/actors/cbor_gen.go

@@ -248,13 +248,13 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("failed to write cid field t.Info: %w", err)
 	}
 
-	// t.CurrentFaultSet (types.BitField) (struct)
+	// t.FaultSet (types.BitField) (struct)
-	if err := t.CurrentFaultSet.MarshalCBOR(w); err != nil {
+	if err := t.FaultSet.MarshalCBOR(w); err != nil {
 		return err
 	}
 
-	// t.NextFaultSet (types.BitField) (struct)
+	// t.LastFaultSubmission (uint64) (uint64)
-	if err := t.NextFaultSet.MarshalCBOR(w); err != nil {
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.LastFaultSubmission))); err != nil {
 		return err
 	}
 
@@ -384,24 +384,25 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error {
 		t.Info = c
 
 	}
-	// t.CurrentFaultSet (types.BitField) (struct)
+	// t.FaultSet (types.BitField) (struct)
 
 	{
 
-		if err := t.CurrentFaultSet.UnmarshalCBOR(br); err != nil {
+		if err := t.FaultSet.UnmarshalCBOR(br); err != nil {
 			return err
 		}
 
 	}
-	// t.NextFaultSet (types.BitField) (struct)
+	// t.LastFaultSubmission (uint64) (uint64)
-
-	{
-
-		if err := t.NextFaultSet.UnmarshalCBOR(br); err != nil {
-			return err
-		}
 
+	maj, extra, err = cbg.CborReadHeader(br)
+	if err != nil {
+		return err
 	}
+	if maj != cbg.MajUnsignedInt {
+		return fmt.Errorf("wrong type for uint64 field")
+	}
+	t.LastFaultSubmission = uint64(extra)
 	// t.Power (types.BigInt) (struct)
 
 	{
@@ -1031,6 +1032,49 @@ func (t *UpdatePeerIDParams) UnmarshalCBOR(r io.Reader) error {
 	return nil
 }
 
+func (t *DeclareFaultsParams) MarshalCBOR(w io.Writer) error {
+	if t == nil {
+		_, err := w.Write(cbg.CborNull)
+		return err
+	}
+	if _, err := w.Write([]byte{129}); err != nil {
+		return err
+	}
+
+	// t.Faults (types.BitField) (struct)
+	if err := t.Faults.MarshalCBOR(w); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *DeclareFaultsParams) UnmarshalCBOR(r io.Reader) error {
+	br := cbg.GetPeeker(r)
+
+	maj, extra, err := cbg.CborReadHeader(br)
+	if err != nil {
+		return err
+	}
+	if maj != cbg.MajArray {
+		return fmt.Errorf("cbor input should be of type array")
+	}
+
+	if extra != 1 {
+		return fmt.Errorf("cbor input had wrong number of fields")
+	}
+
+	// t.Faults (types.BitField) (struct)
+
+	{
+
+		if err := t.Faults.UnmarshalCBOR(br); err != nil {
+			return err
+		}
+
+	}
+	return nil
+}
+
 func (t *MultiSigActorState) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
@@ -3166,7 +3210,7 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{138}); err != nil {
+	if _, err := w.Write([]byte{137}); err != nil {
 		return err
 	}
 
@@ -3183,11 +3227,6 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.PieceSerialization (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSerialization))); err != nil {
-		return err
-	}
-
 	// t.Client (address.Address) (struct)
 	if err := t.Client.MarshalCBOR(w); err != nil {
 		return err
@@ -3236,7 +3275,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type array")
 	}
 
-	if extra != 10 {
+	if extra != 9 {
 		return fmt.Errorf("cbor input had wrong number of fields")
 	}
 
@@ -3267,16 +3306,6 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("wrong type for uint64 field")
 	}
 	t.PieceSize = uint64(extra)
-	// t.PieceSerialization (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.PieceSerialization = uint64(extra)
 	// t.Client (address.Address) (struct)
 
 	{
@@ -3357,75 +3386,6 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error {
 	return nil
 }
 
-func (t *StorageDeal) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{130}); err != nil {
-		return err
-	}
-
-	// t.Proposal (actors.StorageDealProposal) (struct)
-	if err := t.Proposal.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.CounterSignature (types.Signature) (struct)
-	if err := t.CounterSignature.MarshalCBOR(w); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 2 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Proposal (actors.StorageDealProposal) (struct)
-
-	{
-
-		if err := t.Proposal.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.CounterSignature (types.Signature) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-			t.CounterSignature = new(types.Signature)
-			if err := t.CounterSignature.UnmarshalCBOR(br); err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
 func (t *PublishStorageDealsParams) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
@@ -3435,7 +3395,7 @@ func (t *PublishStorageDealsParams) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.Deals ([]actors.StorageDeal) (slice)
+	// t.Deals ([]actors.StorageDealProposal) (slice)
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Deals)))); err != nil {
 		return err
 	}
@@ -3462,7 +3422,7 @@ func (t *PublishStorageDealsParams) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input had wrong number of fields")
 	}
 
-	// t.Deals ([]actors.StorageDeal) (slice)
+	// t.Deals ([]actors.StorageDealProposal) (slice)
 
 	maj, extra, err = cbg.CborReadHeader(br)
 	if err != nil {
@@ -3477,11 +3437,11 @@ func (t *PublishStorageDealsParams) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("expected cbor array")
 	}
 	if extra > 0 {
-		t.Deals = make([]StorageDeal, extra)
+		t.Deals = make([]StorageDealProposal, extra)
 	}
 	for i := 0; i < int(extra); i++ {
 
-		var v StorageDeal
+		var v StorageDealProposal
 		if err := v.UnmarshalCBOR(br); err != nil {
 			return err
 		}
@@ -3707,12 +3667,50 @@ func (t *OnChainDeal) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{130}); err != nil {
+	if _, err := w.Write([]byte{137}); err != nil {
 		return err
 	}
 
-	// t.Deal (actors.StorageDeal) (struct)
+	// t.PieceRef ([]uint8) (slice)
-	if err := t.Deal.MarshalCBOR(w); err != nil {
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PieceRef)))); err != nil {
+		return err
+	}
+	if _, err := w.Write(t.PieceRef); err != nil {
+		return err
+	}
+
+	// t.PieceSize (uint64) (uint64)
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSize))); err != nil {
+		return err
+	}
+
+	// t.Client (address.Address) (struct)
+	if err := t.Client.MarshalCBOR(w); err != nil {
+		return err
+	}
+
+	// t.Provider (address.Address) (struct)
+	if err := t.Provider.MarshalCBOR(w); err != nil {
+		return err
+	}
+
+	// t.ProposalExpiration (uint64) (uint64)
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil {
+		return err
+	}
+
+	// t.Duration (uint64) (uint64)
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil {
+		return err
+	}
+
+	// t.StoragePricePerEpoch (types.BigInt) (struct)
+	if err := t.StoragePricePerEpoch.MarshalCBOR(w); err != nil {
+		return err
+	}
+
+	// t.StorageCollateral (types.BigInt) (struct)
+	if err := t.StorageCollateral.MarshalCBOR(w); err != nil {
 		return err
 	}
 
@@ -3734,15 +3732,89 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type array")
 	}
 
-	if extra != 2 {
+	if extra != 9 {
 		return fmt.Errorf("cbor input had wrong number of fields")
 	}
 
-	// t.Deal (actors.StorageDeal) (struct)
+	// t.PieceRef ([]uint8) (slice)
+
+	maj, extra, err = cbg.CborReadHeader(br)
+	if err != nil {
+		return err
+	}
+
+	if extra > cbg.ByteArrayMaxLen {
+		return fmt.Errorf("t.PieceRef: byte array too large (%d)", extra)
+	}
+	if maj != cbg.MajByteString {
+		return fmt.Errorf("expected byte array")
+	}
+	t.PieceRef = make([]byte, extra)
+	if _, err := io.ReadFull(br, t.PieceRef); err != nil {
+		return err
+	}
+	// t.PieceSize (uint64) (uint64)
+
+	maj, extra, err = cbg.CborReadHeader(br)
+	if err != nil {
+		return err
+	}
+	if maj != cbg.MajUnsignedInt {
+		return fmt.Errorf("wrong type for uint64 field")
+	}
+	t.PieceSize = uint64(extra)
+	// t.Client (address.Address) (struct)
 
 	{
 
-		if err := t.Deal.UnmarshalCBOR(br); err != nil {
+		if err := t.Client.UnmarshalCBOR(br); err != nil {
+			return err
+		}
+
+	}
+	// t.Provider (address.Address) (struct)
+
+	{
+
+		if err := t.Provider.UnmarshalCBOR(br); err != nil {
+			return err
+		}
+
+	}
+	// t.ProposalExpiration (uint64) (uint64)
+
+	maj, extra, err = cbg.CborReadHeader(br)
+	if err != nil {
+		return err
+	}
+	if maj != cbg.MajUnsignedInt {
+		return fmt.Errorf("wrong type for uint64 field")
+	}
+	t.ProposalExpiration = uint64(extra)
+	// t.Duration (uint64) (uint64)
+
+	maj, extra, err = cbg.CborReadHeader(br)
+	if err != nil {
+		return err
+	}
+	if maj != cbg.MajUnsignedInt {
+		return fmt.Errorf("wrong type for uint64 field")
+	}
+	t.Duration = uint64(extra)
+	// t.StoragePricePerEpoch (types.BigInt) (struct)
+
+	{
+
+		if err := t.StoragePricePerEpoch.UnmarshalCBOR(br); err != nil {
+			return err
+		}
+
+	}
+	// t.StorageCollateral (types.BigInt) (struct)
+
+	{
+
+		if err := t.StorageCollateral.UnmarshalCBOR(br); err != nil {
 			return err
 		}
 

chain/actors/harness2_test.go

@@ -3,6 +3,7 @@ package actors_test
 import (
 	"bytes"
 	"context"
+	"math/rand"
 	"testing"
 
 	"github.com/ipfs/go-cid"
@@ -41,10 +42,12 @@ const (
 type HarnessOpt func(testing.TB, *Harness) error
 
 type Harness struct {
-	HI         HarnessInit
+	HI          HarnessInit
-	Stage      HarnessStage
+	Stage       HarnessStage
-	Nonces     map[address.Address]uint64
+	Nonces      map[address.Address]uint64
-	GasCharges map[address.Address]types.BigInt
+	GasCharges  map[address.Address]types.BigInt
+	Rand        vm.Rand
+	BlockHeight uint64
 
 	lastBalanceCheck map[address.Address]types.BigInt
 
@@ -127,6 +130,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness {
 	h := &Harness{
 		Stage:  HarnessPreInit,
 		Nonces: make(map[address.Address]uint64),
+		Rand:   &fakeRand{},
 		HI: HarnessInit{
 			NAddrs: 1,
 			Miner:  blsaddr(0),
@@ -140,6 +144,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness {
 		w:                w,
 		ctx:              context.Background(),
 		bs:               bstore.NewBlockstore(dstore.NewMapDatastore()),
+		BlockHeight:      0,
 	}
 	for _, opt := range options {
 		err := opt(t, h)
@@ -157,8 +162,14 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	stateroot, err = gen.SetupStorageMarketActor(h.bs, stateroot, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	h.cs = store.NewChainStore(h.bs, nil)
-	h.vm, err = vm.NewVM(stateroot, 1, nil, h.HI.Miner, h.cs.Blockstore())
+	h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -246,6 +257,7 @@ func (h *Harness) Invoke(t testing.TB, from address.Address, to address.Address,
 func (h *Harness) InvokeWithValue(t testing.TB, from address.Address, to address.Address,
 	method uint64, value types.BigInt, params cbg.CBORMarshaler) (*vm.ApplyRet, *state.StateTree) {
 	t.Helper()
+	h.vm.SetBlockHeight(h.BlockHeight)
 	return h.Apply(t, types.Message{
 		To:       to,
 		From:     from,
@@ -315,3 +327,11 @@ func DumpObject(t testing.TB, obj cbg.CBORMarshaler) []byte {
 	}
 	return b.Bytes()
 }
+
+type fakeRand struct{}
+
+func (fr *fakeRand) GetRandomness(ctx context.Context, h int64) ([]byte, error) {
+	out := make([]byte, 32)
+	rand.New(rand.NewSource(h)).Read(out)
+	return out, nil
+}

chain/deals/cbor_gen.go

@@ -192,7 +192,7 @@ func (t *Response) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{133}); err != nil {
+	if _, err := w.Write([]byte{132}); err != nil {
 		return err
 	}
 
@@ -215,23 +215,10 @@ func (t *Response) MarshalCBOR(w io.Writer) error {
 		return xerrors.Errorf("failed to write cid field t.Proposal: %w", err)
 	}
 
-	// t.StorageDeal (actors.StorageDeal) (struct)
+	// t.StorageDealSubmission (types.SignedMessage) (struct)
-	if err := t.StorageDeal.MarshalCBOR(w); err != nil {
+	if err := t.StorageDealSubmission.MarshalCBOR(w); err != nil {
 		return err
 	}
-
-	// t.PublishMessage (cid.Cid) (struct)
-
-	if t.PublishMessage == nil {
-		if _, err := w.Write(cbg.CborNull); err != nil {
-			return err
-		}
-	} else {
-		if err := cbg.WriteCid(w, *t.PublishMessage); err != nil {
-			return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err)
-		}
-	}
-
 	return nil
 }
 
@@ -246,7 +233,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type array")
 	}
 
-	if extra != 5 {
+	if extra != 4 {
 		return fmt.Errorf("cbor input had wrong number of fields")
 	}
 
@@ -282,7 +269,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error {
 		t.Proposal = c
 
 	}
-	// t.StorageDeal (actors.StorageDeal) (struct)
+	// t.StorageDealSubmission (types.SignedMessage) (struct)
 
 	{
 
@@ -296,36 +283,12 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error {
 				return err
 			}
 		} else {
-			t.StorageDeal = new(actors.StorageDeal)
+			t.StorageDealSubmission = new(types.SignedMessage)
-			if err := t.StorageDeal.UnmarshalCBOR(br); err != nil {
+			if err := t.StorageDealSubmission.UnmarshalCBOR(br); err != nil {
 				return err
 			}
 		}
 
-	}
-	// t.PublishMessage (cid.Cid) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-
-			c, err := cbg.ReadCid(br)
-			if err != nil {
-				return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err)
-			}
-
-			t.PublishMessage = &c
-		}
-
 	}
 	return nil
 }
@@ -593,18 +556,10 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.PublishMessage (cid.Cid) (struct)
+	// t.PublishMessage (types.SignedMessage) (struct)
-
+	if err := t.PublishMessage.MarshalCBOR(w); err != nil {
-	if t.PublishMessage == nil {
+		return err
-		if _, err := w.Write(cbg.CborNull); err != nil {
-			return err
-		}
-	} else {
-		if err := cbg.WriteCid(w, *t.PublishMessage); err != nil {
-			return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err)
-		}
 	}
-
 	return nil
 }
 
@@ -683,7 +638,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("wrong type for uint64 field")
 	}
 	t.DealID = uint64(extra)
-	// t.PublishMessage (cid.Cid) (struct)
+	// t.PublishMessage (types.SignedMessage) (struct)
 
 	{
 
@@ -697,13 +652,10 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error {
 				return err
 			}
 		} else {
-
+			t.PublishMessage = new(types.SignedMessage)
-			c, err := cbg.ReadCid(br)
+			if err := t.PublishMessage.UnmarshalCBOR(br); err != nil {
-			if err != nil {
+				return err
-				return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err)
 			}
-
-			t.PublishMessage = &c
 		}
 
 	}

chain/deals/client.go

@@ -36,7 +36,7 @@ type ClientDeal struct {
 	MinerWorker address.Address
 	DealID      uint64
 
-	PublishMessage *cid.Cid
+	PublishMessage *types.SignedMessage
 
 	s inet.Stream
 }
@@ -206,7 +206,6 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro
 	dealProposal := &actors.StorageDealProposal{
 		PieceRef:             commP,
 		PieceSize:            uint64(pieceSize),
-		PieceSerialization:   actors.SerializationUnixFSv0,
 		Client:               p.Client,
 		Provider:             p.ProviderAddress,
 		ProposalExpiration:   p.ProposalExpiration,

chain/deals/client_states.go

@@ -44,6 +44,9 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), e
 	if err != nil {
 		return nil, err
 	}
+
+	// TODO: verify StorageDealSubmission
+
 	if err := c.disconnect(deal); err != nil {
 		return nil, err
 	}
@@ -54,18 +57,14 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), e
 	}
 
 	return func(info *ClientDeal) {
-		info.PublishMessage = resp.PublishMessage
+		info.PublishMessage = resp.StorageDealSubmission
 	}, nil
 }
 
 func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
 	log.Infow("DEAL ACCEPTED!")
 
-	pubmsg, err := c.chain.GetMessage(*deal.PublishMessage)
+	pubmsg := deal.PublishMessage.Message
-	if err != nil {
-		return nil, xerrors.Errorf("getting deal pubsish message: %w", err)
-	}
-
 	pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider)
 	if err != nil {
 		return nil, xerrors.Errorf("getting miner worker failed: %w", err)
@@ -91,7 +90,7 @@ func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDea
 	dealIdx := -1
 	for i, storageDeal := range params.Deals {
 		// TODO: make it less hacky
-		eq, err := cborutil.Equals(&deal.Proposal, &storageDeal.Proposal)
+		eq, err := cborutil.Equals(&deal.Proposal, &storageDeal)
 		if err != nil {
 			return nil, err
 		}
@@ -102,11 +101,11 @@ func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDea
 	}
 
 	if dealIdx == -1 {
-		return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage)
+		return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage.Cid())
 	}
 
 	// TODO: timeout
-	_, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage)
+	_, ret, err := c.sm.WaitForMessage(ctx, deal.PublishMessage.Cid())
 	if err != nil {
 		return nil, xerrors.Errorf("waiting for deal publish message: %w", err)
 	}

chain/deals/provider_states.go

@@ -42,13 +42,6 @@ func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandle
 
 // ACCEPTED
 func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
-	switch deal.Proposal.PieceSerialization {
-	//case SerializationRaw:
-	//case SerializationIPLD:
-	case actors.SerializationUnixFSv0:
-	default:
-		return nil, xerrors.Errorf("deal proposal with unsupported serialization: %s", deal.Proposal.PieceSerialization)
-	}
 
 	head, err := p.full.ChainHead(ctx)
 	if err != nil {
@@ -93,15 +86,8 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal)
 
 	log.Info("publishing deal")
 
-	storageDeal := actors.StorageDeal{
-		Proposal: deal.Proposal,
-	}
-	if err := api.SignWith(ctx, p.full.WalletSign, waddr, &storageDeal); err != nil {
-		return nil, xerrors.Errorf("signing storage deal failed: ", err)
-	}
-
 	params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{
-		Deals: []actors.StorageDeal{storageDeal},
+		Deals: []actors.StorageDealProposal{deal.Proposal},
 	})
 	if err != nil {
 		return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err)
@@ -136,13 +122,11 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal)
 	}
 
 	log.Infof("fetching data for a deal %d", resp.DealIDs[0])
-	mcid := smsg.Cid()
 	err = p.sendSignedResponse(&Response{
 		State: api.DealAccepted,
 
-		Proposal:       deal.ProposalCid,
+		Proposal:              deal.ProposalCid,
-		PublishMessage: &mcid,
+		StorageDealSubmission: smsg,
-		StorageDeal:    &storageDeal,
 	})
 	if err != nil {
 		return nil, err

chain/deals/types.go

@@ -58,8 +58,7 @@ type Response struct {
 	Proposal cid.Cid
 
 	// DealAccepted
-	StorageDeal    *actors.StorageDeal
+	StorageDealSubmission *types.SignedMessage
-	PublishMessage *cid.Cid
 }
 
 // TODO: Do we actually need this to be signed?

chain/gen/gen.go

@@ -526,9 +526,11 @@ func IsRoundWinner(ctx context.Context, ts *types.TipSet, round int64, miner add
 		return nil, xerrors.Errorf("failed to look up miners sector size: %w", err)
 	}
 
+	snum := types.BigDiv(pow.MinerPower, types.NewInt(ssize))
+
 	var winners []sectorbuilder.EPostCandidate
 	for _, c := range candidates {
-		if types.IsTicketWinner(c.PartialTicket[:], ssize, pow.TotalPower) {
+		if types.IsTicketWinner(c.PartialTicket[:], ssize, snum.Uint64(), pow.TotalPower) {
 			winners = append(winners, c)
 		}
 	}

chain/gen/utils.go

@@ -197,7 +197,7 @@ func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) {
 	}, nil
 }
 
-func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors.StorageDeal) (cid.Cid, error) {
+func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors.StorageDealProposal) (cid.Cid, error) {
 	cst := hamt.CSTFromBstore(bs)
 	nd := hamt.NewNode(cst)
 	emptyHAMT, err := cst.Put(context.TODO(), nd)
@@ -210,8 +210,15 @@ func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors
 	cdeals := make([]cbg.CBORMarshaler, len(deals))
 	for i, deal := range deals {
 		cdeals[i] = &actors.OnChainDeal{
-			Deal:            deal,
+			PieceRef:             deal.PieceRef,
-			ActivationEpoch: 1,
+			PieceSize:            deal.PieceSize,
+			Client:               deal.Client,
+			Provider:             deal.Provider,
+			ProposalExpiration:   deal.ProposalExpiration,
+			Duration:             deal.Duration,
+			StoragePricePerEpoch: deal.StoragePricePerEpoch,
+			StorageCollateral:    deal.StorageCollateral,
+			ActivationEpoch:      1,
 		}
 	}
 
@@ -267,7 +274,7 @@ func mustEnc(i cbg.CBORMarshaler) []byte {
 	return enc
 }
 
-func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDeal, error) {
+func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDealProposal, error) {
 	vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore())
 	if err != nil {
 		return cid.Undef, nil, xerrors.Errorf("failed to create NewVM: %w", err)
@@ -281,7 +288,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid
 		return cid.Undef, nil, xerrors.Errorf("miner address list, and preseal count doesn't match (%d != %d)", len(gmcfg.MinerAddrs), len(gmcfg.PreSeals))
 	}
 
-	var deals []actors.StorageDeal
+	var deals []actors.StorageDealProposal
 
 	for i, maddr := range gmcfg.MinerAddrs {
 		ps, psok := gmcfg.PreSeals[maddr.String()]

chain/stmgr/stmgr.go

@@ -138,7 +138,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
 		return cid.Undef, cid.Undef, xerrors.Errorf("failed to get network actor: %w", err)
 	}
 	reward := vm.MiningReward(netact.Balance)
-	for _, b := range blks {
+	for tsi, b := range blks {
 		netact, err = vmi.StateTree().GetActor(actors.NetworkAddress)
 		if err != nil {
 			return cid.Undef, cid.Undef, xerrors.Errorf("failed to get network actor: %w", err)
@@ -171,10 +171,10 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
 		}
 		ret, err := vmi.ApplyMessage(ctx, postSubmitMsg)
 		if err != nil {
-			return cid.Undef, cid.Undef, xerrors.Errorf("submit election post message invocation failed: %w", err)
+			return cid.Undef, cid.Undef, xerrors.Errorf("submit election post message for block %s (miner %s) invocation failed: %w", b.Cid(), b.Miner, err)
 		}
 		if ret.ExitCode != 0 {
-			return cid.Undef, cid.Undef, xerrors.Errorf("submit election post invocation returned nonzero exit code: %d", ret.ExitCode)
+			return cid.Undef, cid.Undef, xerrors.Errorf("submit election post invocation returned nonzero exit code: %d (err = %s, block = %s, miner = %s, tsi = %d)", ret.ExitCode, ret.ActorErr, b.Cid(), b.Miner, tsi)
 		}
 	}
 

chain/sync.go

@@ -492,7 +492,16 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
 	}
 
 	winnerCheck := async.Err(func() error {
-		_, tpow, err := stmgr.GetPower(ctx, syncer.sm, baseTs, h.Miner)
+		slashedAt, err := stmgr.GetMinerSlashed(ctx, syncer.sm, baseTs, h.Miner)
+		if err != nil {
+			return xerrors.Errorf("failed to check if block miner was slashed: %w", err)
+		}
+
+		if slashedAt != 0 {
+			return xerrors.Errorf("received block was from miner slashed at height %d", slashedAt)
+		}
+
+		mpow, tpow, err := stmgr.GetPower(ctx, syncer.sm, baseTs, h.Miner)
 		if err != nil {
 			return xerrors.Errorf("failed getting power: %w", err)
 		}
@@ -502,8 +511,10 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
 			return xerrors.Errorf("failed to get sector size for block miner: %w", err)
 		}
 
+		snum := types.BigDiv(mpow, types.NewInt(ssize))
+
 		for _, t := range h.EPostProof.Candidates {
-			if !types.IsTicketWinner(t.Partial, ssize, tpow) {
+			if !types.IsTicketWinner(t.Partial, ssize, snum.Uint64(), tpow) {
 				return xerrors.Errorf("miner created a block but was not a winner")
 			}
 		}

chain/types/bitfield.go

@@ -34,6 +34,38 @@ func BitFieldFromSet(setBits []uint64) BitField {
 	return res
 }
 
+func MergeBitFields(a, b BitField) (BitField, error) {
+	ra, err := a.rle.RunIterator()
+	if err != nil {
+		return BitField{}, err
+	}
+
+	rb, err := b.rle.RunIterator()
+	if err != nil {
+		return BitField{}, err
+	}
+
+	merge, err := rlepluslazy.Sum(ra, rb)
+	if err != nil {
+		return BitField{}, err
+	}
+
+	mergebytes, err := rlepluslazy.EncodeRuns(merge, nil)
+	if err != nil {
+		return BitField{}, err
+	}
+
+	rle, err := rlepluslazy.FromBuf(mergebytes)
+	if err != nil {
+		return BitField{}, err
+	}
+
+	return BitField{
+		rle:  rle,
+		bits: make(map[uint64]struct{}),
+	}, nil
+}
+
 func (bf BitField) sum() (rlepluslazy.RunIterator, error) {
 	if len(bf.bits) == 0 {
 		return bf.rle.RunIterator()
@@ -86,7 +118,26 @@ func (bf BitField) All() ([]uint64, error) {
 		return nil, err
 	}
 
-	return res, err
+	return res, nil
+}
+
+func (bf BitField) AllMap() (map[uint64]bool, error) {
+
+	runs, err := bf.sum()
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := rlepluslazy.SliceFromRuns(runs)
+	if err != nil {
+		return nil, err
+	}
+
+	out := make(map[uint64]bool)
+	for _, i := range res {
+		out[i] = true
+	}
+	return out, nil
 }
 
 func (bf BitField) MarshalCBOR(w io.Writer) error {

chain/types/blockheader.go

@@ -174,9 +174,9 @@ var blocksPerEpoch = NewInt(build.BlocksPerEpoch)
 
 const sha256bits = 256
 
-func IsTicketWinner(partialTicket []byte, ssizeI uint64, totpow BigInt) bool {
+func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow BigInt) bool {
 	ssize := NewInt(ssizeI)
-
+	ssampled := ElectionPostChallengeCount(snum)
 	/*
 		Need to check that
 		(h(vrfout) + 1) / (max(h) + 1) <= e * sectorSize / totalPower
@@ -185,23 +185,42 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, totpow BigInt) bool {
 		(h(vrfout) + 1) * totalPower <= e * sectorSize * 2^256
 		in 2^256 space, it is equivalent to:
 		h(vrfout) * totalPower < e * sectorSize * 2^256
+
+		Because of SectorChallengeRatioDiv sampling for proofs
+		we need to scale this appropriately.
+
+		Let c = ceil(numSectors/SectorChallengeRatioDiv)
+		(c is the number of tickets a miner requests)
+		Accordingly we check
+		(h(vrfout) + 1) / 2^256 <= e * sectorSize / totalPower * snum / c
+		or
+		h(vrfout) * totalPower * c < e * sectorSize * 2^256 * snum
 	*/
 
 	h := sha256.Sum256(partialTicket)
 
 	lhs := BigFromBytes(h[:]).Int
 	lhs = lhs.Mul(lhs, totpow.Int)
+	lhs = lhs.Mul(lhs, new(big.Int).SetUint64(ssampled))
 
 	// rhs = sectorSize * 2^256
 	// rhs = sectorSize << 256
 	rhs := new(big.Int).Lsh(ssize.Int, sha256bits)
-	rhs = rhs.Mul(rhs, big.NewInt(build.SectorChallengeRatioDiv))
+	rhs = rhs.Mul(rhs, new(big.Int).SetUint64(snum))
 	rhs = rhs.Mul(rhs, blocksPerEpoch.Int)
 
 	// h(vrfout) * totalPower < e * sectorSize * 2^256?
 	return lhs.Cmp(rhs) < 0
 }
 
+func ElectionPostChallengeCount(sectors uint64) uint64 {
+	if sectors == 0 {
+		return 0
+	}
+	// ceil(sectors / build.SectorChallengeRatioDiv)
+	return (sectors-1)/build.SectorChallengeRatioDiv + 1
+}
+
 func (t *Ticket) Equals(ot *Ticket) bool {
 	return bytes.Equal(t.VRFProof, ot.VRFProof)
 }

chain/types/vmcontext.go

@@ -40,10 +40,15 @@ type VMContext interface {
 	ChargeGas(uint64) aerrors.ActorError
 	GetRandomness(height uint64) ([]byte, aerrors.ActorError)
 	GetBalance(address.Address) (BigInt, aerrors.ActorError)
+	Sys() *VMSyscalls
 
 	Context() context.Context
 }
 
+type VMSyscalls struct {
+	ValidatePoRep func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError)
+}
+
 type storageWrapper struct {
 	s Storage
 }

chain/vm/mkactor.go

@@ -48,7 +48,7 @@ func makeActor(st *state.StateTree, addr address.Address) (*types.Actor, aerrors
 	case address.SECP256K1:
 		return NewSecp256k1AccountActor(st, addr)
 	case address.ID:
-		return nil, aerrors.New(1, "no actor with given ID")
+		return nil, aerrors.Newf(1, "no actor with given ID: %s", addr)
 	case address.Actor:
 		return nil, aerrors.Newf(1, "no such actor: %s", addr)
 	default:

chain/vm/syscalls.go

@@ -0,0 +1,14 @@
+package vm
+
+import (
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+// Actual type is defined in chain/types/vmcontext.go because the VMContext interface is there
+
+func DefaultSyscalls() *types.VMSyscalls {
+	return &types.VMSyscalls{
+		ValidatePoRep: actors.ValidatePoRep,
+	}
+}

chain/vm/vm.go

@@ -54,6 +54,8 @@ type VMContext struct {
 	gasAvailable types.BigInt
 	gasUsed      types.BigInt
 
+	sys *types.VMSyscalls
+
 	// root cid of the state of the actor this invocation will be on
 	sroot cid.Cid
 
@@ -75,6 +77,10 @@ func (vmc *VMContext) GetRandomness(height uint64) ([]byte, aerrors.ActorError)
 	return res, nil
 }
 
+func (vmc *VMContext) Sys() *types.VMSyscalls {
+	return vmc.sys
+}
+
 // Storage interface
 
 func (vmc *VMContext) Put(i cbg.CBORMarshaler) (cid.Cid, aerrors.ActorError) {
@@ -284,6 +290,7 @@ func (vm *VM) makeVMContext(ctx context.Context, sroot cid.Cid, msg *types.Messa
 		msg:    msg,
 		origin: origin,
 		height: vm.blockHeight,
+		sys:    vm.Syscalls,
 
 		gasUsed:      usedGas,
 		gasAvailable: msg.GasLimit,
@@ -304,6 +311,8 @@ type VM struct {
 	blockMiner  address.Address
 	inv         *invoker
 	rand        Rand
+
+	Syscalls *types.VMSyscalls
 }
 
 func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore) (*VM, error) {
@@ -323,6 +332,7 @@ func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs block
 		blockMiner:  maddr,
 		inv:         newInvoker(),
 		rand:        r,
+		Syscalls:    DefaultSyscalls(),
 	}, nil
 }
 

cli/auth.go

@@ -6,7 +6,7 @@ import (
 
 	"gopkg.in/urfave/cli.v2"
 
-	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/apistruct"
 )
 
 var authCmd = &cli.Command{
@@ -42,18 +42,18 @@ var authCreateAdminToken = &cli.Command{
 
 		perm := cctx.String("perm")
 		idx := 0
-		for i, p := range api.AllPermissions {
+		for i, p := range apistruct.AllPermissions {
 			if perm == p {
 				idx = i + 1
 			}
 		}
 
 		if idx == 0 {
-			return fmt.Errorf("--perm flag has to be one of: %s", api.AllPermissions)
+			return fmt.Errorf("--perm flag has to be one of: %s", apistruct.AllPermissions)
 		}
 
 		// slice on [:idx] so for example: 'sign' gives you [read, write, sign]
-		token, err := napi.AuthNew(ctx, api.AllPermissions[:idx])
+		token, err := napi.AuthNew(ctx, apistruct.AllPermissions[:idx])
 		if err != nil {
 			return err
 		}

cmd/lotus-seed/seed/seed.go

@@ -2,11 +2,11 @@ package seed
 
 import (
 	"context"
+	"crypto/rand"
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"math/rand"
 	"os"
 	"path/filepath"
 
@@ -53,7 +53,6 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb
 		return nil, err
 	}
 
-	r := rand.New(rand.NewSource(101))
 	size := sectorbuilder.UserBytesForSectorSize(ssize)
 
 	var sealedSectors []*genesis.PreSeal
@@ -63,7 +62,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb
 			return nil, err
 		}
 
-		pi, err := sb.AddPiece(size, sid, r, nil)
+		pi, err := sb.AddPiece(size, sid, rand.Reader, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -143,7 +142,6 @@ func createDeals(m *genesis.GenesisMiner, k *wallet.Key, maddr address.Address,
 		proposal := &actors.StorageDealProposal{
 			PieceRef:             pref, // just one deal so this == CommP
 			PieceSize:            sectorbuilder.UserBytesForSectorSize(ssize),
-			PieceSerialization:   actors.SerializationUnixFSv0,
 			Client:               k.Address,
 			Provider:             maddr,
 			ProposalExpiration:   9000, // TODO: allow setting
@@ -153,20 +151,12 @@ func createDeals(m *genesis.GenesisMiner, k *wallet.Key, maddr address.Address,
 			ProposerSignature:    nil,
 		}
 
+		// TODO: pretty sure we don't even need to sign this
 		if err := api.SignWith(context.TODO(), wallet.KeyWallet(k).Sign, k.Address, proposal); err != nil {
 			return err
 		}
 
-		deal := &actors.StorageDeal{
+		sector.Deal = *proposal
-			Proposal:         *proposal,
-			CounterSignature: nil,
-		}
-
-		if err := api.SignWith(context.TODO(), wallet.KeyWallet(k).Sign, k.Address, deal); err != nil {
-			return err
-		}
-
-		sector.Deal = *deal
 	}
 
 	return nil

cmd/lotus-storage-miner/init.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"crypto/rand"
 	"encoding/json"
@@ -267,7 +268,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin
 			return err
 		}
 
-		proposalCid, err := sector.Deal.Proposal.Cid()
+		proposalCid, err := sector.Deal.Cid()
 		if err != nil {
 			return err
 		}
@@ -275,7 +276,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin
 		dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String())
 
 		deal := &deals.MinerDeal{
-			Proposal:    sector.Deal.Proposal,
+			Proposal:    sector.Deal,
 			ProposalCid: proposalCid,
 			State:       lapi.DealComplete,
 			Ref:         proposalCid, // TODO: This is super wrong, but there
@@ -298,7 +299,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin
 	return nil
 }
 
-func findMarketDealID(ctx context.Context, api lapi.FullNode, deal actors.StorageDeal) (uint64, error) {
+func findMarketDealID(ctx context.Context, api lapi.FullNode, deal actors.StorageDealProposal) (uint64, error) {
 	// TODO: find a better way
 	//  (this is only used by genesis miners)
 
@@ -308,11 +309,7 @@ func findMarketDealID(ctx context.Context, api lapi.FullNode, deal actors.Storag
 	}
 
 	for k, v := range deals {
-		eq, err := cborutil.Equals(&v.Deal, &deal)
+		if bytes.Equal(v.PieceRef, deal.PieceRef) {
-		if err != nil {
-			return 0, err
-		}
-		if eq {
 			return strconv.ParseUint(k, 10, 64)
 		}
 	}

cmd/lotus-storage-miner/run.go

@@ -15,6 +15,7 @@ import (
 	"gopkg.in/urfave/cli.v2"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/apistruct"
 	"github.com/filecoin-project/lotus/build"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/lib/auth"
@@ -131,7 +132,7 @@ var runCmd = &cli.Command{
 		mux := mux.NewRouter()
 
 		rpcServer := jsonrpc.NewServer()
-		rpcServer.Register("Filecoin", api.PermissionedStorMinerAPI(minerapi))
+		rpcServer.Register("Filecoin", apistruct.PermissionedStorMinerAPI(minerapi))
 
 		mux.Handle("/rpc/v0", rpcServer)
 		mux.PathPrefix("/remote").HandlerFunc(minerapi.(*impl.StorageMinerAPI).ServeRemote)

cmd/lotus/rpc.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"github.com/filecoin-project/lotus/api/apistruct"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
@@ -26,7 +27,7 @@ var log = logging.Logger("main")
 
 func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr) error {
 	rpcServer := jsonrpc.NewServer()
-	rpcServer.Register("Filecoin", api.PermissionedFullAPI(a))
+	rpcServer.Register("Filecoin", apistruct.PermissionedFullAPI(a))
 
 	ah := &auth.Handler{
 		Verify: a.AuthVerify,
@@ -70,7 +71,7 @@ func handleImport(a *impl.FullNodeAPI) func(w http.ResponseWriter, r *http.Reque
 			w.WriteHeader(404)
 			return
 		}
-		if !api.HasPerm(r.Context(), api.PermWrite) {
+		if !apistruct.HasPerm(r.Context(), apistruct.PermWrite) {
 			w.WriteHeader(401)
 			json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"})
 			return

gen/main.go

@@ -96,6 +96,7 @@ func main() {
 		actors.SubmitFallbackPoStParams{},
 		actors.PaymentVerifyParams{},
 		actors.UpdatePeerIDParams{},
+		actors.DeclareFaultsParams{},
 		actors.MultiSigActorState{},
 		actors.MultiSigConstructorParams{},
 		actors.MultiSigProposeParams{},
@@ -122,7 +123,6 @@ func main() {
 		actors.StorageMarketState{},
 		actors.WithdrawBalanceParams{},
 		actors.StorageDealProposal{},
-		actors.StorageDeal{},
 		actors.PublishStorageDealsParams{},
 		actors.PublishStorageDealResponse{},
 		actors.ActivateStorageDealsParams{},

genesis/types.go

@@ -10,7 +10,7 @@ type PreSeal struct {
 	CommR    [32]byte
 	CommD    [32]byte
 	SectorID uint64
-	Deal     actors.StorageDeal
+	Deal     actors.StorageDealProposal
 }
 
 type GenesisMiner struct {

lib/auth/handler.go

@@ -5,8 +5,10 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/filecoin-project/lotus/api"
 	logging "github.com/ipfs/go-log"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/apistruct"
 )
 
 var log = logging.Logger("auth")
@@ -42,7 +44,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		ctx = api.WithPerm(ctx, allow)
+		ctx = apistruct.WithPerm(ctx, allow)
 	}
 
 	h.Next(w, r.WithContext(ctx))

lib/sectorbuilder/sectorbuilder.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/address"
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
 
@@ -623,7 +624,7 @@ func (sb *SectorBuilder) GenerateEPostCandidates(sectorInfo SortedPublicSectorIn
 		return nil, err
 	}
 
-	challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo.Values())))
+	challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())))
 
 	proverID := addressToProverID(sb.Miner)
 	return sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors)
@@ -674,13 +675,8 @@ func (sb *SectorBuilder) Stop() {
 	close(sb.stopping)
 }
 
-func ElectionPostChallengeCount(sectors uint64) uint64 {
-	// ceil(sectors / build.SectorChallengeRatioDiv)
-	return (sectors + build.SectorChallengeRatioDiv - 1) / build.SectorChallengeRatioDiv
-}
-
 func fallbackPostChallengeCount(sectors uint64) uint64 {
-	challengeCount := ElectionPostChallengeCount(sectors)
+	challengeCount := types.ElectionPostChallengeCount(sectors)
 	if challengeCount > build.MaxFallbackPostChallengeCount {
 		return build.MaxFallbackPostChallengeCount
 	}

lib/sectorbuilder/sectorbuilder_test.go

@@ -124,6 +124,9 @@ func post(t *testing.T, sb *sectorbuilder.SectorBuilder, seals ...seal) time.Tim
 }
 
 func TestSealAndVerify(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode")
+	}
 	if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
 		t.Skip("this is slow")
 	}
@@ -193,6 +196,9 @@ func TestSealAndVerify(t *testing.T) {
 }
 
 func TestSealPoStNoCommit(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode")
+	}
 	if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
 		t.Skip("this is slow")
 	}
@@ -259,6 +265,9 @@ func TestSealPoStNoCommit(t *testing.T) {
 }
 
 func TestSealAndVerify2(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode")
+	}
 	if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware
 		t.Skip("this is slow")
 	}

lib/sectorbuilder/simple.go

@@ -8,6 +8,7 @@ import (
 	"go.opencensus.io/trace"
 
 	"github.com/filecoin-project/lotus/chain/address"
+	"github.com/filecoin-project/lotus/chain/types"
 )
 
 func (sb *SectorBuilder) SectorSize() uint64 {
@@ -36,7 +37,7 @@ func NewSortedPublicSectorInfo(sectors []sectorbuilder.PublicSectorInfo) SortedP
 }
 
 func VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) {
-	challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo.Values())))
+	challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values())))
 	return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID)
 }
 

node/impl/common.go

@@ -2,7 +2,6 @@ package impl
 
 import (
 	"context"
-
 	"github.com/gbrlsnchs/jwt/v3"
 	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/network"

node/impl/full/state.go

@@ -148,7 +148,7 @@ func (a *StateAPI) stateForTs(ctx context.Context, ts *types.TipSet) (*state.Sta
 func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error) {
 	state, err := a.stateForTs(ctx, ts)
 	if err != nil {
-		return nil, err
+		return nil, xerrors.Errorf("computing tipset state failed: %w", err)
 	}
 
 	return state.GetActor(actor)

node/impl/storminer.go

@@ -3,6 +3,7 @@ package impl
 import (
 	"context"
 	"encoding/json"
+	"github.com/filecoin-project/lotus/api/apistruct"
 	"io"
 	"mime"
 	"net/http"
@@ -33,7 +34,7 @@ type StorageMinerAPI struct {
 }
 
 func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) {
-	if !api.HasPerm(r.Context(), api.PermAdmin) {
+	if !apistruct.HasPerm(r.Context(), apistruct.PermAdmin) {
 		w.WriteHeader(401)
 		json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"})
 		return

node/modules/core.go

@@ -3,10 +3,10 @@ package modules
 import (
 	"context"
 	"crypto/rand"
+	"github.com/filecoin-project/lotus/api/apistruct"
 	"io"
 	"io/ioutil"
 
-	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/lib/addrutil"
@@ -57,7 +57,7 @@ func APISecret(keystore types.KeyStore, lr repo.LockedRepo) (*dtypes.APIAlg, err
 
 		// TODO: make this configurable
 		p := jwtPayload{
-			Allow: api.AllPermissions,
+			Allow: apistruct.AllPermissions,
 		}
 
 		cliToken, err := jwt.Sign(&p, jwt.NewHS256(key.PrivateKey))

node/node_test.go

@@ -292,5 +292,8 @@ func TestAPIRPC(t *testing.T) {
 }
 
 func TestAPIDealFlow(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode")
+	}
 	test.TestDealFlow(t, builder)
 }

storage/cbor_gen.go

@@ -385,7 +385,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{172}); err != nil {
+	if _, err := w.Write([]byte{173}); err != nil {
 		return err
 	}
 
@@ -547,6 +547,24 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error {
 		}
 	}
 
+	// t.FaultReportMsg (cid.Cid) (struct)
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("FaultReportMsg")))); err != nil {
+		return err
+	}
+	if _, err := w.Write([]byte("FaultReportMsg")); err != nil {
+		return err
+	}
+
+	if t.FaultReportMsg == nil {
+		if _, err := w.Write(cbg.CborNull); err != nil {
+			return err
+		}
+	} else {
+		if err := cbg.WriteCid(w, *t.FaultReportMsg); err != nil {
+			return xerrors.Errorf("failed to write cid field t.FaultReportMsg: %w", err)
+		}
+	}
+
 	// t.LastErr (string) (string)
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("LastErr")))); err != nil {
 		return err
@@ -575,7 +593,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type map")
 	}
 
-	if extra != 12 {
+	if extra != 13 {
 		return fmt.Errorf("cbor input had wrong number of fields")
 	}
 
@@ -897,6 +915,43 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error {
 			t.CommitMessage = &c
 		}
 
+	}
+	// t.FaultReportMsg (cid.Cid) (struct)
+
+	{
+		sval, err := cbg.ReadString(br)
+		if err != nil {
+			return err
+		}
+
+		name = string(sval)
+	}
+
+	if name != "FaultReportMsg" {
+		return fmt.Errorf("expected struct map entry %s to be FaultReportMsg", name)
+	}
+
+	{
+
+		pb, err := br.PeekByte()
+		if err != nil {
+			return err
+		}
+		if pb == cbg.CborNull[0] {
+			var nbuf [1]byte
+			if _, err := br.Read(nbuf[:]); err != nil {
+				return err
+			}
+		} else {
+
+			c, err := cbg.ReadCid(br)
+			if err != nil {
+				return xerrors.Errorf("failed to read cid field t.FaultReportMsg: %w", err)
+			}
+
+			t.FaultReportMsg = &c
+		}
+
 	}
 	// t.LastErr (string) (string)
 

storage/garbage.go

@@ -20,7 +20,7 @@ func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPiece
 		return nil, nil
 	}
 
-	deals := make([]actors.StorageDeal, len(sizes))
+	deals := make([]actors.StorageDealProposal, len(sizes))
 	for i, size := range sizes {
 		release := m.sb.RateLimit()
 		commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size)
@@ -33,7 +33,6 @@ func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPiece
 		sdp := actors.StorageDealProposal{
 			PieceRef:             commP[:],
 			PieceSize:            size,
-			PieceSerialization:   actors.SerializationUnixFSv0,
 			Client:               m.worker,
 			Provider:             m.maddr,
 			ProposalExpiration:   math.MaxUint64,
@@ -47,14 +46,7 @@ func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPiece
 			return nil, xerrors.Errorf("signing storage deal failed: ", err)
 		}
 
-		storageDeal := actors.StorageDeal{
+		deals[i] = sdp
-			Proposal: sdp,
-		}
-		if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &storageDeal); err != nil {
-			return nil, xerrors.Errorf("signing storage deal failed: ", err)
-		}
-
-		deals[i] = storageDeal
 	}
 
 	params, aerr := actors.SerializeParams(&actors.PublishStorageDealsParams{

storage/sector_states.go

@@ -227,9 +227,63 @@ func (m *Miner) handleCommitWait(ctx context.Context, sector SectorInfo) *sector
 
 	if mw.Receipt.ExitCode != 0 {
 		log.Errorf("UNHANDLED: submitting sector proof failed (exit=%d, msg=%s) (t:%x; s:%x(%d); p:%x)", mw.Receipt.ExitCode, sector.CommitMessage, sector.Ticket.TicketBytes, sector.Seed.TicketBytes, sector.Seed.BlockHeight, sector.Proof)
-		return sector.upd().fatal(xerrors.New("UNHANDLED: submitting sector proof failed"))
+		return sector.upd().fatal(xerrors.Errorf("UNHANDLED: submitting sector proof failed (exit: %d)", mw.Receipt.ExitCode))
 	}
 
 	return sector.upd().to(api.Proving).state(func(info *SectorInfo) {
 	})
 }
+
+func (m *Miner) handleFaulty(ctx context.Context, sector SectorInfo) *sectorUpdate {
+	// TODO: check if the fault has already been reported, and that this sector is even valid
+
+	// TODO: coalesce faulty sector reporting
+	bf := types.NewBitField()
+	bf.Set(sector.SectorID)
+
+	fp := &actors.DeclareFaultsParams{bf}
+	_ = fp
+	enc, aerr := actors.SerializeParams(nil)
+	if aerr != nil {
+		return sector.upd().fatal(xerrors.Errorf("failed to serialize declare fault params: %w", aerr))
+	}
+
+	msg := &types.Message{
+		To:       m.maddr,
+		From:     m.worker,
+		Method:   actors.MAMethods.DeclareFaults,
+		Params:   enc,
+		Value:    types.NewInt(0), // TODO: need to ensure sufficient collateral
+		GasLimit: types.NewInt(1000000 /* i dont know help */),
+		GasPrice: types.NewInt(1),
+	}
+
+	smsg, err := m.api.MpoolPushMessage(ctx, msg)
+	if err != nil {
+		return sector.upd().to(api.FailedUnrecoverable).error(xerrors.Errorf("failed to push declare faults message to network: %w", err))
+	}
+
+	return sector.upd().to(api.FaultReported).state(func(info *SectorInfo) {
+		c := smsg.Cid()
+		info.FaultReportMsg = &c
+	})
+}
+
+func (m *Miner) handleFaultReported(ctx context.Context, sector SectorInfo) *sectorUpdate {
+	if sector.FaultReportMsg == nil {
+		return sector.upd().to(api.FailedUnrecoverable).error(xerrors.Errorf("entered fault reported state without a FaultReportMsg cid"))
+	}
+
+	mw, err := m.api.StateWaitMsg(ctx, *sector.FaultReportMsg)
+	if err != nil {
+		return sector.upd().to(api.CommitFailed).error(xerrors.Errorf("failed to wait for fault declaration: %w", err))
+	}
+
+	if mw.Receipt.ExitCode != 0 {
+		log.Errorf("UNHANDLED: declaring sector fault failed (exit=%d, msg=%s) (id: %d)", mw.Receipt.ExitCode, *sector.FaultReportMsg, sector.SectorID)
+		return sector.upd().fatal(xerrors.Errorf("UNHANDLED: submitting fault declaration failed (exit %d)", mw.Receipt.ExitCode))
+	}
+
+	return sector.upd().to(api.FaultedFinal).state(func(info *SectorInfo) {})
+
+}

storage/sector_types.go

@@ -69,6 +69,9 @@ type SectorInfo struct {
 	// Committing
 	CommitMessage *cid.Cid
 
+	// Faults
+	FaultReportMsg *cid.Cid
+
 	// Debug
 	LastErr string
 }

storage/sectors.go

@@ -255,6 +255,12 @@ func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) {
 	case api.CommitFailed:
 		log.Warnf("sector %d entered unimplemented state 'CommitFailed'", update.id)
 
+		// Faults
+	case api.Faulty:
+		m.handleSectorUpdate(ctx, sector, m.handleFaulty)
+	case api.FaultReported:
+		m.handleSectorUpdate(ctx, sector, m.handleFaultReported)
+
 	// Fatal errors
 	case api.UndefinedSectorState:
 		log.Error("sector update with undefined state!")