diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ad761862..c5bdeaa1b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,7 +76,7 @@ jobs: - store_artifacts: path: lotus-storage-miner - test: + test: &test description: | Run tests with gotestsum. parameters: @@ -154,6 +154,9 @@ jobs: - "~/go/src/github.com" - "~/go/src/golang.org" + test-short: + <<: *test + build_macos: description: build darwin lotus binary macos: @@ -254,8 +257,9 @@ workflows: jobs: - lint-changes: args: "--new-from-rev origin/master" - - test: - codecov-upload: true + - test + - test-short: + go-test-flags: "--timeout 10m --short" - mod-tidy-check - build-all - build_macos: diff --git a/api/api_common.go b/api/api_common.go index c83a4c260..ee99f6d76 100644 --- a/api/api_common.go +++ b/api/api_common.go @@ -10,6 +10,8 @@ import ( "github.com/filecoin-project/lotus/build" ) +type Permission = string + type Common interface { // Auth AuthVerify(ctx context.Context, token string) ([]Permission, error) diff --git a/api/api_storage.go b/api/api_storage.go index f34bb3367..1a1aff853 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -29,6 +29,10 @@ const ( CommitFailed FailedUnrecoverable + + Faulty // sector is corrupted or gone for some reason + FaultReported // sector has been declared as a fault on chain + FaultedFinal // fault declared on chain ) var SectorStates = []string{ @@ -39,6 +43,7 @@ var SectorStates = []string{ PreCommitting: "PreCommitting", PreCommitted: "PreCommitted", Committing: "Committing", + CommitWait: "CommitWait", Proving: "Proving", SealFailed: "SealFailed", @@ -47,6 +52,10 @@ var SectorStates = []string{ CommitFailed: "CommitFailed", FailedUnrecoverable: "FailedUnrecoverable", + + Faulty: "Faulty", + FaultReported: "FaultReported", + FaultedFinal: "FaultedFinal", } // StorageMiner is a low-level interface to the Filecoin network storage miner node diff --git a/api/permissioned.go b/api/apistruct/permissioned.go similarity index 68% rename from api/permissioned.go rename to api/apistruct/permissioned.go index 58d027278..4c29f6688 100644 --- a/api/permissioned.go +++ b/api/apistruct/permissioned.go @@ -1,50 +1,50 @@ -package api +package apistruct import ( "context" "reflect" "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" ) type permKey int var permCtxKey permKey -type Permission = string - const ( // When changing these, update docs/API.md too - PermRead Permission = "read" // default - PermWrite Permission = "write" - PermSign Permission = "sign" // Use wallet keys for signing - PermAdmin Permission = "admin" // Manage permissions + PermRead api.Permission = "read" // default + PermWrite api.Permission = "write" + PermSign api.Permission = "sign" // Use wallet keys for signing + PermAdmin api.Permission = "admin" // Manage permissions ) -var AllPermissions = []Permission{PermRead, PermWrite, PermSign, PermAdmin} -var defaultPerms = []Permission{PermRead} +var AllPermissions = []api.Permission{PermRead, PermWrite, PermSign, PermAdmin} +var defaultPerms = []api.Permission{PermRead} -func WithPerm(ctx context.Context, perms []Permission) context.Context { +func WithPerm(ctx context.Context, perms []api.Permission) context.Context { return context.WithValue(ctx, permCtxKey, perms) } -func PermissionedStorMinerAPI(a StorageMiner) StorageMiner { +func PermissionedStorMinerAPI(a api.StorageMiner) api.StorageMiner { var out StorageMinerStruct permissionedAny(a, &out.Internal) permissionedAny(a, &out.CommonStruct.Internal) return &out } -func PermissionedFullAPI(a FullNode) FullNode { +func PermissionedFullAPI(a api.FullNode) api.FullNode { var out FullNodeStruct permissionedAny(a, &out.Internal) permissionedAny(a, &out.CommonStruct.Internal) return &out } -func HasPerm(ctx context.Context, perm Permission) bool { - callerPerms, ok := ctx.Value(permCtxKey).([]Permission) +func HasPerm(ctx context.Context, perm api.Permission) bool { + callerPerms, ok := ctx.Value(permCtxKey).([]api.Permission) if !ok { callerPerms = defaultPerms } @@ -63,7 +63,7 @@ func permissionedAny(in interface{}, out interface{}) { for f := 0; f < rint.NumField(); f++ { field := rint.Type().Field(f) - requiredPerm := Permission(field.Tag.Get("perm")) + requiredPerm := api.Permission(field.Tag.Get("perm")) if requiredPerm == "" { panic("missing 'perm' tag on " + field.Name) // ok } diff --git a/api/struct.go b/api/apistruct/struct.go similarity index 86% rename from api/struct.go rename to api/apistruct/struct.go index c0ab86713..d2ea6bb8e 100644 --- a/api/struct.go +++ b/api/apistruct/struct.go @@ -1,4 +1,4 @@ -package api +package apistruct import ( "context" @@ -9,6 +9,7 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/store" @@ -20,8 +21,8 @@ var _ = AllPermissions type CommonStruct struct { Internal struct { - AuthVerify func(ctx context.Context, token string) ([]Permission, error) `perm:"read"` - AuthNew func(ctx context.Context, perms []Permission) ([]byte, error) `perm:"admin"` + AuthVerify func(ctx context.Context, token string) ([]api.Permission, error) `perm:"read"` + AuthNew func(ctx context.Context, perms []api.Permission) ([]byte, error) `perm:"admin"` NetConnectedness func(context.Context, peer.ID) (network.Connectedness, error) `perm:"read"` NetPeers func(context.Context) ([]peer.AddrInfo, error) `perm:"read"` @@ -29,8 +30,8 @@ type CommonStruct struct { NetAddrsListen func(context.Context) (peer.AddrInfo, error) `perm:"read"` NetDisconnect func(context.Context, peer.ID) error `perm:"write"` - ID func(context.Context) (peer.ID, error) `perm:"read"` - Version func(context.Context) (Version, error) `perm:"read"` + ID func(context.Context) (peer.ID, error) `perm:"read"` + Version func(context.Context) (api.Version, error) `perm:"read"` } } @@ -44,16 +45,16 @@ type FullNodeStruct struct { ChainGetRandomness func(context.Context, types.TipSetKey, int64) ([]byte, error) `perm:"read"` ChainGetBlock func(context.Context, cid.Cid) (*types.BlockHeader, error) `perm:"read"` ChainGetTipSet func(context.Context, types.TipSetKey) (*types.TipSet, error) `perm:"read"` - ChainGetBlockMessages func(context.Context, cid.Cid) (*BlockMessages, error) `perm:"read"` + ChainGetBlockMessages func(context.Context, cid.Cid) (*api.BlockMessages, error) `perm:"read"` ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error) `perm:"read"` - ChainGetParentMessages func(context.Context, cid.Cid) ([]Message, error) `perm:"read"` + ChainGetParentMessages func(context.Context, cid.Cid) ([]api.Message, error) `perm:"read"` ChainGetTipSetByHeight func(context.Context, uint64, *types.TipSet) (*types.TipSet, error) `perm:"read"` ChainReadObj func(context.Context, cid.Cid) ([]byte, error) `perm:"read"` ChainSetHead func(context.Context, *types.TipSet) error `perm:"admin"` ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"` ChainTipSetWeight func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` - SyncState func(context.Context) (*SyncState, error) `perm:"read"` + SyncState func(context.Context) (*api.SyncState, error) `perm:"read"` SyncSubmitBlock func(ctx context.Context, blk *types.BlockMsg) error `perm:"write"` SyncIncomingBlocks func(ctx context.Context) (<-chan *types.BlockHeader, error) `perm:"read"` @@ -61,7 +62,7 @@ type FullNodeStruct struct { MpoolPush func(context.Context, *types.SignedMessage) error `perm:"write"` MpoolPushMessage func(context.Context, *types.Message) (*types.SignedMessage, error) `perm:"sign"` MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"` - MpoolSub func(context.Context) (<-chan MpoolUpdate, error) `perm:"read"` + MpoolSub func(context.Context) (<-chan api.MpoolUpdate, error) `perm:"read"` MinerCreateBlock func(context.Context, address.Address, *types.TipSet, *types.Ticket, *types.EPostProof, []*types.SignedMessage, uint64, uint64) (*types.BlockMsg, error) `perm:"write"` @@ -77,28 +78,28 @@ type FullNodeStruct struct { WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` ClientImport func(ctx context.Context, path string) (cid.Cid, error) `perm:"admin"` - ClientListImports func(ctx context.Context) ([]Import, error) `perm:"write"` + ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` - ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"` + ClientFindData func(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) `perm:"read"` ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"` - ClientGetDealInfo func(context.Context, cid.Cid) (*DealInfo, error) `perm:"read"` - ClientListDeals func(ctx context.Context) ([]DealInfo, error) `perm:"write"` - ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"` + ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` + ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"` + ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, path string) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) `perm:"read"` - StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"` - StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"` - StateMinerPower func(context.Context, address.Address, *types.TipSet) (MinerPower, error) `perm:"read"` + StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) `perm:"read"` + StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) `perm:"read"` + StateMinerPower func(context.Context, address.Address, *types.TipSet) (api.MinerPower, error) `perm:"read"` StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"` StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"` StateMinerElectionPeriodStart func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) `perm:"read"` StateMinerSectorSize func(context.Context, address.Address, *types.TipSet) (uint64, error) `perm:"read"` StateCall func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"` - StateReplay func(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error) `perm:"read"` + StateReplay func(context.Context, *types.TipSet, cid.Cid) (*api.ReplayResults, error) `perm:"read"` StateGetActor func(context.Context, address.Address, *types.TipSet) (*types.Actor, error) `perm:"read"` - StateReadState func(context.Context, *types.Actor, *types.TipSet) (*ActorState, error) `perm:"read"` + StateReadState func(context.Context, *types.Actor, *types.TipSet) (*api.ActorState, error) `perm:"read"` StatePledgeCollateral func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` - StateWaitMsg func(context.Context, cid.Cid) (*MsgWait, error) `perm:"read"` + StateWaitMsg func(context.Context, cid.Cid) (*api.MsgWait, error) `perm:"read"` StateListMiners func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` StateListActors func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` StateMarketBalance func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"` @@ -111,19 +112,19 @@ type FullNodeStruct struct { MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"` - PaychGet func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) `perm:"sign"` - PaychList func(context.Context) ([]address.Address, error) `perm:"read"` - PaychStatus func(context.Context, address.Address) (*PaychStatus, error) `perm:"read"` - PaychClose func(context.Context, address.Address) (cid.Cid, error) `perm:"sign"` - PaychAllocateLane func(context.Context, address.Address) (uint64, error) `perm:"sign"` - PaychNewPayment func(ctx context.Context, from, to address.Address, vouchers []VoucherSpec) (*PaymentInfo, error) `perm:"sign"` - PaychVoucherCheck func(context.Context, *types.SignedVoucher) error `perm:"read"` - PaychVoucherCheckValid func(context.Context, address.Address, *types.SignedVoucher) error `perm:"read"` - PaychVoucherCheckSpendable func(context.Context, address.Address, *types.SignedVoucher, []byte, []byte) (bool, error) `perm:"read"` - PaychVoucherAdd func(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) `perm:"write"` + PaychGet func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error) `perm:"sign"` + PaychList func(context.Context) ([]address.Address, error) `perm:"read"` + PaychStatus func(context.Context, address.Address) (*api.PaychStatus, error) `perm:"read"` + PaychClose func(context.Context, address.Address) (cid.Cid, error) `perm:"sign"` + PaychAllocateLane func(context.Context, address.Address) (uint64, error) `perm:"sign"` + PaychNewPayment func(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) `perm:"sign"` + PaychVoucherCheck func(context.Context, *types.SignedVoucher) error `perm:"read"` + PaychVoucherCheckValid func(context.Context, address.Address, *types.SignedVoucher) error `perm:"read"` + PaychVoucherCheckSpendable func(context.Context, address.Address, *types.SignedVoucher, []byte, []byte) (bool, error) `perm:"read"` + PaychVoucherAdd func(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) `perm:"write"` PaychVoucherCreate func(context.Context, address.Address, types.BigInt, uint64) (*types.SignedVoucher, error) `perm:"sign"` PaychVoucherList func(context.Context, address.Address) ([]*types.SignedVoucher, error) `perm:"write"` - PaychVoucherSubmit func(context.Context, address.Address, *types.SignedVoucher) (cid.Cid, error) `perm:"sign"` + PaychVoucherSubmit func(context.Context, address.Address, *types.SignedVoucher) (cid.Cid, error) `perm:"sign"` } } @@ -136,10 +137,10 @@ type StorageMinerStruct struct { PledgeSector func(context.Context) error `perm:"write"` - SectorsStatus func(context.Context, uint64) (SectorInfo, error) `perm:"read"` - SectorsList func(context.Context) ([]uint64, error) `perm:"read"` - SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` - SectorsUpdate func(context.Context, uint64, SectorState) error `perm:"write"` + SectorsStatus func(context.Context, uint64) (api.SectorInfo, error) `perm:"read"` + SectorsList func(context.Context) ([]uint64, error) `perm:"read"` + SectorsRefs func(context.Context) (map[string][]api.SealedRef, error) `perm:"read"` + SectorsUpdate func(context.Context, uint64, api.SectorState) error `perm:"write"` WorkerStats func(context.Context) (sectorbuilder.WorkerStats, error) `perm:"read"` @@ -148,11 +149,11 @@ type StorageMinerStruct struct { } } -func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]Permission, error) { +func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]api.Permission, error) { return c.Internal.AuthVerify(ctx, token) } -func (c *CommonStruct) AuthNew(ctx context.Context, perms []Permission) ([]byte, error) { +func (c *CommonStruct) AuthNew(ctx context.Context, perms []api.Permission) ([]byte, error) { return c.Internal.AuthNew(ctx, perms) } @@ -182,11 +183,11 @@ func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) { } // Version implements API.Version -func (c *CommonStruct) Version(ctx context.Context) (Version, error) { +func (c *CommonStruct) Version(ctx context.Context) (api.Version, error) { return c.Internal.Version(ctx) } -func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]Import, error) { +func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]api.Import, error) { return c.Internal.ClientListImports(ctx) } @@ -198,22 +199,22 @@ func (c *FullNodeStruct) ClientHasLocal(ctx context.Context, root cid.Cid) (bool return c.Internal.ClientHasLocal(ctx, root) } -func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) { +func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) { return c.Internal.ClientFindData(ctx, root) } func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) { return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration) } -func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*DealInfo, error) { +func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*api.DealInfo, error) { return c.Internal.ClientGetDealInfo(ctx, deal) } -func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]DealInfo, error) { +func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { return c.Internal.ClientListDeals(ctx) } -func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error { +func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path string) error { return c.Internal.ClientRetrieve(ctx, order, path) } @@ -233,7 +234,7 @@ func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Messag return c.Internal.MpoolPushMessage(ctx, msg) } -func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan MpoolUpdate, error) { +func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan api.MpoolUpdate, error) { return c.Internal.MpoolSub(ctx) } @@ -305,7 +306,7 @@ func (c *FullNodeStruct) ChainGetTipSet(ctx context.Context, key types.TipSetKey return c.Internal.ChainGetTipSet(ctx, key) } -func (c *FullNodeStruct) ChainGetBlockMessages(ctx context.Context, b cid.Cid) (*BlockMessages, error) { +func (c *FullNodeStruct) ChainGetBlockMessages(ctx context.Context, b cid.Cid) (*api.BlockMessages, error) { return c.Internal.ChainGetBlockMessages(ctx, b) } @@ -313,7 +314,7 @@ func (c *FullNodeStruct) ChainGetParentReceipts(ctx context.Context, b cid.Cid) return c.Internal.ChainGetParentReceipts(ctx, b) } -func (c *FullNodeStruct) ChainGetParentMessages(ctx context.Context, b cid.Cid) ([]Message, error) { +func (c *FullNodeStruct) ChainGetParentMessages(ctx context.Context, b cid.Cid) ([]api.Message, error) { return c.Internal.ChainGetParentMessages(ctx, b) } @@ -337,7 +338,7 @@ func (c *FullNodeStruct) ChainTipSetWeight(ctx context.Context, ts *types.TipSet return c.Internal.ChainTipSetWeight(ctx, ts) } -func (c *FullNodeStruct) SyncState(ctx context.Context) (*SyncState, error) { +func (c *FullNodeStruct) SyncState(ctx context.Context) (*api.SyncState, error) { return c.Internal.SyncState(ctx) } @@ -349,15 +350,15 @@ func (c *FullNodeStruct) SyncIncomingBlocks(ctx context.Context) (<-chan *types. return c.Internal.SyncIncomingBlocks(ctx) } -func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) { +func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) { return c.Internal.StateMinerSectors(ctx, addr, ts) } -func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) { +func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) { return c.Internal.StateMinerProvingSet(ctx, addr, ts) } -func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, ts *types.TipSet) (MinerPower, error) { +func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, ts *types.TipSet) (api.MinerPower, error) { return c.Internal.StateMinerPower(ctx, a, ts) } @@ -381,7 +382,7 @@ func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, ts * return c.Internal.StateCall(ctx, msg, ts) } -func (c *FullNodeStruct) StateReplay(ctx context.Context, ts *types.TipSet, mc cid.Cid) (*ReplayResults, error) { +func (c *FullNodeStruct) StateReplay(ctx context.Context, ts *types.TipSet, mc cid.Cid) (*api.ReplayResults, error) { return c.Internal.StateReplay(ctx, ts, mc) } @@ -389,7 +390,7 @@ func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Addres return c.Internal.StateGetActor(ctx, actor, ts) } -func (c *FullNodeStruct) StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*ActorState, error) { +func (c *FullNodeStruct) StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*api.ActorState, error) { return c.Internal.StateReadState(ctx, act, ts) } @@ -397,7 +398,7 @@ func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, ts *types.Ti return c.Internal.StatePledgeCollateral(ctx, ts) } -func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid) (*MsgWait, error) { +func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid) (*api.MsgWait, error) { return c.Internal.StateWaitMsg(ctx, msgc) } func (c *FullNodeStruct) StateListMiners(ctx context.Context, ts *types.TipSet) ([]address.Address, error) { @@ -440,7 +441,7 @@ func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr address return c.Internal.MarketEnsureAvailable(ctx, addr, amt) } -func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) { +func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error) { return c.Internal.PaychGet(ctx, from, to, ensureFunds) } @@ -448,7 +449,7 @@ func (c *FullNodeStruct) PaychList(ctx context.Context) ([]address.Address, erro return c.Internal.PaychList(ctx) } -func (c *FullNodeStruct) PaychStatus(ctx context.Context, pch address.Address) (*PaychStatus, error) { +func (c *FullNodeStruct) PaychStatus(ctx context.Context, pch address.Address) (*api.PaychStatus, error) { return c.Internal.PaychStatus(ctx, pch) } @@ -480,7 +481,7 @@ func (c *FullNodeStruct) PaychAllocateLane(ctx context.Context, ch address.Addre return c.Internal.PaychAllocateLane(ctx, ch) } -func (c *FullNodeStruct) PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []VoucherSpec) (*PaymentInfo, error) { +func (c *FullNodeStruct) PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) { return c.Internal.PaychNewPayment(ctx, from, to, vouchers) } @@ -501,7 +502,7 @@ func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error { } // Get the status of a given sector by ID -func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (SectorInfo, error) { +func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (api.SectorInfo, error) { return c.Internal.SectorsStatus(ctx, sid) } @@ -510,11 +511,11 @@ func (c *StorageMinerStruct) SectorsList(ctx context.Context) ([]uint64, error) return c.Internal.SectorsList(ctx) } -func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]SealedRef, error) { +func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]api.SealedRef, error) { return c.Internal.SectorsRefs(ctx) } -func (c *StorageMinerStruct) SectorsUpdate(ctx context.Context, id uint64, state SectorState) error { +func (c *StorageMinerStruct) SectorsUpdate(ctx context.Context, id uint64, state api.SectorState) error { return c.Internal.SectorsUpdate(ctx, id, state) } @@ -530,6 +531,6 @@ func (c *StorageMinerStruct) WorkerDone(ctx context.Context, task uint64, res se return c.Internal.WorkerDone(ctx, task, res) } -var _ Common = &CommonStruct{} -var _ FullNode = &FullNodeStruct{} -var _ StorageMiner = &StorageMinerStruct{} +var _ api.Common = &CommonStruct{} +var _ api.FullNode = &FullNodeStruct{} +var _ api.StorageMiner = &StorageMinerStruct{} diff --git a/api/client/client.go b/api/client/client.go index ffca93363..0e19f65c2 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -1,6 +1,7 @@ package client import ( + "github.com/filecoin-project/lotus/api/apistruct" "net/http" "github.com/filecoin-project/lotus/api" @@ -9,7 +10,7 @@ import ( // NewCommonRPC creates a new http jsonrpc client. func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) { - var res api.CommonStruct + var res apistruct.CommonStruct closer, err := jsonrpc.NewMergeClient(addr, "Filecoin", []interface{}{ &res.Internal, @@ -20,7 +21,7 @@ func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.C // NewFullNodeRPC creates a new http jsonrpc client. func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) { - var res api.FullNodeStruct + var res apistruct.FullNodeStruct closer, err := jsonrpc.NewMergeClient(addr, "Filecoin", []interface{}{ &res.CommonStruct.Internal, @@ -32,7 +33,7 @@ func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonr // NewStorageMinerRPC creates a new http jsonrpc client for storage miner func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) { - var res api.StorageMinerStruct + var res apistruct.StorageMinerStruct closer, err := jsonrpc.NewMergeClient(addr, "Filecoin", []interface{}{ &res.CommonStruct.Internal, diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 6c775eac0..4e1ba5570 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -53,14 +53,10 @@ type StorageMinerActorState struct { // Contains mostly static info about this miner Info cid.Cid - // Faulty sectors reported since last SubmitPost, - // up to the current proving period's challenge time. - CurrentFaultSet types.BitField + // Faulty sectors reported since last SubmitPost + FaultSet types.BitField - // Faults submitted after the current proving period's challenge time, - // but before the PoSt for that period is submitted. - // These become the currentFaultSet when a PoSt is submitted. - NextFaultSet types.BitField + LastFaultSubmission uint64 // Amount of power this miner has. Power types.BigInt @@ -340,7 +336,7 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC return nil, aerrors.Wrapf(err, "failed to compute data commitment (sector %d, deals: %v)", params.SectorID, params.DealIDs) } - if ok, err := ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { + if ok, err := vmctx.Sys().ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { return nil, err } else if !ok { return nil, aerrors.Newf(2, "porep proof was invalid (t:%x; s:%x(%d); p:%s)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, truncateHexPrint(params.Proof)) @@ -463,8 +459,17 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.HandleExternalError(lerr, "could not load proving set node") } + faults, nerr := self.FaultSet.AllMap() + if nerr != nil { + return nil, aerrors.Absorb(err, 5, "RLE+ invalid") + } + var sectorInfos []ffi.PublicSectorInfo if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error { + if faults[id] { + return nil + } + var comms [][]byte if err := cbor.DecodeInto(v.Raw, &comms); err != nil { return xerrors.New("could not decode comms") @@ -485,12 +490,6 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.Absorb(err, 3, "could not decode sectorset") } - faults, nerr := self.CurrentFaultSet.All() - if nerr != nil { - return nil, aerrors.Absorb(err, 5, "RLE+ invalid") - } - _ = faults - proverID := vmctx.Message().To // TODO: normalize to ID address var candidates []sectorbuilder.EPostCandidate @@ -598,6 +597,25 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID return true, comms[0], comms[1], nil } +func RemoveFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, ids []uint64) (cid.Cid, aerrors.ActorError) { + + ssr, err := amt.LoadAMT(types.WrapStorage(s), ss) + if err != nil { + return cid.Undef, aerrors.HandleExternalError(err, "could not load sector set node") + } + + if err := ssr.BatchDelete(ids); err != nil { + return cid.Undef, aerrors.HandleExternalError(err, "failed to delete from sector set") + } + + ncid, err := ssr.Flush() + if err != nil { + return cid.Undef, aerrors.HandleExternalError(err, "failed to flush sector set") + } + + return ncid, nil +} + func ValidatePoRep(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, ActorError) { _, span := trace.StartSpan(ctx, "ValidatePoRep") defer span.End() @@ -787,34 +805,28 @@ type DeclareFaultsParams struct { } func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) { - /* - oldstate, self, aerr := loadState(vmctx) - if aerr != nil { - return nil, aerr - } + oldstate, self, aerr := loadState(vmctx) + if aerr != nil { + return nil, aerr + } - challengeHeight := self.ProvingPeriodEnd - build.PoStChallangeTime + nfaults, err := types.MergeBitFields(params.Faults, self.FaultSet) + if err != nil { + return nil, aerrors.Absorb(err, 1, "failed to merge bitfields") + } - if vmctx.BlockHeight() < challengeHeight { - // TODO: optimized bitfield methods - for _, v := range params.Faults.All() { - self.CurrentFaultSet.Set(v) - } - } else { - for _, v := range params.Faults.All() { - self.NextFaultSet.Set(v) - } - } + self.FaultSet = nfaults - nstate, err := vmctx.Storage().Put(self) - if err != nil { - return nil, err - } - if err := vmctx.Storage().Commit(oldstate, nstate); err != nil { - return nil, err - } + self.LastFaultSubmission = vmctx.BlockHeight() + + nstate, aerr := vmctx.Storage().Put(self) + if err != nil { + return nil, aerr + } + if err := vmctx.Storage().Commit(oldstate, nstate); err != nil { + return nil, err + } - */ return nil, nil } @@ -906,10 +918,12 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerro return aerrors.HandleExternalError(nerr, "failed to load proving set") } - self.CurrentFaultSet = self.NextFaultSet - self.NextFaultSet = types.NewBitField() + faults, nerr := self.FaultSet.All() + if nerr != nil { + return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)") + } - faults := []uint64{} // TODO + self.FaultSet = types.NewBitField() oldPower := self.Power self.Power = types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), @@ -941,7 +955,13 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerro return err } - self.ProvingSet = self.Sectors + ncid, err := RemoveFromSectorSet(vmctx.Context(), vmctx.Storage(), self.Sectors, faults) + if err != nil { + return err + } + + self.Sectors = ncid + self.ProvingSet = ncid self.ElectionPeriodStart = vmctx.BlockHeight() return nil } diff --git a/chain/actors/actor_miner_test.go b/chain/actors/actor_miner_test.go new file mode 100644 index 000000000..28a5ce046 --- /dev/null +++ b/chain/actors/actor_miner_test.go @@ -0,0 +1,157 @@ +package actors_test + +import ( + "bytes" + "context" + "math/rand" + "testing" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/aerrors" + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" + hamt "github.com/ipfs/go-hamt-ipld" + blockstore "github.com/ipfs/go-ipfs-blockstore" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func TestMinerCommitSectors(t *testing.T) { + var worker, client address.Address + var minerAddr address.Address + opts := []HarnessOpt{ + HarnessAddr(&worker, 1000000), + HarnessAddr(&client, 1000000), + HarnessActor(&minerAddr, &worker, actors.StorageMinerCodeCid, + func() cbg.CBORMarshaler { + return &actors.StorageMinerConstructorParams{ + Owner: worker, + Worker: worker, + SectorSize: 1024, + PeerID: "fakepeerid", + } + }), + } + + h := NewHarness(t, opts...) + h.vm.Syscalls.ValidatePoRep = func(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, aerrors.ActorError) { + // all proofs are valid + return true, nil + } + + ret, _ := h.SendFunds(t, worker, minerAddr, types.NewInt(100000)) + ApplyOK(t, ret) + + ret, _ = h.InvokeWithValue(t, client, actors.StorageMarketAddress, actors.SMAMethods.AddBalance, types.NewInt(2000), nil) + ApplyOK(t, ret) + + addSectorToMiner(h, t, minerAddr, worker, client, 1) + + assertSectorIDs(h, t, minerAddr, []uint64{1}) +} + +func addSectorToMiner(h *Harness, t *testing.T, minerAddr, worker, client address.Address, sid uint64) { + t.Helper() + s := sectorbuilder.UserBytesForSectorSize(1024) + deal := h.makeFakeDeal(t, minerAddr, worker, client, s) + ret, _ := h.Invoke(t, worker, actors.StorageMarketAddress, actors.SMAMethods.PublishStorageDeals, + &actors.PublishStorageDealsParams{ + Deals: []actors.StorageDealProposal{*deal}, + }) + ApplyOK(t, ret) + var dealIds actors.PublishStorageDealResponse + if err := dealIds.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil { + t.Fatal(err) + } + + dealid := dealIds.DealIDs[0] + + ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.PreCommitSector, + &actors.SectorPreCommitInfo{ + SectorNumber: sid, + CommR: []byte("cats"), + SealEpoch: 10, + DealIDs: []uint64{dealid}, + }) + ApplyOK(t, ret) + + h.BlockHeight += 100 + ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.ProveCommitSector, + &actors.SectorProveCommitInfo{ + Proof: []byte("prooofy"), + SectorID: sid, + DealIDs: []uint64{dealid}, // TODO: weird that i have to pass this again + }) + ApplyOK(t, ret) +} + +func assertSectorIDs(h *Harness, t *testing.T, maddr address.Address, ids []uint64) { + t.Helper() + sectors, err := getMinerSectorSet(context.TODO(), h.vm.StateTree(), h.bs, maddr) + if err != nil { + t.Fatal(err) + } + + if len(sectors) != len(ids) { + t.Fatal("miner has wrong number of sectors in their sector set") + } + + all := make(map[uint64]bool) + for _, s := range sectors { + all[s.SectorID] = true + } + + for _, id := range ids { + if !all[id] { + t.Fatal("expected to find sector ID: ", id) + } + } +} + +func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) { + mact, err := st.GetActor(maddr) + if err != nil { + return nil, err + } + + cst := hamt.CSTFromBstore(bs) + + var mstate actors.StorageMinerActorState + if err := cst.Get(ctx, mact.Head, &mstate); err != nil { + return nil, err + } + + return stmgr.LoadSectorsFromSet(ctx, bs, mstate.Sectors) +} + +func (h *Harness) makeFakeDeal(t *testing.T, miner, worker, client address.Address, size uint64) *actors.StorageDealProposal { + data := make([]byte, size) + rand.Read(data) + commP, err := sectorbuilder.GeneratePieceCommitment(bytes.NewReader(data), size) + if err != nil { + t.Fatal(err) + } + + prop := actors.StorageDealProposal{ + PieceRef: commP[:], + PieceSize: size, + //PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid + + Client: client, + Provider: miner, + + ProposalExpiration: 10000, + Duration: 150, + + StoragePricePerEpoch: types.NewInt(1), + StorageCollateral: types.NewInt(0), + } + + if err := api.SignWith(context.TODO(), h.w.Sign, client, &prop); err != nil { + t.Fatal(err) + } + + return &prop +} diff --git a/chain/actors/actor_paych_test.go b/chain/actors/actor_paych_test.go index e23acdf4a..55de306c0 100644 --- a/chain/actors/actor_paych_test.go +++ b/chain/actors/actor_paych_test.go @@ -83,7 +83,7 @@ func TestPaychUpdate(t *testing.T) { ApplyOK(t, ret) // now we have to 'wait' for the chain to advance. - h.vm.SetBlockHeight(1000) + h.BlockHeight = 1000 ret, _ = h.Invoke(t, targetAddr, pch, actors.PCAMethods.Collect, nil) ApplyOK(t, ret) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 651fd8f71..dd3e78bec 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -3,6 +3,8 @@ package actors import ( "bytes" "context" + "sort" + "golang.org/x/xerrors" "github.com/filecoin-project/go-amt-ipld" @@ -73,9 +75,8 @@ const ( ) type StorageDealProposal struct { - PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea - PieceSize uint64 - PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid + PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea + PieceSize uint64 Client address.Address Provider address.Address @@ -131,36 +132,19 @@ func (sdp *StorageDealProposal) Verify() error { return sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()) } -func (d *StorageDeal) Sign(ctx context.Context, sign SignFunc) error { - var buf bytes.Buffer - if err := d.Proposal.MarshalCBOR(&buf); err != nil { - return err - } - sig, err := sign(ctx, buf.Bytes()) - if err != nil { - return err - } - d.CounterSignature = sig - return nil -} - -func (d *StorageDeal) Verify(proposerWorker address.Address) error { - var buf bytes.Buffer - if err := d.Proposal.MarshalCBOR(&buf); err != nil { - return err - } - - return d.CounterSignature.Verify(proposerWorker, buf.Bytes()) -} - -type StorageDeal struct { - Proposal StorageDealProposal - CounterSignature *types.Signature -} - type OnChainDeal struct { - Deal StorageDeal - ActivationEpoch uint64 // 0 = inactive + PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea + PieceSize uint64 + + Client address.Address + Provider address.Address + + ProposalExpiration uint64 + Duration uint64 // TODO: spec + + StoragePricePerEpoch types.BigInt + StorageCollateral types.BigInt + ActivationEpoch uint64 // 0 = inactive } type WithdrawBalanceParams struct { @@ -245,8 +229,15 @@ func (sma StorageMarketActor) AddBalance(act *types.Actor, vmctx types.VMContext } func setMarketBalances(vmctx types.VMContext, nd *hamt.Node, set map[address.Address]StorageParticipantBalance) (cid.Cid, ActorError) { - for addr, b := range set { - balance := b // to stop linter complaining + keys := make([]address.Address, 0, len(set)) + for k := range set { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + return bytes.Compare(keys[i].Bytes(), keys[j].Bytes()) < 0 + }) + for _, addr := range keys { + balance := set[addr] if err := nd.Set(vmctx.Context(), string(addr.Bytes()), &balance); err != nil { return cid.Undef, aerrors.HandleExternalError(err, "setting new balance") } @@ -297,7 +288,7 @@ func (sma StorageMarketActor) CheckLockedBalance(act *types.Actor, vmctx types.V */ type PublishStorageDealsParams struct { - Deals []StorageDeal + Deals []StorageDealProposal } type PublishStorageDealResponse struct { @@ -326,7 +317,7 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. DealIDs: make([]uint64, len(params.Deals)), } - workerBytes, aerr := vmctx.Send(params.Deals[0].Proposal.Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil) + workerBytes, aerr := vmctx.Send(params.Deals[0].Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil) if aerr != nil { return nil, aerr } @@ -342,7 +333,20 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. return nil, err } - err := deals.Set(self.NextDealID, &OnChainDeal{Deal: deal}) + err := deals.Set(self.NextDealID, &OnChainDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + + Client: deal.Client, + Provider: deal.Provider, + + ProposalExpiration: deal.ProposalExpiration, + Duration: deal.Duration, + + StoragePricePerEpoch: deal.StoragePricePerEpoch, + StorageCollateral: deal.StorageCollateral, + ActivationEpoch: 0, + }) if err != nil { return nil, aerrors.HandleExternalError(err, "setting deal in deal AMT") } @@ -376,34 +380,28 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. return outBuf.Bytes(), nil } -func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDeal, providerWorker address.Address) aerrors.ActorError { - if vmctx.BlockHeight() > deal.Proposal.ProposalExpiration { +func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDealProposal, providerWorker address.Address) aerrors.ActorError { + if vmctx.BlockHeight() > deal.ProposalExpiration { return aerrors.New(1, "deal proposal already expired") } - if err := deal.Proposal.Verify(); err != nil { - return aerrors.Absorb(err, 2, "verifying proposer signature") + if vmctx.Message().From != providerWorker { + return aerrors.New(2, "Deals must be submitted by the miner worker") } - err := deal.Verify(providerWorker) - if err != nil { - return aerrors.Absorb(err, 2, "verifying provider signature") - } - - // TODO: maybe this is actually fine - if vmctx.Message().From != providerWorker && vmctx.Message().From != deal.Proposal.Client { - return aerrors.New(4, "message not sent by deal participant") + if err := deal.Verify(); err != nil { + return aerrors.Absorb(err, 3, "verifying proposer signature") } // TODO: do some caching (changes gas so needs to be in spec too) - b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), st.Balances, deal.Proposal.Client, providerWorker) + b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), st.Balances, deal.Client, providerWorker) if aerr != nil { return aerrors.Wrap(aerr, "getting client, and provider balances") } clientBalance := b[0] providerBalance := b[1] - totalPrice := deal.Proposal.TotalStoragePrice() + totalPrice := deal.TotalStoragePrice() if clientBalance.Available.LessThan(totalPrice) { return aerrors.Newf(5, "client doesn't have enough available funds to cover storage price; %d < %d", clientBalance.Available, totalPrice) @@ -412,17 +410,17 @@ func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDe clientBalance = lockFunds(clientBalance, totalPrice) // TODO: REVIEW: Not clear who pays for this - if providerBalance.Available.LessThan(deal.Proposal.StorageCollateral) { - return aerrors.Newf(6, "provider doesn't have enough available funds to cover StorageCollateral; %d < %d", providerBalance.Available, deal.Proposal.StorageCollateral) + if providerBalance.Available.LessThan(deal.StorageCollateral) { + return aerrors.Newf(6, "provider doesn't have enough available funds to cover StorageCollateral; %d < %d", providerBalance.Available, deal.StorageCollateral) } - providerBalance = lockFunds(providerBalance, deal.Proposal.StorageCollateral) + providerBalance = lockFunds(providerBalance, deal.StorageCollateral) // TODO: piece checks (e.g. size > sectorSize)? bcid, aerr := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ - deal.Proposal.Client: clientBalance, - providerWorker: providerBalance, + deal.Client: clientBalance, + providerWorker: providerBalance, }) if aerr != nil { return aerr @@ -458,11 +456,11 @@ func (sma StorageMarketActor) ActivateStorageDeals(act *types.Actor, vmctx types return nil, aerrors.HandleExternalError(err, "getting deal info failed") } - if vmctx.Message().From != dealInfo.Deal.Proposal.Provider { + if vmctx.Message().From != dealInfo.Provider { return nil, aerrors.New(1, "ActivateStorageDeals can only be called by the deal provider") } - if vmctx.BlockHeight() > dealInfo.Deal.Proposal.ProposalExpiration { + if vmctx.BlockHeight() > dealInfo.ProposalExpiration { return nil, aerrors.New(2, "deal cannot be activated: proposal expired") } @@ -533,7 +531,7 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx return nil, aerrors.HandleExternalError(err, "getting deal info failed") } - if dealInfo.Deal.Proposal.Provider != vmctx.Message().From { + if dealInfo.Provider != vmctx.Message().From { return nil, aerrors.New(3, "ProcessStorageDealsPayment can only be called by deal provider") } @@ -542,15 +540,15 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx return nil, aerrors.New(4, "ActivationEpoch lower than block height") } - if vmctx.BlockHeight() > dealInfo.ActivationEpoch+dealInfo.Deal.Proposal.Duration { + if vmctx.BlockHeight() > dealInfo.ActivationEpoch+dealInfo.Duration { // Deal expired, miner should drop it // TODO: process payment for the remainder of last proving period return nil, nil } - toPay := types.BigMul(dealInfo.Deal.Proposal.StoragePricePerEpoch, types.NewInt(build.SlashablePowerDelay)) + toPay := types.BigMul(dealInfo.StoragePricePerEpoch, types.NewInt(build.SlashablePowerDelay)) - b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Deal.Proposal.Client, providerWorker) + b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Client, providerWorker) if aerr != nil { return nil, aerr } @@ -561,8 +559,8 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx // TODO: call set once bcid, aerr := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ - dealInfo.Deal.Proposal.Client: clientBal, - providerWorker: providerBal, + dealInfo.Client: clientBal, + providerWorker: providerBal, }) if aerr != nil { return nil, aerr @@ -625,15 +623,15 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type return nil, aerrors.HandleExternalError(err, "getting deal info failed") } - if dealInfo.Deal.Proposal.Provider != vmctx.Message().From { + if dealInfo.Provider != vmctx.Message().From { return nil, aerrors.New(5, "referenced deal was not from caller") } var commP [32]byte - copy(commP[:], dealInfo.Deal.Proposal.PieceRef) + copy(commP[:], dealInfo.PieceRef) pieces = append(pieces, sectorbuilder.PublicPieceInfo{ - Size: dealInfo.Deal.Proposal.PieceSize, + Size: dealInfo.PieceSize, CommP: commP, }) } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 6c8577e33..83582a44b 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -248,13 +248,13 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Info: %w", err) } - // t.CurrentFaultSet (types.BitField) (struct) - if err := t.CurrentFaultSet.MarshalCBOR(w); err != nil { + // t.FaultSet (types.BitField) (struct) + if err := t.FaultSet.MarshalCBOR(w); err != nil { return err } - // t.NextFaultSet (types.BitField) (struct) - if err := t.NextFaultSet.MarshalCBOR(w); err != nil { + // t.LastFaultSubmission (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.LastFaultSubmission))); err != nil { return err } @@ -384,24 +384,25 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { t.Info = c } - // t.CurrentFaultSet (types.BitField) (struct) + // t.FaultSet (types.BitField) (struct) { - if err := t.CurrentFaultSet.UnmarshalCBOR(br); err != nil { + if err := t.FaultSet.UnmarshalCBOR(br); err != nil { return err } } - // t.NextFaultSet (types.BitField) (struct) - - { - - if err := t.NextFaultSet.UnmarshalCBOR(br); err != nil { - return err - } + // t.LastFaultSubmission (uint64) (uint64) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.LastFaultSubmission = uint64(extra) // t.Power (types.BigInt) (struct) { @@ -1031,6 +1032,49 @@ func (t *UpdatePeerIDParams) UnmarshalCBOR(r io.Reader) error { return nil } +func (t *DeclareFaultsParams) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.Faults (types.BitField) (struct) + if err := t.Faults.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *DeclareFaultsParams) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.Faults (types.BitField) (struct) + + { + + if err := t.Faults.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} + func (t *MultiSigActorState) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -3166,7 +3210,7 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{138}); err != nil { + if _, err := w.Write([]byte{137}); err != nil { return err } @@ -3183,11 +3227,6 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.PieceSerialization (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSerialization))); err != nil { - return err - } - // t.Client (address.Address) (struct) if err := t.Client.MarshalCBOR(w); err != nil { return err @@ -3236,7 +3275,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 10 { + if extra != 9 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -3267,16 +3306,6 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.PieceSize = uint64(extra) - // t.PieceSerialization (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.PieceSerialization = uint64(extra) // t.Client (address.Address) (struct) { @@ -3357,75 +3386,6 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *StorageDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Proposal (actors.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - - // t.CounterSignature (types.Signature) (struct) - if err := t.CounterSignature.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Proposal (actors.StorageDealProposal) (struct) - - { - - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.CounterSignature (types.Signature) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.CounterSignature = new(types.Signature) - if err := t.CounterSignature.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - func (t *PublishStorageDealsParams) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -3435,7 +3395,7 @@ func (t *PublishStorageDealsParams) MarshalCBOR(w io.Writer) error { return err } - // t.Deals ([]actors.StorageDeal) (slice) + // t.Deals ([]actors.StorageDealProposal) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Deals)))); err != nil { return err } @@ -3462,7 +3422,7 @@ func (t *PublishStorageDealsParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Deals ([]actors.StorageDeal) (slice) + // t.Deals ([]actors.StorageDealProposal) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3477,11 +3437,11 @@ func (t *PublishStorageDealsParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("expected cbor array") } if extra > 0 { - t.Deals = make([]StorageDeal, extra) + t.Deals = make([]StorageDealProposal, extra) } for i := 0; i < int(extra); i++ { - var v StorageDeal + var v StorageDealProposal if err := v.UnmarshalCBOR(br); err != nil { return err } @@ -3707,12 +3667,50 @@ func (t *OnChainDeal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{130}); err != nil { + if _, err := w.Write([]byte{137}); err != nil { return err } - // t.Deal (actors.StorageDeal) (struct) - if err := t.Deal.MarshalCBOR(w); err != nil { + // t.PieceRef ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PieceRef)))); err != nil { + return err + } + if _, err := w.Write(t.PieceRef); err != nil { + return err + } + + // t.PieceSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSize))); err != nil { + return err + } + + // t.Client (address.Address) (struct) + if err := t.Client.MarshalCBOR(w); err != nil { + return err + } + + // t.Provider (address.Address) (struct) + if err := t.Provider.MarshalCBOR(w); err != nil { + return err + } + + // t.ProposalExpiration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { + return err + } + + // t.Duration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { + return err + } + + // t.StoragePricePerEpoch (types.BigInt) (struct) + if err := t.StoragePricePerEpoch.MarshalCBOR(w); err != nil { + return err + } + + // t.StorageCollateral (types.BigInt) (struct) + if err := t.StorageCollateral.MarshalCBOR(w); err != nil { return err } @@ -3734,15 +3732,89 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 2 { + if extra != 9 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.Deal (actors.StorageDeal) (struct) + // t.PieceRef ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.PieceRef: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.PieceRef = make([]byte, extra) + if _, err := io.ReadFull(br, t.PieceRef); err != nil { + return err + } + // t.PieceSize (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.PieceSize = uint64(extra) + // t.Client (address.Address) (struct) { - if err := t.Deal.UnmarshalCBOR(br); err != nil { + if err := t.Client.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.Provider (address.Address) (struct) + + { + + if err := t.Provider.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.ProposalExpiration (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.ProposalExpiration = uint64(extra) + // t.Duration (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Duration = uint64(extra) + // t.StoragePricePerEpoch (types.BigInt) (struct) + + { + + if err := t.StoragePricePerEpoch.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.StorageCollateral (types.BigInt) (struct) + + { + + if err := t.StorageCollateral.UnmarshalCBOR(br); err != nil { return err } diff --git a/chain/actors/harness2_test.go b/chain/actors/harness2_test.go index 0ffc02b1b..6f1da8d58 100644 --- a/chain/actors/harness2_test.go +++ b/chain/actors/harness2_test.go @@ -3,6 +3,7 @@ package actors_test import ( "bytes" "context" + "math/rand" "testing" "github.com/ipfs/go-cid" @@ -41,10 +42,12 @@ const ( type HarnessOpt func(testing.TB, *Harness) error type Harness struct { - HI HarnessInit - Stage HarnessStage - Nonces map[address.Address]uint64 - GasCharges map[address.Address]types.BigInt + HI HarnessInit + Stage HarnessStage + Nonces map[address.Address]uint64 + GasCharges map[address.Address]types.BigInt + Rand vm.Rand + BlockHeight uint64 lastBalanceCheck map[address.Address]types.BigInt @@ -127,6 +130,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness { h := &Harness{ Stage: HarnessPreInit, Nonces: make(map[address.Address]uint64), + Rand: &fakeRand{}, HI: HarnessInit{ NAddrs: 1, Miner: blsaddr(0), @@ -140,6 +144,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness { w: w, ctx: context.Background(), bs: bstore.NewBlockstore(dstore.NewMapDatastore()), + BlockHeight: 0, } for _, opt := range options { err := opt(t, h) @@ -157,8 +162,14 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness { if err != nil { t.Fatal(err) } + + stateroot, err = gen.SetupStorageMarketActor(h.bs, stateroot, nil) + if err != nil { + t.Fatal(err) + } + h.cs = store.NewChainStore(h.bs, nil) - h.vm, err = vm.NewVM(stateroot, 1, nil, h.HI.Miner, h.cs.Blockstore()) + h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore()) if err != nil { t.Fatal(err) } @@ -246,6 +257,7 @@ func (h *Harness) Invoke(t testing.TB, from address.Address, to address.Address, func (h *Harness) InvokeWithValue(t testing.TB, from address.Address, to address.Address, method uint64, value types.BigInt, params cbg.CBORMarshaler) (*vm.ApplyRet, *state.StateTree) { t.Helper() + h.vm.SetBlockHeight(h.BlockHeight) return h.Apply(t, types.Message{ To: to, From: from, @@ -315,3 +327,11 @@ func DumpObject(t testing.TB, obj cbg.CBORMarshaler) []byte { } return b.Bytes() } + +type fakeRand struct{} + +func (fr *fakeRand) GetRandomness(ctx context.Context, h int64) ([]byte, error) { + out := make([]byte, 32) + rand.New(rand.NewSource(h)).Read(out) + return out, nil +} diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index fee662f8c..5d175fc72 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -192,7 +192,7 @@ func (t *Response) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{133}); err != nil { + if _, err := w.Write([]byte{132}); err != nil { return err } @@ -215,23 +215,10 @@ func (t *Response) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) } - // t.StorageDeal (actors.StorageDeal) (struct) - if err := t.StorageDeal.MarshalCBOR(w); err != nil { + // t.StorageDealSubmission (types.SignedMessage) (struct) + if err := t.StorageDealSubmission.MarshalCBOR(w); err != nil { return err } - - // t.PublishMessage (cid.Cid) (struct) - - if t.PublishMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) - } - } - return nil } @@ -246,7 +233,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 5 { + if extra != 4 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -282,7 +269,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { t.Proposal = c } - // t.StorageDeal (actors.StorageDeal) (struct) + // t.StorageDealSubmission (types.SignedMessage) (struct) { @@ -296,36 +283,12 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { return err } } else { - t.StorageDeal = new(actors.StorageDeal) - if err := t.StorageDeal.UnmarshalCBOR(br); err != nil { + t.StorageDealSubmission = new(types.SignedMessage) + if err := t.StorageDealSubmission.UnmarshalCBOR(br); err != nil { return err } } - } - // t.PublishMessage (cid.Cid) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) - } - - t.PublishMessage = &c - } - } return nil } @@ -593,18 +556,10 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { return err } - // t.PublishMessage (cid.Cid) (struct) - - if t.PublishMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) - } + // t.PublishMessage (types.SignedMessage) (struct) + if err := t.PublishMessage.MarshalCBOR(w); err != nil { + return err } - return nil } @@ -683,7 +638,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.DealID = uint64(extra) - // t.PublishMessage (cid.Cid) (struct) + // t.PublishMessage (types.SignedMessage) (struct) { @@ -697,13 +652,10 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return err } } else { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) + t.PublishMessage = new(types.SignedMessage) + if err := t.PublishMessage.UnmarshalCBOR(br); err != nil { + return err } - - t.PublishMessage = &c } } diff --git a/chain/deals/client.go b/chain/deals/client.go index 988bc52b6..8580a2bbd 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -36,7 +36,7 @@ type ClientDeal struct { MinerWorker address.Address DealID uint64 - PublishMessage *cid.Cid + PublishMessage *types.SignedMessage s inet.Stream } @@ -206,7 +206,6 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro dealProposal := &actors.StorageDealProposal{ PieceRef: commP, PieceSize: uint64(pieceSize), - PieceSerialization: actors.SerializationUnixFSv0, Client: p.Client, Provider: p.ProviderAddress, ProposalExpiration: p.ProposalExpiration, diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index bac7aab47..7ee93b188 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -44,6 +44,9 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), e if err != nil { return nil, err } + + // TODO: verify StorageDealSubmission + if err := c.disconnect(deal); err != nil { return nil, err } @@ -54,18 +57,14 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), e } return func(info *ClientDeal) { - info.PublishMessage = resp.PublishMessage + info.PublishMessage = resp.StorageDealSubmission }, nil } func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { log.Infow("DEAL ACCEPTED!") - pubmsg, err := c.chain.GetMessage(*deal.PublishMessage) - if err != nil { - return nil, xerrors.Errorf("getting deal pubsish message: %w", err) - } - + pubmsg := deal.PublishMessage.Message pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) if err != nil { return nil, xerrors.Errorf("getting miner worker failed: %w", err) @@ -91,7 +90,7 @@ func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDea dealIdx := -1 for i, storageDeal := range params.Deals { // TODO: make it less hacky - eq, err := cborutil.Equals(&deal.Proposal, &storageDeal.Proposal) + eq, err := cborutil.Equals(&deal.Proposal, &storageDeal) if err != nil { return nil, err } @@ -102,11 +101,11 @@ func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDea } if dealIdx == -1 { - return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage) + return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage.Cid()) } // TODO: timeout - _, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage) + _, ret, err := c.sm.WaitForMessage(ctx, deal.PublishMessage.Cid()) if err != nil { return nil, xerrors.Errorf("waiting for deal publish message: %w", err) } diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 56843e7d0..4faa44973 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -42,13 +42,6 @@ func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandle // ACCEPTED func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - switch deal.Proposal.PieceSerialization { - //case SerializationRaw: - //case SerializationIPLD: - case actors.SerializationUnixFSv0: - default: - return nil, xerrors.Errorf("deal proposal with unsupported serialization: %s", deal.Proposal.PieceSerialization) - } head, err := p.full.ChainHead(ctx) if err != nil { @@ -93,15 +86,8 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) log.Info("publishing deal") - storageDeal := actors.StorageDeal{ - Proposal: deal.Proposal, - } - if err := api.SignWith(ctx, p.full.WalletSign, waddr, &storageDeal); err != nil { - return nil, xerrors.Errorf("signing storage deal failed: ", err) - } - params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ - Deals: []actors.StorageDeal{storageDeal}, + Deals: []actors.StorageDealProposal{deal.Proposal}, }) if err != nil { return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err) @@ -136,13 +122,11 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) } log.Infof("fetching data for a deal %d", resp.DealIDs[0]) - mcid := smsg.Cid() err = p.sendSignedResponse(&Response{ State: api.DealAccepted, - Proposal: deal.ProposalCid, - PublishMessage: &mcid, - StorageDeal: &storageDeal, + Proposal: deal.ProposalCid, + StorageDealSubmission: smsg, }) if err != nil { return nil, err diff --git a/chain/deals/types.go b/chain/deals/types.go index 3ea80ebe7..eebca4f60 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -58,8 +58,7 @@ type Response struct { Proposal cid.Cid // DealAccepted - StorageDeal *actors.StorageDeal - PublishMessage *cid.Cid + StorageDealSubmission *types.SignedMessage } // TODO: Do we actually need this to be signed? diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 866a75357..8c82895f9 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -526,9 +526,11 @@ func IsRoundWinner(ctx context.Context, ts *types.TipSet, round int64, miner add return nil, xerrors.Errorf("failed to look up miners sector size: %w", err) } + snum := types.BigDiv(pow.MinerPower, types.NewInt(ssize)) + var winners []sectorbuilder.EPostCandidate for _, c := range candidates { - if types.IsTicketWinner(c.PartialTicket[:], ssize, pow.TotalPower) { + if types.IsTicketWinner(c.PartialTicket[:], ssize, snum.Uint64(), pow.TotalPower) { winners = append(winners, c) } } diff --git a/chain/gen/utils.go b/chain/gen/utils.go index 34cdce95b..e4da3cde2 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -197,7 +197,7 @@ func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) { }, nil } -func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors.StorageDeal) (cid.Cid, error) { +func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors.StorageDealProposal) (cid.Cid, error) { cst := hamt.CSTFromBstore(bs) nd := hamt.NewNode(cst) emptyHAMT, err := cst.Put(context.TODO(), nd) @@ -210,8 +210,15 @@ func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors cdeals := make([]cbg.CBORMarshaler, len(deals)) for i, deal := range deals { cdeals[i] = &actors.OnChainDeal{ - Deal: deal, - ActivationEpoch: 1, + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + Client: deal.Client, + Provider: deal.Provider, + ProposalExpiration: deal.ProposalExpiration, + Duration: deal.Duration, + StoragePricePerEpoch: deal.StoragePricePerEpoch, + StorageCollateral: deal.StorageCollateral, + ActivationEpoch: 1, } } @@ -267,7 +274,7 @@ func mustEnc(i cbg.CBORMarshaler) []byte { return enc } -func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDeal, error) { +func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDealProposal, error) { vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore()) if err != nil { return cid.Undef, nil, xerrors.Errorf("failed to create NewVM: %w", err) @@ -281,7 +288,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, nil, xerrors.Errorf("miner address list, and preseal count doesn't match (%d != %d)", len(gmcfg.MinerAddrs), len(gmcfg.PreSeals)) } - var deals []actors.StorageDeal + var deals []actors.StorageDealProposal for i, maddr := range gmcfg.MinerAddrs { ps, psok := gmcfg.PreSeals[maddr.String()] diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 079a51ad1..bd3e16032 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -138,7 +138,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl return cid.Undef, cid.Undef, xerrors.Errorf("failed to get network actor: %w", err) } reward := vm.MiningReward(netact.Balance) - for _, b := range blks { + for tsi, b := range blks { netact, err = vmi.StateTree().GetActor(actors.NetworkAddress) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to get network actor: %w", err) @@ -171,10 +171,10 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl } ret, err := vmi.ApplyMessage(ctx, postSubmitMsg) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("submit election post message invocation failed: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("submit election post message for block %s (miner %s) invocation failed: %w", b.Cid(), b.Miner, err) } if ret.ExitCode != 0 { - return cid.Undef, cid.Undef, xerrors.Errorf("submit election post invocation returned nonzero exit code: %d", ret.ExitCode) + return cid.Undef, cid.Undef, xerrors.Errorf("submit election post invocation returned nonzero exit code: %d (err = %s, block = %s, miner = %s, tsi = %d)", ret.ExitCode, ret.ActorErr, b.Cid(), b.Miner, tsi) } } diff --git a/chain/sync.go b/chain/sync.go index 9d9e799f1..e594749e9 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -492,7 +492,16 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err } winnerCheck := async.Err(func() error { - _, tpow, err := stmgr.GetPower(ctx, syncer.sm, baseTs, h.Miner) + slashedAt, err := stmgr.GetMinerSlashed(ctx, syncer.sm, baseTs, h.Miner) + if err != nil { + return xerrors.Errorf("failed to check if block miner was slashed: %w", err) + } + + if slashedAt != 0 { + return xerrors.Errorf("received block was from miner slashed at height %d", slashedAt) + } + + mpow, tpow, err := stmgr.GetPower(ctx, syncer.sm, baseTs, h.Miner) if err != nil { return xerrors.Errorf("failed getting power: %w", err) } @@ -502,8 +511,10 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err return xerrors.Errorf("failed to get sector size for block miner: %w", err) } + snum := types.BigDiv(mpow, types.NewInt(ssize)) + for _, t := range h.EPostProof.Candidates { - if !types.IsTicketWinner(t.Partial, ssize, tpow) { + if !types.IsTicketWinner(t.Partial, ssize, snum.Uint64(), tpow) { return xerrors.Errorf("miner created a block but was not a winner") } } diff --git a/chain/types/bitfield.go b/chain/types/bitfield.go index b4687f5e8..20e63dd95 100644 --- a/chain/types/bitfield.go +++ b/chain/types/bitfield.go @@ -34,6 +34,38 @@ func BitFieldFromSet(setBits []uint64) BitField { return res } +func MergeBitFields(a, b BitField) (BitField, error) { + ra, err := a.rle.RunIterator() + if err != nil { + return BitField{}, err + } + + rb, err := b.rle.RunIterator() + if err != nil { + return BitField{}, err + } + + merge, err := rlepluslazy.Sum(ra, rb) + if err != nil { + return BitField{}, err + } + + mergebytes, err := rlepluslazy.EncodeRuns(merge, nil) + if err != nil { + return BitField{}, err + } + + rle, err := rlepluslazy.FromBuf(mergebytes) + if err != nil { + return BitField{}, err + } + + return BitField{ + rle: rle, + bits: make(map[uint64]struct{}), + }, nil +} + func (bf BitField) sum() (rlepluslazy.RunIterator, error) { if len(bf.bits) == 0 { return bf.rle.RunIterator() @@ -86,7 +118,26 @@ func (bf BitField) All() ([]uint64, error) { return nil, err } - return res, err + return res, nil +} + +func (bf BitField) AllMap() (map[uint64]bool, error) { + + runs, err := bf.sum() + if err != nil { + return nil, err + } + + res, err := rlepluslazy.SliceFromRuns(runs) + if err != nil { + return nil, err + } + + out := make(map[uint64]bool) + for _, i := range res { + out[i] = true + } + return out, nil } func (bf BitField) MarshalCBOR(w io.Writer) error { diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 7ba5ef436..e44ca54b4 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -174,9 +174,9 @@ var blocksPerEpoch = NewInt(build.BlocksPerEpoch) const sha256bits = 256 -func IsTicketWinner(partialTicket []byte, ssizeI uint64, totpow BigInt) bool { +func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow BigInt) bool { ssize := NewInt(ssizeI) - + ssampled := ElectionPostChallengeCount(snum) /* Need to check that (h(vrfout) + 1) / (max(h) + 1) <= e * sectorSize / totalPower @@ -185,23 +185,42 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, totpow BigInt) bool { (h(vrfout) + 1) * totalPower <= e * sectorSize * 2^256 in 2^256 space, it is equivalent to: h(vrfout) * totalPower < e * sectorSize * 2^256 + + Because of SectorChallengeRatioDiv sampling for proofs + we need to scale this appropriately. + + Let c = ceil(numSectors/SectorChallengeRatioDiv) + (c is the number of tickets a miner requests) + Accordingly we check + (h(vrfout) + 1) / 2^256 <= e * sectorSize / totalPower * snum / c + or + h(vrfout) * totalPower * c < e * sectorSize * 2^256 * snum */ h := sha256.Sum256(partialTicket) lhs := BigFromBytes(h[:]).Int lhs = lhs.Mul(lhs, totpow.Int) + lhs = lhs.Mul(lhs, new(big.Int).SetUint64(ssampled)) // rhs = sectorSize * 2^256 // rhs = sectorSize << 256 rhs := new(big.Int).Lsh(ssize.Int, sha256bits) - rhs = rhs.Mul(rhs, big.NewInt(build.SectorChallengeRatioDiv)) + rhs = rhs.Mul(rhs, new(big.Int).SetUint64(snum)) rhs = rhs.Mul(rhs, blocksPerEpoch.Int) // h(vrfout) * totalPower < e * sectorSize * 2^256? return lhs.Cmp(rhs) < 0 } +func ElectionPostChallengeCount(sectors uint64) uint64 { + if sectors == 0 { + return 0 + } + // ceil(sectors / build.SectorChallengeRatioDiv) + return (sectors-1)/build.SectorChallengeRatioDiv + 1 +} + func (t *Ticket) Equals(ot *Ticket) bool { return bytes.Equal(t.VRFProof, ot.VRFProof) } diff --git a/chain/types/vmcontext.go b/chain/types/vmcontext.go index 5e2963483..39667d2a4 100644 --- a/chain/types/vmcontext.go +++ b/chain/types/vmcontext.go @@ -40,10 +40,15 @@ type VMContext interface { ChargeGas(uint64) aerrors.ActorError GetRandomness(height uint64) ([]byte, aerrors.ActorError) GetBalance(address.Address) (BigInt, aerrors.ActorError) + Sys() *VMSyscalls Context() context.Context } +type VMSyscalls struct { + ValidatePoRep func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError) +} + type storageWrapper struct { s Storage } diff --git a/chain/vm/mkactor.go b/chain/vm/mkactor.go index 1e1885825..3419f9f3c 100644 --- a/chain/vm/mkactor.go +++ b/chain/vm/mkactor.go @@ -48,7 +48,7 @@ func makeActor(st *state.StateTree, addr address.Address) (*types.Actor, aerrors case address.SECP256K1: return NewSecp256k1AccountActor(st, addr) case address.ID: - return nil, aerrors.New(1, "no actor with given ID") + return nil, aerrors.Newf(1, "no actor with given ID: %s", addr) case address.Actor: return nil, aerrors.Newf(1, "no such actor: %s", addr) default: diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go new file mode 100644 index 000000000..92f9ea269 --- /dev/null +++ b/chain/vm/syscalls.go @@ -0,0 +1,14 @@ +package vm + +import ( + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" +) + +// Actual type is defined in chain/types/vmcontext.go because the VMContext interface is there + +func DefaultSyscalls() *types.VMSyscalls { + return &types.VMSyscalls{ + ValidatePoRep: actors.ValidatePoRep, + } +} diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 1eed0f895..7c1c5a802 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -54,6 +54,8 @@ type VMContext struct { gasAvailable types.BigInt gasUsed types.BigInt + sys *types.VMSyscalls + // root cid of the state of the actor this invocation will be on sroot cid.Cid @@ -75,6 +77,10 @@ func (vmc *VMContext) GetRandomness(height uint64) ([]byte, aerrors.ActorError) return res, nil } +func (vmc *VMContext) Sys() *types.VMSyscalls { + return vmc.sys +} + // Storage interface func (vmc *VMContext) Put(i cbg.CBORMarshaler) (cid.Cid, aerrors.ActorError) { @@ -284,6 +290,7 @@ func (vm *VM) makeVMContext(ctx context.Context, sroot cid.Cid, msg *types.Messa msg: msg, origin: origin, height: vm.blockHeight, + sys: vm.Syscalls, gasUsed: usedGas, gasAvailable: msg.GasLimit, @@ -304,6 +311,8 @@ type VM struct { blockMiner address.Address inv *invoker rand Rand + + Syscalls *types.VMSyscalls } func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore) (*VM, error) { @@ -323,6 +332,7 @@ func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs block blockMiner: maddr, inv: newInvoker(), rand: r, + Syscalls: DefaultSyscalls(), }, nil } diff --git a/cli/auth.go b/cli/auth.go index 121f39463..d957881b6 100644 --- a/cli/auth.go +++ b/cli/auth.go @@ -6,7 +6,7 @@ import ( "gopkg.in/urfave/cli.v2" - "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apistruct" ) var authCmd = &cli.Command{ @@ -42,18 +42,18 @@ var authCreateAdminToken = &cli.Command{ perm := cctx.String("perm") idx := 0 - for i, p := range api.AllPermissions { + for i, p := range apistruct.AllPermissions { if perm == p { idx = i + 1 } } if idx == 0 { - return fmt.Errorf("--perm flag has to be one of: %s", api.AllPermissions) + return fmt.Errorf("--perm flag has to be one of: %s", apistruct.AllPermissions) } // slice on [:idx] so for example: 'sign' gives you [read, write, sign] - token, err := napi.AuthNew(ctx, api.AllPermissions[:idx]) + token, err := napi.AuthNew(ctx, apistruct.AllPermissions[:idx]) if err != nil { return err } diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index c54a953f5..1fa9cc65e 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -2,11 +2,11 @@ package seed import ( "context" + "crypto/rand" "crypto/sha256" "encoding/json" "fmt" "io/ioutil" - "math/rand" "os" "path/filepath" @@ -53,7 +53,6 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb return nil, err } - r := rand.New(rand.NewSource(101)) size := sectorbuilder.UserBytesForSectorSize(ssize) var sealedSectors []*genesis.PreSeal @@ -63,7 +62,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb return nil, err } - pi, err := sb.AddPiece(size, sid, r, nil) + pi, err := sb.AddPiece(size, sid, rand.Reader, nil) if err != nil { return nil, err } @@ -143,7 +142,6 @@ func createDeals(m *genesis.GenesisMiner, k *wallet.Key, maddr address.Address, proposal := &actors.StorageDealProposal{ PieceRef: pref, // just one deal so this == CommP PieceSize: sectorbuilder.UserBytesForSectorSize(ssize), - PieceSerialization: actors.SerializationUnixFSv0, Client: k.Address, Provider: maddr, ProposalExpiration: 9000, // TODO: allow setting @@ -153,20 +151,12 @@ func createDeals(m *genesis.GenesisMiner, k *wallet.Key, maddr address.Address, ProposerSignature: nil, } + // TODO: pretty sure we don't even need to sign this if err := api.SignWith(context.TODO(), wallet.KeyWallet(k).Sign, k.Address, proposal); err != nil { return err } - deal := &actors.StorageDeal{ - Proposal: *proposal, - CounterSignature: nil, - } - - if err := api.SignWith(context.TODO(), wallet.KeyWallet(k).Sign, k.Address, deal); err != nil { - return err - } - - sector.Deal = *deal + sector.Deal = *proposal } return nil diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index c396f2def..e16bc95ce 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "crypto/rand" "encoding/json" @@ -267,7 +268,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin return err } - proposalCid, err := sector.Deal.Proposal.Cid() + proposalCid, err := sector.Deal.Cid() if err != nil { return err } @@ -275,7 +276,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String()) deal := &deals.MinerDeal{ - Proposal: sector.Deal.Proposal, + Proposal: sector.Deal, ProposalCid: proposalCid, State: lapi.DealComplete, Ref: proposalCid, // TODO: This is super wrong, but there @@ -298,7 +299,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin return nil } -func findMarketDealID(ctx context.Context, api lapi.FullNode, deal actors.StorageDeal) (uint64, error) { +func findMarketDealID(ctx context.Context, api lapi.FullNode, deal actors.StorageDealProposal) (uint64, error) { // TODO: find a better way // (this is only used by genesis miners) @@ -308,11 +309,7 @@ func findMarketDealID(ctx context.Context, api lapi.FullNode, deal actors.Storag } for k, v := range deals { - eq, err := cborutil.Equals(&v.Deal, &deal) - if err != nil { - return 0, err - } - if eq { + if bytes.Equal(v.PieceRef, deal.PieceRef) { return strconv.ParseUint(k, 10, 64) } } diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 814e93686..72826d585 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -15,6 +15,7 @@ import ( "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/auth" @@ -131,7 +132,7 @@ var runCmd = &cli.Command{ mux := mux.NewRouter() rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", api.PermissionedStorMinerAPI(minerapi)) + rpcServer.Register("Filecoin", apistruct.PermissionedStorMinerAPI(minerapi)) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/remote").HandlerFunc(minerapi.(*impl.StorageMinerAPI).ServeRemote) diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index 3b203fff2..5178e1b6c 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "github.com/filecoin-project/lotus/api/apistruct" "net/http" _ "net/http/pprof" "os" @@ -26,7 +27,7 @@ var log = logging.Logger("main") func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr) error { rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", api.PermissionedFullAPI(a)) + rpcServer.Register("Filecoin", apistruct.PermissionedFullAPI(a)) ah := &auth.Handler{ Verify: a.AuthVerify, @@ -70,7 +71,7 @@ func handleImport(a *impl.FullNodeAPI) func(w http.ResponseWriter, r *http.Reque w.WriteHeader(404) return } - if !api.HasPerm(r.Context(), api.PermWrite) { + if !apistruct.HasPerm(r.Context(), apistruct.PermWrite) { w.WriteHeader(401) json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"}) return diff --git a/gen/main.go b/gen/main.go index 76d87a49b..52fcf4665 100644 --- a/gen/main.go +++ b/gen/main.go @@ -96,6 +96,7 @@ func main() { actors.SubmitFallbackPoStParams{}, actors.PaymentVerifyParams{}, actors.UpdatePeerIDParams{}, + actors.DeclareFaultsParams{}, actors.MultiSigActorState{}, actors.MultiSigConstructorParams{}, actors.MultiSigProposeParams{}, @@ -122,7 +123,6 @@ func main() { actors.StorageMarketState{}, actors.WithdrawBalanceParams{}, actors.StorageDealProposal{}, - actors.StorageDeal{}, actors.PublishStorageDealsParams{}, actors.PublishStorageDealResponse{}, actors.ActivateStorageDealsParams{}, diff --git a/genesis/types.go b/genesis/types.go index e8a208452..712c116cc 100644 --- a/genesis/types.go +++ b/genesis/types.go @@ -10,7 +10,7 @@ type PreSeal struct { CommR [32]byte CommD [32]byte SectorID uint64 - Deal actors.StorageDeal + Deal actors.StorageDealProposal } type GenesisMiner struct { diff --git a/lib/auth/handler.go b/lib/auth/handler.go index 67fc1ac9b..b6112ce8c 100644 --- a/lib/auth/handler.go +++ b/lib/auth/handler.go @@ -5,8 +5,10 @@ import ( "net/http" "strings" - "github.com/filecoin-project/lotus/api" logging "github.com/ipfs/go-log" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apistruct" ) var log = logging.Logger("auth") @@ -42,7 +44,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - ctx = api.WithPerm(ctx, allow) + ctx = apistruct.WithPerm(ctx, allow) } h.Next(w, r.WithContext(ctx)) diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 7a5d744ab..c4fca12d4 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -16,6 +16,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -623,7 +624,7 @@ func (sb *SectorBuilder) GenerateEPostCandidates(sectorInfo SortedPublicSectorIn return nil, err } - challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo.Values()))) + challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values()))) proverID := addressToProverID(sb.Miner) return sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors) @@ -674,13 +675,8 @@ func (sb *SectorBuilder) Stop() { close(sb.stopping) } -func ElectionPostChallengeCount(sectors uint64) uint64 { - // ceil(sectors / build.SectorChallengeRatioDiv) - return (sectors + build.SectorChallengeRatioDiv - 1) / build.SectorChallengeRatioDiv -} - func fallbackPostChallengeCount(sectors uint64) uint64 { - challengeCount := ElectionPostChallengeCount(sectors) + challengeCount := types.ElectionPostChallengeCount(sectors) if challengeCount > build.MaxFallbackPostChallengeCount { return build.MaxFallbackPostChallengeCount } diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 13c80f0fc..5aa03b447 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -124,6 +124,9 @@ func post(t *testing.T, sb *sectorbuilder.SectorBuilder, seals ...seal) time.Tim } func TestSealAndVerify(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware t.Skip("this is slow") } @@ -193,6 +196,9 @@ func TestSealAndVerify(t *testing.T) { } func TestSealPoStNoCommit(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware t.Skip("this is slow") } @@ -259,6 +265,9 @@ func TestSealPoStNoCommit(t *testing.T) { } func TestSealAndVerify2(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware t.Skip("this is slow") } diff --git a/lib/sectorbuilder/simple.go b/lib/sectorbuilder/simple.go index 0c9231092..457c4b2e1 100644 --- a/lib/sectorbuilder/simple.go +++ b/lib/sectorbuilder/simple.go @@ -8,6 +8,7 @@ import ( "go.opencensus.io/trace" "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/types" ) func (sb *SectorBuilder) SectorSize() uint64 { @@ -36,7 +37,7 @@ func NewSortedPublicSectorInfo(sectors []sectorbuilder.PublicSectorInfo) SortedP } func VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) { - challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo.Values()))) + challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values()))) return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID) } diff --git a/node/impl/common.go b/node/impl/common.go index f8037ca1d..990a46a8a 100644 --- a/node/impl/common.go +++ b/node/impl/common.go @@ -2,7 +2,6 @@ package impl import ( "context" - "github.com/gbrlsnchs/jwt/v3" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 04037c079..2aa14a2e5 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -148,7 +148,7 @@ func (a *StateAPI) stateForTs(ctx context.Context, ts *types.TipSet) (*state.Sta func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error) { state, err := a.stateForTs(ctx, ts) if err != nil { - return nil, err + return nil, xerrors.Errorf("computing tipset state failed: %w", err) } return state.GetActor(actor) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index a31bd07a9..d7eba489a 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -3,6 +3,7 @@ package impl import ( "context" "encoding/json" + "github.com/filecoin-project/lotus/api/apistruct" "io" "mime" "net/http" @@ -33,7 +34,7 @@ type StorageMinerAPI struct { } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { - if !api.HasPerm(r.Context(), api.PermAdmin) { + if !apistruct.HasPerm(r.Context(), apistruct.PermAdmin) { w.WriteHeader(401) json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"}) return diff --git a/node/modules/core.go b/node/modules/core.go index 5eab490eb..186ad9936 100644 --- a/node/modules/core.go +++ b/node/modules/core.go @@ -3,10 +3,10 @@ package modules import ( "context" "crypto/rand" + "github.com/filecoin-project/lotus/api/apistruct" "io" "io/ioutil" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/addrutil" @@ -57,7 +57,7 @@ func APISecret(keystore types.KeyStore, lr repo.LockedRepo) (*dtypes.APIAlg, err // TODO: make this configurable p := jwtPayload{ - Allow: api.AllPermissions, + Allow: apistruct.AllPermissions, } cliToken, err := jwt.Sign(&p, jwt.NewHS256(key.PrivateKey)) diff --git a/node/node_test.go b/node/node_test.go index 28d66a013..0fe2ac7f9 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -292,5 +292,8 @@ func TestAPIRPC(t *testing.T) { } func TestAPIDealFlow(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } test.TestDealFlow(t, builder) } diff --git a/storage/cbor_gen.go b/storage/cbor_gen.go index 207747b6b..94a0bfb0d 100644 --- a/storage/cbor_gen.go +++ b/storage/cbor_gen.go @@ -385,7 +385,7 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{172}); err != nil { + if _, err := w.Write([]byte{173}); err != nil { return err } @@ -547,6 +547,24 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } } + // t.FaultReportMsg (cid.Cid) (struct) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("FaultReportMsg")))); err != nil { + return err + } + if _, err := w.Write([]byte("FaultReportMsg")); err != nil { + return err + } + + if t.FaultReportMsg == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.FaultReportMsg); err != nil { + return xerrors.Errorf("failed to write cid field t.FaultReportMsg: %w", err) + } + } + // t.LastErr (string) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("LastErr")))); err != nil { return err @@ -575,7 +593,7 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type map") } - if extra != 12 { + if extra != 13 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -897,6 +915,43 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { t.CommitMessage = &c } + } + // t.FaultReportMsg (cid.Cid) (struct) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + name = string(sval) + } + + if name != "FaultReportMsg" { + return fmt.Errorf("expected struct map entry %s to be FaultReportMsg", name) + } + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.FaultReportMsg: %w", err) + } + + t.FaultReportMsg = &c + } + } // t.LastErr (string) (string) diff --git a/storage/garbage.go b/storage/garbage.go index 18a50a405..9432d7336 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -20,7 +20,7 @@ func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPiece return nil, nil } - deals := make([]actors.StorageDeal, len(sizes)) + deals := make([]actors.StorageDealProposal, len(sizes)) for i, size := range sizes { release := m.sb.RateLimit() commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) @@ -33,7 +33,6 @@ func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPiece sdp := actors.StorageDealProposal{ PieceRef: commP[:], PieceSize: size, - PieceSerialization: actors.SerializationUnixFSv0, Client: m.worker, Provider: m.maddr, ProposalExpiration: math.MaxUint64, @@ -47,14 +46,7 @@ func (m *Miner) pledgeSector(ctx context.Context, sectorID uint64, existingPiece return nil, xerrors.Errorf("signing storage deal failed: ", err) } - storageDeal := actors.StorageDeal{ - Proposal: sdp, - } - if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &storageDeal); err != nil { - return nil, xerrors.Errorf("signing storage deal failed: ", err) - } - - deals[i] = storageDeal + deals[i] = sdp } params, aerr := actors.SerializeParams(&actors.PublishStorageDealsParams{ diff --git a/storage/sector_states.go b/storage/sector_states.go index b61fadd60..ffad0119e 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -227,9 +227,63 @@ func (m *Miner) handleCommitWait(ctx context.Context, sector SectorInfo) *sector if mw.Receipt.ExitCode != 0 { log.Errorf("UNHANDLED: submitting sector proof failed (exit=%d, msg=%s) (t:%x; s:%x(%d); p:%x)", mw.Receipt.ExitCode, sector.CommitMessage, sector.Ticket.TicketBytes, sector.Seed.TicketBytes, sector.Seed.BlockHeight, sector.Proof) - return sector.upd().fatal(xerrors.New("UNHANDLED: submitting sector proof failed")) + return sector.upd().fatal(xerrors.Errorf("UNHANDLED: submitting sector proof failed (exit: %d)", mw.Receipt.ExitCode)) } return sector.upd().to(api.Proving).state(func(info *SectorInfo) { }) } + +func (m *Miner) handleFaulty(ctx context.Context, sector SectorInfo) *sectorUpdate { + // TODO: check if the fault has already been reported, and that this sector is even valid + + // TODO: coalesce faulty sector reporting + bf := types.NewBitField() + bf.Set(sector.SectorID) + + fp := &actors.DeclareFaultsParams{bf} + _ = fp + enc, aerr := actors.SerializeParams(nil) + if aerr != nil { + return sector.upd().fatal(xerrors.Errorf("failed to serialize declare fault params: %w", aerr)) + } + + msg := &types.Message{ + To: m.maddr, + From: m.worker, + Method: actors.MAMethods.DeclareFaults, + Params: enc, + Value: types.NewInt(0), // TODO: need to ensure sufficient collateral + GasLimit: types.NewInt(1000000 /* i dont know help */), + GasPrice: types.NewInt(1), + } + + smsg, err := m.api.MpoolPushMessage(ctx, msg) + if err != nil { + return sector.upd().to(api.FailedUnrecoverable).error(xerrors.Errorf("failed to push declare faults message to network: %w", err)) + } + + return sector.upd().to(api.FaultReported).state(func(info *SectorInfo) { + c := smsg.Cid() + info.FaultReportMsg = &c + }) +} + +func (m *Miner) handleFaultReported(ctx context.Context, sector SectorInfo) *sectorUpdate { + if sector.FaultReportMsg == nil { + return sector.upd().to(api.FailedUnrecoverable).error(xerrors.Errorf("entered fault reported state without a FaultReportMsg cid")) + } + + mw, err := m.api.StateWaitMsg(ctx, *sector.FaultReportMsg) + if err != nil { + return sector.upd().to(api.CommitFailed).error(xerrors.Errorf("failed to wait for fault declaration: %w", err)) + } + + if mw.Receipt.ExitCode != 0 { + log.Errorf("UNHANDLED: declaring sector fault failed (exit=%d, msg=%s) (id: %d)", mw.Receipt.ExitCode, *sector.FaultReportMsg, sector.SectorID) + return sector.upd().fatal(xerrors.Errorf("UNHANDLED: submitting fault declaration failed (exit %d)", mw.Receipt.ExitCode)) + } + + return sector.upd().to(api.FaultedFinal).state(func(info *SectorInfo) {}) + +} diff --git a/storage/sector_types.go b/storage/sector_types.go index aed3898db..3c328b9c0 100644 --- a/storage/sector_types.go +++ b/storage/sector_types.go @@ -69,6 +69,9 @@ type SectorInfo struct { // Committing CommitMessage *cid.Cid + // Faults + FaultReportMsg *cid.Cid + // Debug LastErr string } diff --git a/storage/sectors.go b/storage/sectors.go index d9144a5cd..8dad777cc 100644 --- a/storage/sectors.go +++ b/storage/sectors.go @@ -255,6 +255,12 @@ func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { case api.CommitFailed: log.Warnf("sector %d entered unimplemented state 'CommitFailed'", update.id) + // Faults + case api.Faulty: + m.handleSectorUpdate(ctx, sector, m.handleFaulty) + case api.FaultReported: + m.handleSectorUpdate(ctx, sector, m.handleFaultReported) + // Fatal errors case api.UndefinedSectorState: log.Error("sector update with undefined state!")