From e6a08784f860e1e98fc2189bfea915a092ee9bad Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sat, 7 Dec 2019 17:24:42 +0100 Subject: [PATCH 01/14] print more errors if epost fails during tipset state evaluation --- chain/stmgr/stmgr.go | 4 ++-- node/impl/full/state.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index c65dfe596..7303705cb 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -147,10 +147,10 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl } ret, err := vmi.ApplyMessage(ctx, postSubmitMsg) if err != nil { - return cid.Undef, cid.Undef, xerrors.Errorf("submit election post message invocation failed: %w", err) + return cid.Undef, cid.Undef, xerrors.Errorf("submit election post message for block %s (miner %s) invocation failed: %w", b.Cid(), b.Miner, err) } if ret.ExitCode != 0 { - return cid.Undef, cid.Undef, xerrors.Errorf("submit election post invocation returned nonzero exit code: %d", ret.ExitCode) + return cid.Undef, cid.Undef, xerrors.Errorf("submit election post invocation returned nonzero exit code: %d (err = %s)", ret.ExitCode, ret.ActorErr) } } diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 04037c079..2aa14a2e5 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -148,7 +148,7 @@ func (a *StateAPI) stateForTs(ctx context.Context, ts *types.TipSet) (*state.Sta func (a *StateAPI) StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error) { state, err := a.stateForTs(ctx, ts) if err != nil { - return nil, err + return nil, xerrors.Errorf("computing tipset state failed: %w", err) } return state.GetActor(actor) From 8181b283c0b6afcc7f83497cda6c232473aac746 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sat, 7 Dec 2019 18:59:34 +0100 Subject: [PATCH 02/14] don't accept blocks from slashed miners --- chain/stmgr/stmgr.go | 4 ++-- chain/sync.go | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 7303705cb..0f0717879 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -114,7 +114,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl return cid.Undef, cid.Undef, xerrors.Errorf("failed to get network actor: %w", err) } reward := vm.MiningReward(netact.Balance) - for _, b := range blks { + for tsi, b := range blks { netact, err = vmi.StateTree().GetActor(actors.NetworkAddress) if err != nil { return cid.Undef, cid.Undef, xerrors.Errorf("failed to get network actor: %w", err) @@ -150,7 +150,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl return cid.Undef, cid.Undef, xerrors.Errorf("submit election post message for block %s (miner %s) invocation failed: %w", b.Cid(), b.Miner, err) } if ret.ExitCode != 0 { - return cid.Undef, cid.Undef, xerrors.Errorf("submit election post invocation returned nonzero exit code: %d (err = %s)", ret.ExitCode, ret.ActorErr) + return cid.Undef, cid.Undef, xerrors.Errorf("submit election post invocation returned nonzero exit code: %d (err = %s, block = %s, miner = %s, tsi = %d)", ret.ExitCode, ret.ActorErr, b.Cid(), b.Miner, tsi) } } diff --git a/chain/sync.go b/chain/sync.go index 9d9e799f1..c632c80a0 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -492,6 +492,15 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err } winnerCheck := async.Err(func() error { + slashedAt, err := stmgr.GetMinerSlashed(ctx, syncer.sm, baseTs, h.Miner) + if err != nil { + return xerrors.Errorf("failed to check if block miner was slashed: %w", err) + } + + if slashedAt != 0 { + return xerrors.Errorf("received block was from miner slashed at height %d", slashedAt) + } + _, tpow, err := stmgr.GetPower(ctx, syncer.sm, baseTs, h.Miner) if err != nil { return xerrors.Errorf("failed getting power: %w", err) From f0653decf3f38bf3289f928f8d01c0ee6c94ce39 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sat, 7 Dec 2019 15:12:10 +0100 Subject: [PATCH 03/14] Clean up storage deal structures --- chain/actors/actor_storagemarket.go | 118 ++++++++------- chain/actors/cbor_gen.go | 216 ++++++++++++++++++---------- chain/blocksync/cbor_gen.go | 2 +- chain/deals/cbor_gen.go | 76 ++-------- chain/deals/client.go | 2 +- chain/deals/client_states.go | 17 ++- chain/deals/provider_states.go | 15 +- chain/deals/types.go | 3 +- chain/gen/utils.go | 18 ++- chain/types/cbor_gen.go | 2 +- cmd/lotus-seed/seed/seed.go | 17 +-- cmd/lotus-storage-miner/init.go | 13 +- gen/main.go | 1 - genesis/types.go | 2 +- storage/garbage.go | 11 +- 15 files changed, 247 insertions(+), 266 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 651fd8f71..84dd2a306 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -3,6 +3,7 @@ package actors import ( "bytes" "context" + "golang.org/x/xerrors" "github.com/filecoin-project/go-amt-ipld" @@ -131,36 +132,20 @@ func (sdp *StorageDealProposal) Verify() error { return sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()) } -func (d *StorageDeal) Sign(ctx context.Context, sign SignFunc) error { - var buf bytes.Buffer - if err := d.Proposal.MarshalCBOR(&buf); err != nil { - return err - } - sig, err := sign(ctx, buf.Bytes()) - if err != nil { - return err - } - d.CounterSignature = sig - return nil -} - -func (d *StorageDeal) Verify(proposerWorker address.Address) error { - var buf bytes.Buffer - if err := d.Proposal.MarshalCBOR(&buf); err != nil { - return err - } - - return d.CounterSignature.Verify(proposerWorker, buf.Bytes()) -} - -type StorageDeal struct { - Proposal StorageDealProposal - CounterSignature *types.Signature -} - type OnChainDeal struct { - Deal StorageDeal - ActivationEpoch uint64 // 0 = inactive + PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea + PieceSize uint64 + PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid + + Client address.Address + Provider address.Address + + ProposalExpiration uint64 + Duration uint64 // TODO: spec + + StoragePricePerEpoch types.BigInt + StorageCollateral types.BigInt + ActivationEpoch uint64 // 0 = inactive } type WithdrawBalanceParams struct { @@ -245,6 +230,7 @@ func (sma StorageMarketActor) AddBalance(act *types.Actor, vmctx types.VMContext } func setMarketBalances(vmctx types.VMContext, nd *hamt.Node, set map[address.Address]StorageParticipantBalance) (cid.Cid, ActorError) { + // TODO: iterating over a map might happen in the wrong order, this could have gas implications for addr, b := range set { balance := b // to stop linter complaining if err := nd.Set(vmctx.Context(), string(addr.Bytes()), &balance); err != nil { @@ -297,7 +283,7 @@ func (sma StorageMarketActor) CheckLockedBalance(act *types.Actor, vmctx types.V */ type PublishStorageDealsParams struct { - Deals []StorageDeal + Deals []StorageDealProposal } type PublishStorageDealResponse struct { @@ -326,7 +312,7 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. DealIDs: make([]uint64, len(params.Deals)), } - workerBytes, aerr := vmctx.Send(params.Deals[0].Proposal.Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil) + workerBytes, aerr := vmctx.Send(params.Deals[0].Provider, MAMethods.GetWorkerAddr, types.NewInt(0), nil) if aerr != nil { return nil, aerr } @@ -342,7 +328,21 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. return nil, err } - err := deals.Set(self.NextDealID, &OnChainDeal{Deal: deal}) + err := deals.Set(self.NextDealID, &OnChainDeal{ + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + PieceSerialization: deal.PieceSerialization, // TODO: this isnt needed anymore, right? + + Client: deal.Client, + Provider: deal.Provider, + + ProposalExpiration: deal.ProposalExpiration, + Duration: deal.Duration, + + StoragePricePerEpoch: deal.StoragePricePerEpoch, + StorageCollateral: deal.StorageCollateral, + ActivationEpoch: 0, + }) if err != nil { return nil, aerrors.HandleExternalError(err, "setting deal in deal AMT") } @@ -376,34 +376,28 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. return outBuf.Bytes(), nil } -func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDeal, providerWorker address.Address) aerrors.ActorError { - if vmctx.BlockHeight() > deal.Proposal.ProposalExpiration { +func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDealProposal, providerWorker address.Address) aerrors.ActorError { + if vmctx.BlockHeight() > deal.ProposalExpiration { return aerrors.New(1, "deal proposal already expired") } - if err := deal.Proposal.Verify(); err != nil { - return aerrors.Absorb(err, 2, "verifying proposer signature") + if vmctx.Message().From != providerWorker { + return aerrors.New(2, "Deals must be submitted by the miner worker") } - err := deal.Verify(providerWorker) - if err != nil { - return aerrors.Absorb(err, 2, "verifying provider signature") - } - - // TODO: maybe this is actually fine - if vmctx.Message().From != providerWorker && vmctx.Message().From != deal.Proposal.Client { - return aerrors.New(4, "message not sent by deal participant") + if err := deal.Verify(); err != nil { + return aerrors.Absorb(err, 3, "verifying proposer signature") } // TODO: do some caching (changes gas so needs to be in spec too) - b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), st.Balances, deal.Proposal.Client, providerWorker) + b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), st.Balances, deal.Client, providerWorker) if aerr != nil { return aerrors.Wrap(aerr, "getting client, and provider balances") } clientBalance := b[0] providerBalance := b[1] - totalPrice := deal.Proposal.TotalStoragePrice() + totalPrice := deal.TotalStoragePrice() if clientBalance.Available.LessThan(totalPrice) { return aerrors.Newf(5, "client doesn't have enough available funds to cover storage price; %d < %d", clientBalance.Available, totalPrice) @@ -412,17 +406,17 @@ func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDe clientBalance = lockFunds(clientBalance, totalPrice) // TODO: REVIEW: Not clear who pays for this - if providerBalance.Available.LessThan(deal.Proposal.StorageCollateral) { - return aerrors.Newf(6, "provider doesn't have enough available funds to cover StorageCollateral; %d < %d", providerBalance.Available, deal.Proposal.StorageCollateral) + if providerBalance.Available.LessThan(deal.StorageCollateral) { + return aerrors.Newf(6, "provider doesn't have enough available funds to cover StorageCollateral; %d < %d", providerBalance.Available, deal.StorageCollateral) } - providerBalance = lockFunds(providerBalance, deal.Proposal.StorageCollateral) + providerBalance = lockFunds(providerBalance, deal.StorageCollateral) // TODO: piece checks (e.g. size > sectorSize)? bcid, aerr := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ - deal.Proposal.Client: clientBalance, - providerWorker: providerBalance, + deal.Client: clientBalance, + providerWorker: providerBalance, }) if aerr != nil { return aerr @@ -458,11 +452,11 @@ func (sma StorageMarketActor) ActivateStorageDeals(act *types.Actor, vmctx types return nil, aerrors.HandleExternalError(err, "getting deal info failed") } - if vmctx.Message().From != dealInfo.Deal.Proposal.Provider { + if vmctx.Message().From != dealInfo.Provider { return nil, aerrors.New(1, "ActivateStorageDeals can only be called by the deal provider") } - if vmctx.BlockHeight() > dealInfo.Deal.Proposal.ProposalExpiration { + if vmctx.BlockHeight() > dealInfo.ProposalExpiration { return nil, aerrors.New(2, "deal cannot be activated: proposal expired") } @@ -533,7 +527,7 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx return nil, aerrors.HandleExternalError(err, "getting deal info failed") } - if dealInfo.Deal.Proposal.Provider != vmctx.Message().From { + if dealInfo.Provider != vmctx.Message().From { return nil, aerrors.New(3, "ProcessStorageDealsPayment can only be called by deal provider") } @@ -542,15 +536,15 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx return nil, aerrors.New(4, "ActivationEpoch lower than block height") } - if vmctx.BlockHeight() > dealInfo.ActivationEpoch+dealInfo.Deal.Proposal.Duration { + if vmctx.BlockHeight() > dealInfo.ActivationEpoch+dealInfo.Duration { // Deal expired, miner should drop it // TODO: process payment for the remainder of last proving period return nil, nil } - toPay := types.BigMul(dealInfo.Deal.Proposal.StoragePricePerEpoch, types.NewInt(build.SlashablePowerDelay)) + toPay := types.BigMul(dealInfo.StoragePricePerEpoch, types.NewInt(build.SlashablePowerDelay)) - b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Deal.Proposal.Client, providerWorker) + b, bnd, aerr := GetMarketBalances(vmctx.Context(), vmctx.Ipld(), self.Balances, dealInfo.Client, providerWorker) if aerr != nil { return nil, aerr } @@ -561,8 +555,8 @@ func (sma StorageMarketActor) ProcessStorageDealsPayment(act *types.Actor, vmctx // TODO: call set once bcid, aerr := setMarketBalances(vmctx, bnd, map[address.Address]StorageParticipantBalance{ - dealInfo.Deal.Proposal.Client: clientBal, - providerWorker: providerBal, + dealInfo.Client: clientBal, + providerWorker: providerBal, }) if aerr != nil { return nil, aerr @@ -625,15 +619,15 @@ func (sma StorageMarketActor) ComputeDataCommitment(act *types.Actor, vmctx type return nil, aerrors.HandleExternalError(err, "getting deal info failed") } - if dealInfo.Deal.Proposal.Provider != vmctx.Message().From { + if dealInfo.Provider != vmctx.Message().From { return nil, aerrors.New(5, "referenced deal was not from caller") } var commP [32]byte - copy(commP[:], dealInfo.Deal.Proposal.PieceRef) + copy(commP[:], dealInfo.PieceRef) pieces = append(pieces, sectorbuilder.PublicPieceInfo{ - Size: dealInfo.Deal.Proposal.PieceSize, + Size: dealInfo.PieceSize, CommP: commP, }) } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index e77ac2089..08d52cc48 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -3350,75 +3350,6 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return nil } -func (t *StorageDeal) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.t.Proposal (actors.StorageDealProposal) (struct) - if err := t.Proposal.MarshalCBOR(w); err != nil { - return err - } - - // t.t.CounterSignature (types.Signature) (struct) - if err := t.CounterSignature.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *StorageDeal) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.t.Proposal (actors.StorageDealProposal) (struct) - - { - - if err := t.Proposal.UnmarshalCBOR(br); err != nil { - return err - } - - } - // t.t.CounterSignature (types.Signature) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.CounterSignature = new(types.Signature) - if err := t.CounterSignature.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} - func (t *PublishStorageDealsParams) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -3428,7 +3359,7 @@ func (t *PublishStorageDealsParams) MarshalCBOR(w io.Writer) error { return err } - // t.t.Deals ([]actors.StorageDeal) (slice) + // t.t.Deals ([]actors.StorageDealProposal) (slice) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Deals)))); err != nil { return err } @@ -3455,7 +3386,7 @@ func (t *PublishStorageDealsParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Deals ([]actors.StorageDeal) (slice) + // t.t.Deals ([]actors.StorageDealProposal) (slice) maj, extra, err = cbg.CborReadHeader(br) if err != nil { @@ -3469,11 +3400,11 @@ func (t *PublishStorageDealsParams) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("expected cbor array") } if extra > 0 { - t.Deals = make([]StorageDeal, extra) + t.Deals = make([]StorageDealProposal, extra) } for i := 0; i < int(extra); i++ { - var v StorageDeal + var v StorageDealProposal if err := v.UnmarshalCBOR(br); err != nil { return err } @@ -3696,12 +3627,55 @@ func (t *OnChainDeal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{130}); err != nil { + if _, err := w.Write([]byte{138}); err != nil { return err } - // t.t.Deal (actors.StorageDeal) (struct) - if err := t.Deal.MarshalCBOR(w); err != nil { + // t.t.PieceRef ([]uint8) (slice) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PieceRef)))); err != nil { + return err + } + if _, err := w.Write(t.PieceRef); err != nil { + return err + } + + // t.t.PieceSize (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSize))); err != nil { + return err + } + + // t.t.PieceSerialization (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSerialization))); err != nil { + return err + } + + // t.t.Client (address.Address) (struct) + if err := t.Client.MarshalCBOR(w); err != nil { + return err + } + + // t.t.Provider (address.Address) (struct) + if err := t.Provider.MarshalCBOR(w); err != nil { + return err + } + + // t.t.ProposalExpiration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil { + return err + } + + // t.t.Duration (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil { + return err + } + + // t.t.StoragePricePerEpoch (types.BigInt) (struct) + if err := t.StoragePricePerEpoch.MarshalCBOR(w); err != nil { + return err + } + + // t.t.StorageCollateral (types.BigInt) (struct) + if err := t.StorageCollateral.MarshalCBOR(w); err != nil { return err } @@ -3723,15 +3697,99 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 2 { + if extra != 10 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.t.Deal (actors.StorageDeal) (struct) + // t.t.PieceRef ([]uint8) (slice) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if extra > 8192 { + return fmt.Errorf("t.PieceRef: array too large (%d)", extra) + } + + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + t.PieceRef = make([]byte, extra) + if _, err := io.ReadFull(br, t.PieceRef); err != nil { + return err + } + // t.t.PieceSize (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.PieceSize = uint64(extra) + // t.t.PieceSerialization (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.PieceSerialization = uint64(extra) + // t.t.Client (address.Address) (struct) { - if err := t.Deal.UnmarshalCBOR(br); err != nil { + if err := t.Client.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.Provider (address.Address) (struct) + + { + + if err := t.Provider.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.ProposalExpiration (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.ProposalExpiration = uint64(extra) + // t.t.Duration (uint64) (uint64) + + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.Duration = uint64(extra) + // t.t.StoragePricePerEpoch (types.BigInt) (struct) + + { + + if err := t.StoragePricePerEpoch.UnmarshalCBOR(br); err != nil { + return err + } + + } + // t.t.StorageCollateral (types.BigInt) (struct) + + { + + if err := t.StorageCollateral.UnmarshalCBOR(br); err != nil { return err } diff --git a/chain/blocksync/cbor_gen.go b/chain/blocksync/cbor_gen.go index 579b3f857..4af61a490 100644 --- a/chain/blocksync/cbor_gen.go +++ b/chain/blocksync/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "github.com/filecoin-project/lotus/chain/types" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) diff --git a/chain/deals/cbor_gen.go b/chain/deals/cbor_gen.go index a5e20bb54..e69df5528 100644 --- a/chain/deals/cbor_gen.go +++ b/chain/deals/cbor_gen.go @@ -192,7 +192,7 @@ func (t *Response) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{133}); err != nil { + if _, err := w.Write([]byte{132}); err != nil { return err } @@ -215,23 +215,10 @@ func (t *Response) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Proposal: %w", err) } - // t.t.StorageDeal (actors.StorageDeal) (struct) - if err := t.StorageDeal.MarshalCBOR(w); err != nil { + // t.t.StorageDealSubmission (types.SignedMessage) (struct) + if err := t.StorageDealSubmission.MarshalCBOR(w); err != nil { return err } - - // t.t.PublishMessage (cid.Cid) (struct) - - if t.PublishMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) - } - } - return nil } @@ -246,7 +233,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 5 { + if extra != 4 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -282,7 +269,7 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { t.Proposal = c } - // t.t.StorageDeal (actors.StorageDeal) (struct) + // t.t.StorageDealSubmission (types.SignedMessage) (struct) { @@ -296,36 +283,12 @@ func (t *Response) UnmarshalCBOR(r io.Reader) error { return err } } else { - t.StorageDeal = new(actors.StorageDeal) - if err := t.StorageDeal.UnmarshalCBOR(br); err != nil { + t.StorageDealSubmission = new(types.SignedMessage) + if err := t.StorageDealSubmission.UnmarshalCBOR(br); err != nil { return err } } - } - // t.t.PublishMessage (cid.Cid) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) - } - - t.PublishMessage = &c - } - } return nil } @@ -593,18 +556,10 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.PublishMessage (cid.Cid) (struct) - - if t.PublishMessage == nil { - if _, err := w.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(w, *t.PublishMessage); err != nil { - return xerrors.Errorf("failed to write cid field t.PublishMessage: %w", err) - } + // t.t.PublishMessage (types.SignedMessage) (struct) + if err := t.PublishMessage.MarshalCBOR(w); err != nil { + return err } - return nil } @@ -683,7 +638,7 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.DealID = uint64(extra) - // t.t.PublishMessage (cid.Cid) (struct) + // t.t.PublishMessage (types.SignedMessage) (struct) { @@ -697,13 +652,10 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return err } } else { - - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PublishMessage: %w", err) + t.PublishMessage = new(types.SignedMessage) + if err := t.PublishMessage.UnmarshalCBOR(br); err != nil { + return err } - - t.PublishMessage = &c } } diff --git a/chain/deals/client.go b/chain/deals/client.go index 988bc52b6..52f53f8d6 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -36,7 +36,7 @@ type ClientDeal struct { MinerWorker address.Address DealID uint64 - PublishMessage *cid.Cid + PublishMessage *types.SignedMessage s inet.Stream } diff --git a/chain/deals/client_states.go b/chain/deals/client_states.go index bac7aab47..7ee93b188 100644 --- a/chain/deals/client_states.go +++ b/chain/deals/client_states.go @@ -44,6 +44,9 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), e if err != nil { return nil, err } + + // TODO: verify StorageDealSubmission + if err := c.disconnect(deal); err != nil { return nil, err } @@ -54,18 +57,14 @@ func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), e } return func(info *ClientDeal) { - info.PublishMessage = resp.PublishMessage + info.PublishMessage = resp.StorageDealSubmission }, nil } func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) { log.Infow("DEAL ACCEPTED!") - pubmsg, err := c.chain.GetMessage(*deal.PublishMessage) - if err != nil { - return nil, xerrors.Errorf("getting deal pubsish message: %w", err) - } - + pubmsg := deal.PublishMessage.Message pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider) if err != nil { return nil, xerrors.Errorf("getting miner worker failed: %w", err) @@ -91,7 +90,7 @@ func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDea dealIdx := -1 for i, storageDeal := range params.Deals { // TODO: make it less hacky - eq, err := cborutil.Equals(&deal.Proposal, &storageDeal.Proposal) + eq, err := cborutil.Equals(&deal.Proposal, &storageDeal) if err != nil { return nil, err } @@ -102,11 +101,11 @@ func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDea } if dealIdx == -1 { - return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage) + return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage.Cid()) } // TODO: timeout - _, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage) + _, ret, err := c.sm.WaitForMessage(ctx, deal.PublishMessage.Cid()) if err != nil { return nil, xerrors.Errorf("waiting for deal publish message: %w", err) } diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index 56843e7d0..b09639554 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -93,15 +93,8 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) log.Info("publishing deal") - storageDeal := actors.StorageDeal{ - Proposal: deal.Proposal, - } - if err := api.SignWith(ctx, p.full.WalletSign, waddr, &storageDeal); err != nil { - return nil, xerrors.Errorf("signing storage deal failed: ", err) - } - params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{ - Deals: []actors.StorageDeal{storageDeal}, + Deals: []actors.StorageDealProposal{deal.Proposal}, }) if err != nil { return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err) @@ -136,13 +129,11 @@ func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal) } log.Infof("fetching data for a deal %d", resp.DealIDs[0]) - mcid := smsg.Cid() err = p.sendSignedResponse(&Response{ State: api.DealAccepted, - Proposal: deal.ProposalCid, - PublishMessage: &mcid, - StorageDeal: &storageDeal, + Proposal: deal.ProposalCid, + StorageDealSubmission: smsg, }) if err != nil { return nil, err diff --git a/chain/deals/types.go b/chain/deals/types.go index 3ea80ebe7..eebca4f60 100644 --- a/chain/deals/types.go +++ b/chain/deals/types.go @@ -58,8 +58,7 @@ type Response struct { Proposal cid.Cid // DealAccepted - StorageDeal *actors.StorageDeal - PublishMessage *cid.Cid + StorageDealSubmission *types.SignedMessage } // TODO: Do we actually need this to be signed? diff --git a/chain/gen/utils.go b/chain/gen/utils.go index 34cdce95b..c84053a55 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -197,7 +197,7 @@ func SetupStoragePowerActor(bs bstore.Blockstore) (*types.Actor, error) { }, nil } -func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors.StorageDeal) (cid.Cid, error) { +func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors.StorageDealProposal) (cid.Cid, error) { cst := hamt.CSTFromBstore(bs) nd := hamt.NewNode(cst) emptyHAMT, err := cst.Put(context.TODO(), nd) @@ -210,8 +210,16 @@ func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors cdeals := make([]cbg.CBORMarshaler, len(deals)) for i, deal := range deals { cdeals[i] = &actors.OnChainDeal{ - Deal: deal, - ActivationEpoch: 1, + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, + PieceSerialization: deal.PieceSerialization, + Client: deal.Client, + Provider: deal.Provider, + ProposalExpiration: deal.ProposalExpiration, + Duration: deal.Duration, + StoragePricePerEpoch: deal.StoragePricePerEpoch, + StorageCollateral: deal.StorageCollateral, + ActivationEpoch: 1, } } @@ -267,7 +275,7 @@ func mustEnc(i cbg.CBORMarshaler) []byte { return enc } -func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDeal, error) { +func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDealProposal, error) { vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore()) if err != nil { return cid.Undef, nil, xerrors.Errorf("failed to create NewVM: %w", err) @@ -281,7 +289,7 @@ func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid return cid.Undef, nil, xerrors.Errorf("miner address list, and preseal count doesn't match (%d != %d)", len(gmcfg.MinerAddrs), len(gmcfg.PreSeals)) } - var deals []actors.StorageDeal + var deals []actors.StorageDealProposal for i, maddr := range gmcfg.MinerAddrs { ps, psok := gmcfg.PreSeals[maddr.String()] diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index fc7a7888c..d2b60f268 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "math" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index c54a953f5..2b69d6179 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -2,11 +2,11 @@ package seed import ( "context" + "crypto/rand" "crypto/sha256" "encoding/json" "fmt" "io/ioutil" - "math/rand" "os" "path/filepath" @@ -53,7 +53,6 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb return nil, err } - r := rand.New(rand.NewSource(101)) size := sectorbuilder.UserBytesForSectorSize(ssize) var sealedSectors []*genesis.PreSeal @@ -63,7 +62,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb return nil, err } - pi, err := sb.AddPiece(size, sid, r, nil) + pi, err := sb.AddPiece(size, sid, rand.Reader, nil) if err != nil { return nil, err } @@ -153,20 +152,12 @@ func createDeals(m *genesis.GenesisMiner, k *wallet.Key, maddr address.Address, ProposerSignature: nil, } + // TODO: pretty sure we don't even need to sign this if err := api.SignWith(context.TODO(), wallet.KeyWallet(k).Sign, k.Address, proposal); err != nil { return err } - deal := &actors.StorageDeal{ - Proposal: *proposal, - CounterSignature: nil, - } - - if err := api.SignWith(context.TODO(), wallet.KeyWallet(k).Sign, k.Address, deal); err != nil { - return err - } - - sector.Deal = *deal + sector.Deal = *proposal } return nil diff --git a/cmd/lotus-storage-miner/init.go b/cmd/lotus-storage-miner/init.go index c396f2def..e16bc95ce 100644 --- a/cmd/lotus-storage-miner/init.go +++ b/cmd/lotus-storage-miner/init.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "crypto/rand" "encoding/json" @@ -267,7 +268,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin return err } - proposalCid, err := sector.Deal.Proposal.Cid() + proposalCid, err := sector.Deal.Cid() if err != nil { return err } @@ -275,7 +276,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String()) deal := &deals.MinerDeal{ - Proposal: sector.Deal.Proposal, + Proposal: sector.Deal, ProposalCid: proposalCid, State: lapi.DealComplete, Ref: proposalCid, // TODO: This is super wrong, but there @@ -298,7 +299,7 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin return nil } -func findMarketDealID(ctx context.Context, api lapi.FullNode, deal actors.StorageDeal) (uint64, error) { +func findMarketDealID(ctx context.Context, api lapi.FullNode, deal actors.StorageDealProposal) (uint64, error) { // TODO: find a better way // (this is only used by genesis miners) @@ -308,11 +309,7 @@ func findMarketDealID(ctx context.Context, api lapi.FullNode, deal actors.Storag } for k, v := range deals { - eq, err := cborutil.Equals(&v.Deal, &deal) - if err != nil { - return 0, err - } - if eq { + if bytes.Equal(v.PieceRef, deal.PieceRef) { return strconv.ParseUint(k, 10, 64) } } diff --git a/gen/main.go b/gen/main.go index 8fd251eb7..9356f5a8b 100644 --- a/gen/main.go +++ b/gen/main.go @@ -122,7 +122,6 @@ func main() { actors.StorageMarketState{}, actors.WithdrawBalanceParams{}, actors.StorageDealProposal{}, - actors.StorageDeal{}, actors.PublishStorageDealsParams{}, actors.PublishStorageDealResponse{}, actors.ActivateStorageDealsParams{}, diff --git a/genesis/types.go b/genesis/types.go index e8a208452..712c116cc 100644 --- a/genesis/types.go +++ b/genesis/types.go @@ -10,7 +10,7 @@ type PreSeal struct { CommR [32]byte CommD [32]byte SectorID uint64 - Deal actors.StorageDeal + Deal actors.StorageDealProposal } type GenesisMiner struct { diff --git a/storage/garbage.go b/storage/garbage.go index 8b60392cf..7166171ce 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -20,7 +20,7 @@ func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, existingPiece return nil, nil } - deals := make([]actors.StorageDeal, len(sizes)) + deals := make([]actors.StorageDealProposal, len(sizes)) for i, size := range sizes { release := m.sb.RateLimit() commP, err := sectorbuilder.GeneratePieceCommitment(io.LimitReader(rand.New(rand.NewSource(42)), int64(size)), size) @@ -47,14 +47,7 @@ func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, existingPiece return nil, xerrors.Errorf("signing storage deal failed: ", err) } - storageDeal := actors.StorageDeal{ - Proposal: sdp, - } - if err := api.SignWith(ctx, m.api.WalletSign, m.worker, &storageDeal); err != nil { - return nil, xerrors.Errorf("signing storage deal failed: ", err) - } - - deals[i] = storageDeal + deals[i] = sdp } params, aerr := actors.SerializeParams(&actors.PublishStorageDealsParams{ From 98f252da65069b7a0cf2eb8b4dcc6ef26767075e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sat, 7 Dec 2019 21:36:15 +0100 Subject: [PATCH 04/14] better error message for sector failure --- storage/sector_states.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/sector_states.go b/storage/sector_states.go index 5d2a616a8..ce33b3198 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -223,7 +223,7 @@ func (m *Miner) handleCommitWait(ctx context.Context, sector SectorInfo) *sector if mw.Receipt.ExitCode != 0 { log.Errorf("UNHANDLED: submitting sector proof failed (exit=%d, msg=%s) (t:%x; s:%x(%d); p:%x)", mw.Receipt.ExitCode, sector.CommitMessage, sector.Ticket.TicketBytes, sector.Seed.TicketBytes, sector.Seed.BlockHeight, sector.Proof) - return sector.upd().fatal(xerrors.New("UNHANDLED: submitting sector proof failed")) + return sector.upd().fatal(xerrors.New("UNHANDLED: submitting sector proof failed (exit: %d)", mw.Receipt.ExitCode)) } return sector.upd().to(api.Proving).state(func(info *SectorInfo) { From eba3f4af8663544e52497e9cbb86d9d375c287b2 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sun, 8 Dec 2019 23:39:08 +0100 Subject: [PATCH 05/14] address PR review --- chain/actors/actor_storagemarket.go | 28 ++++++++++++--------- chain/actors/cbor_gen.go | 38 +++-------------------------- chain/deals/client.go | 1 - chain/deals/provider_states.go | 7 ------ chain/gen/utils.go | 1 - cmd/lotus-seed/seed/seed.go | 1 - go.mod | 1 + go.sum | 5 ++++ storage/garbage.go | 1 - storage/sector_states.go | 2 +- 10 files changed, 27 insertions(+), 58 deletions(-) diff --git a/chain/actors/actor_storagemarket.go b/chain/actors/actor_storagemarket.go index 84dd2a306..dd3e78bec 100644 --- a/chain/actors/actor_storagemarket.go +++ b/chain/actors/actor_storagemarket.go @@ -3,6 +3,7 @@ package actors import ( "bytes" "context" + "sort" "golang.org/x/xerrors" @@ -74,9 +75,8 @@ const ( ) type StorageDealProposal struct { - PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea - PieceSize uint64 - PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid + PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea + PieceSize uint64 Client address.Address Provider address.Address @@ -133,9 +133,8 @@ func (sdp *StorageDealProposal) Verify() error { } type OnChainDeal struct { - PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea - PieceSize uint64 - PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid + PieceRef []byte // cid bytes // TODO: spec says to use cid.Cid, probably not a good idea + PieceSize uint64 Client address.Address Provider address.Address @@ -230,9 +229,15 @@ func (sma StorageMarketActor) AddBalance(act *types.Actor, vmctx types.VMContext } func setMarketBalances(vmctx types.VMContext, nd *hamt.Node, set map[address.Address]StorageParticipantBalance) (cid.Cid, ActorError) { - // TODO: iterating over a map might happen in the wrong order, this could have gas implications - for addr, b := range set { - balance := b // to stop linter complaining + keys := make([]address.Address, 0, len(set)) + for k := range set { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + return bytes.Compare(keys[i].Bytes(), keys[j].Bytes()) < 0 + }) + for _, addr := range keys { + balance := set[addr] if err := nd.Set(vmctx.Context(), string(addr.Bytes()), &balance); err != nil { return cid.Undef, aerrors.HandleExternalError(err, "setting new balance") } @@ -329,9 +334,8 @@ func (sma StorageMarketActor) PublishStorageDeals(act *types.Actor, vmctx types. } err := deals.Set(self.NextDealID, &OnChainDeal{ - PieceRef: deal.PieceRef, - PieceSize: deal.PieceSize, - PieceSerialization: deal.PieceSerialization, // TODO: this isnt needed anymore, right? + PieceRef: deal.PieceRef, + PieceSize: deal.PieceSize, Client: deal.Client, Provider: deal.Provider, diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 08d52cc48..ed086513e 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -3159,7 +3159,7 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{138}); err != nil { + if _, err := w.Write([]byte{137}); err != nil { return err } @@ -3176,11 +3176,6 @@ func (t *StorageDealProposal) MarshalCBOR(w io.Writer) error { return err } - // t.t.PieceSerialization (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSerialization))); err != nil { - return err - } - // t.t.Client (address.Address) (struct) if err := t.Client.MarshalCBOR(w); err != nil { return err @@ -3229,7 +3224,7 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 10 { + if extra != 9 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -3260,16 +3255,6 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.PieceSize = uint64(extra) - // t.t.PieceSerialization (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.PieceSerialization = uint64(extra) // t.t.Client (address.Address) (struct) { @@ -3627,7 +3612,7 @@ func (t *OnChainDeal) MarshalCBOR(w io.Writer) error { _, err := w.Write(cbg.CborNull) return err } - if _, err := w.Write([]byte{138}); err != nil { + if _, err := w.Write([]byte{137}); err != nil { return err } @@ -3644,11 +3629,6 @@ func (t *OnChainDeal) MarshalCBOR(w io.Writer) error { return err } - // t.t.PieceSerialization (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.PieceSerialization))); err != nil { - return err - } - // t.t.Client (address.Address) (struct) if err := t.Client.MarshalCBOR(w); err != nil { return err @@ -3697,7 +3677,7 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 10 { + if extra != 9 { return fmt.Errorf("cbor input had wrong number of fields") } @@ -3728,16 +3708,6 @@ func (t *OnChainDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.PieceSize = uint64(extra) - // t.t.PieceSerialization (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.PieceSerialization = uint64(extra) // t.t.Client (address.Address) (struct) { diff --git a/chain/deals/client.go b/chain/deals/client.go index 52f53f8d6..8580a2bbd 100644 --- a/chain/deals/client.go +++ b/chain/deals/client.go @@ -206,7 +206,6 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro dealProposal := &actors.StorageDealProposal{ PieceRef: commP, PieceSize: uint64(pieceSize), - PieceSerialization: actors.SerializationUnixFSv0, Client: p.Client, Provider: p.ProviderAddress, ProposalExpiration: p.ProposalExpiration, diff --git a/chain/deals/provider_states.go b/chain/deals/provider_states.go index b09639554..4faa44973 100644 --- a/chain/deals/provider_states.go +++ b/chain/deals/provider_states.go @@ -42,13 +42,6 @@ func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandle // ACCEPTED func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { - switch deal.Proposal.PieceSerialization { - //case SerializationRaw: - //case SerializationIPLD: - case actors.SerializationUnixFSv0: - default: - return nil, xerrors.Errorf("deal proposal with unsupported serialization: %s", deal.Proposal.PieceSerialization) - } head, err := p.full.ChainHead(ctx) if err != nil { diff --git a/chain/gen/utils.go b/chain/gen/utils.go index c84053a55..e4da3cde2 100644 --- a/chain/gen/utils.go +++ b/chain/gen/utils.go @@ -212,7 +212,6 @@ func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors cdeals[i] = &actors.OnChainDeal{ PieceRef: deal.PieceRef, PieceSize: deal.PieceSize, - PieceSerialization: deal.PieceSerialization, Client: deal.Client, Provider: deal.Provider, ProposalExpiration: deal.ProposalExpiration, diff --git a/cmd/lotus-seed/seed/seed.go b/cmd/lotus-seed/seed/seed.go index 2b69d6179..1fa9cc65e 100644 --- a/cmd/lotus-seed/seed/seed.go +++ b/cmd/lotus-seed/seed/seed.go @@ -142,7 +142,6 @@ func createDeals(m *genesis.GenesisMiner, k *wallet.Key, maddr address.Address, proposal := &actors.StorageDealProposal{ PieceRef: pref, // just one deal so this == CommP PieceSize: sectorbuilder.UserBytesForSectorSize(ssize), - PieceSerialization: actors.SerializationUnixFSv0, Client: k.Address, Provider: maddr, ProposalExpiration: 9000, // TODO: allow setting diff --git a/go.mod b/go.mod index 795e4eb39..1537c70f7 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,7 @@ require ( github.com/onsi/ginkgo v1.9.0 // indirect github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/opentracing-go v1.1.0 + github.com/otiai10/copy v1.0.2 // indirect github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/smartystreets/assertions v1.0.1 // indirect github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect diff --git a/go.sum b/go.sum index e06831208..937485ffa 100644 --- a/go.sum +++ b/go.sum @@ -510,6 +510,11 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc= +github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/storage/garbage.go b/storage/garbage.go index 7166171ce..36792fd2d 100644 --- a/storage/garbage.go +++ b/storage/garbage.go @@ -33,7 +33,6 @@ func (m *Miner) storeGarbage(ctx context.Context, sectorID uint64, existingPiece sdp := actors.StorageDealProposal{ PieceRef: commP[:], PieceSize: size, - PieceSerialization: actors.SerializationUnixFSv0, Client: m.worker, Provider: m.maddr, ProposalExpiration: math.MaxUint64, diff --git a/storage/sector_states.go b/storage/sector_states.go index ce33b3198..d62654942 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -223,7 +223,7 @@ func (m *Miner) handleCommitWait(ctx context.Context, sector SectorInfo) *sector if mw.Receipt.ExitCode != 0 { log.Errorf("UNHANDLED: submitting sector proof failed (exit=%d, msg=%s) (t:%x; s:%x(%d); p:%x)", mw.Receipt.ExitCode, sector.CommitMessage, sector.Ticket.TicketBytes, sector.Seed.TicketBytes, sector.Seed.BlockHeight, sector.Proof) - return sector.upd().fatal(xerrors.New("UNHANDLED: submitting sector proof failed (exit: %d)", mw.Receipt.ExitCode)) + return sector.upd().fatal(xerrors.Errorf("UNHANDLED: submitting sector proof failed (exit: %d)", mw.Receipt.ExitCode)) } return sector.upd().to(api.Proving).state(func(info *SectorInfo) { From 7c89dd5339066ae82a90c25f732836331a711561 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sun, 8 Dec 2019 23:41:19 +0100 Subject: [PATCH 06/14] go mod tidy --- go.mod | 1 - go.sum | 5 ----- 2 files changed, 6 deletions(-) diff --git a/go.mod b/go.mod index 1537c70f7..795e4eb39 100644 --- a/go.mod +++ b/go.mod @@ -80,7 +80,6 @@ require ( github.com/onsi/ginkgo v1.9.0 // indirect github.com/onsi/gomega v1.6.0 // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/otiai10/copy v1.0.2 // indirect github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/smartystreets/assertions v1.0.1 // indirect github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect diff --git a/go.sum b/go.sum index 937485ffa..e06831208 100644 --- a/go.sum +++ b/go.sum @@ -510,11 +510,6 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc= -github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY= -github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= -github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc= -github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From d5103a7fbde2bacbbbfb0815ee95d6e438706589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 9 Dec 2019 13:15:05 +0100 Subject: [PATCH 07/14] Fix rterieval of large files --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 795e4eb39..9e1851dd8 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba - github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d + github.com/whyrusleeping/cbor-gen v0.0.0-20191208220313-d43e400b4942 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d go.opencensus.io v0.22.1 diff --git a/go.sum b/go.sum index e06831208..55b092acb 100644 --- a/go.sum +++ b/go.sum @@ -580,6 +580,8 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:x github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d h1:NRa/Vs7+b91GdXrp0AqsG7pspWV6CLk5Gk7i46L4tGo= github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20191208220313-d43e400b4942 h1:EIKesTogdQi76lVOmZleTRQtNNIXSy037QVBVMZ8rug= +github.com/whyrusleeping/cbor-gen v0.0.0-20191208220313-d43e400b4942/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= From 570d725ce55f366101129116d9d678eb4f9351d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 9 Dec 2019 15:41:44 +0100 Subject: [PATCH 08/14] type-gen --- api/cbor_gen.go | 6 ++- chain/actors/cbor_gen.go | 105 ++++++++++++++++++++---------------- chain/blocksync/cbor_gen.go | 29 ++++++---- chain/types/cbor_gen.go | 71 +++++++++++++----------- paych/cbor_gen.go | 9 ++-- retrieval/cbor_gen.go | 12 ++--- storage/cbor_gen.go | 51 +++++++++--------- 7 files changed, 158 insertions(+), 125 deletions(-) diff --git a/api/cbor_gen.go b/api/cbor_gen.go index 0cd224876..15464c648 100644 --- a/api/cbor_gen.go +++ b/api/cbor_gen.go @@ -105,7 +105,8 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Vouchers: array too large (%d)", extra) } @@ -244,7 +245,8 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Refs: array too large (%d)", extra) } diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index e77ac2089..111ba7532 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -136,10 +136,10 @@ func (t *ExecParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Params: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Params: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -611,10 +611,10 @@ func (t *SectorPreCommitInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.CommR: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.CommR: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -638,7 +638,8 @@ func (t *SectorPreCommitInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.DealIDs: array too large (%d)", extra) } @@ -863,10 +864,10 @@ func (t *SubmitFallbackPoStParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Proof: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Proof: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -880,7 +881,8 @@ func (t *SubmitFallbackPoStParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Candidates: array too large (%d)", extra) } @@ -951,10 +953,10 @@ func (t *PaymentVerifyParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Extra: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Extra: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -968,10 +970,10 @@ func (t *PaymentVerifyParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Proof: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Proof: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -1106,7 +1108,8 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Signers: array too large (%d)", extra) } @@ -1181,7 +1184,8 @@ func (t *MultiSigActorState) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Transactions: array too large (%d)", extra) } @@ -1256,7 +1260,8 @@ func (t *MultiSigConstructorParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Signers: array too large (%d)", extra) } @@ -1382,10 +1387,10 @@ func (t *MultiSigProposeParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Params: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Params: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -1679,10 +1684,10 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Params: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Params: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -1696,7 +1701,8 @@ func (t *MTransaction) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Approved: array too large (%d)", extra) } @@ -2260,10 +2266,10 @@ func (t *PCAUpdateChannelStateParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Secret: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Secret: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -2277,10 +2283,10 @@ func (t *PCAUpdateChannelStateParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Proof: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Proof: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -2397,7 +2403,8 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Vouchers: array too large (%d)", extra) } @@ -3239,10 +3246,10 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.PieceRef: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.PieceRef: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -3461,7 +3468,8 @@ func (t *PublishStorageDealsParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Deals: array too large (%d)", extra) } @@ -3526,7 +3534,8 @@ func (t *PublishStorageDealResponse) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.DealIDs: array too large (%d)", extra) } @@ -3595,7 +3604,8 @@ func (t *ActivateStorageDealsParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Deals: array too large (%d)", extra) } @@ -3664,7 +3674,8 @@ func (t *ProcessStorageDealsPaymentParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.DealIDs: array too large (%d)", extra) } @@ -3796,7 +3807,8 @@ func (t *ComputeDataCommitmentParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.DealIDs: array too large (%d)", extra) } @@ -3888,10 +3900,10 @@ func (t *SectorProveCommitInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Proof: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Proof: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -3915,7 +3927,8 @@ func (t *SectorProveCommitInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.DealIDs: array too large (%d)", extra) } diff --git a/chain/blocksync/cbor_gen.go b/chain/blocksync/cbor_gen.go index 579b3f857..2f6be4e18 100644 --- a/chain/blocksync/cbor_gen.go +++ b/chain/blocksync/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "github.com/filecoin-project/lotus/chain/types" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -66,7 +66,8 @@ func (t *BlockSyncRequest) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Start: array too large (%d)", extra) } @@ -163,7 +164,8 @@ func (t *BlockSyncResponse) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Chain: array too large (%d)", extra) } @@ -298,7 +300,8 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Blocks: array too large (%d)", extra) } @@ -324,7 +327,8 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.BlsMessages: array too large (%d)", extra) } @@ -350,7 +354,8 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.BlsMsgIncludes: array too large (%d)", extra) } @@ -370,7 +375,8 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.BlsMsgIncludes[i]: array too large (%d)", extra) } @@ -403,7 +409,8 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.SecpkMessages: array too large (%d)", extra) } @@ -429,7 +436,8 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.SecpkMsgIncludes: array too large (%d)", extra) } @@ -449,7 +457,8 @@ func (t *BSTipSet) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.SecpkMsgIncludes[i]: array too large (%d)", extra) } diff --git a/chain/types/cbor_gen.go b/chain/types/cbor_gen.go index fc7a7888c..9a359cd45 100644 --- a/chain/types/cbor_gen.go +++ b/chain/types/cbor_gen.go @@ -5,7 +5,7 @@ import ( "io" "math" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -153,7 +153,8 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Parents: array too large (%d)", extra) } @@ -310,10 +311,10 @@ func (t *Ticket) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.VRFProof: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.VRFProof: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -382,10 +383,10 @@ func (t *EPostProof) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Proof: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Proof: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -399,10 +400,10 @@ func (t *EPostProof) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.PostRand: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.PostRand: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -416,7 +417,8 @@ func (t *EPostProof) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Candidates: array too large (%d)", extra) } @@ -489,10 +491,10 @@ func (t *EPostTicket) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Partial: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Partial: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -663,10 +665,10 @@ func (t *Message) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Params: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Params: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -895,10 +897,10 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.SecretPreimage: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.SecretPreimage: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -972,7 +974,8 @@ func (t *SignedVoucher) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Merges: array too large (%d)", extra) } @@ -1085,10 +1088,10 @@ func (t *ModVerifyParams) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Data: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Data: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -1315,10 +1318,10 @@ func (t *MessageReceipt) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Return: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Return: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -1416,7 +1419,8 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.BlsMessages: array too large (%d)", extra) } @@ -1441,7 +1445,8 @@ func (t *BlockMsg) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.SecpkMessages: array too large (%d)", extra) } @@ -1718,7 +1723,8 @@ func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Cids: array too large (%d)", extra) } @@ -1743,7 +1749,8 @@ func (t *ExpTipSet) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Blocks: array too large (%d)", extra) } diff --git a/paych/cbor_gen.go b/paych/cbor_gen.go index e640f3057..ef13863ad 100644 --- a/paych/cbor_gen.go +++ b/paych/cbor_gen.go @@ -79,10 +79,10 @@ func (t *VoucherInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Proof: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Proof: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -197,7 +197,8 @@ func (t *ChannelInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Vouchers: array too large (%d)", extra) } diff --git a/retrieval/cbor_gen.go b/retrieval/cbor_gen.go index 3feb15dd1..9881a370f 100644 --- a/retrieval/cbor_gen.go +++ b/retrieval/cbor_gen.go @@ -432,10 +432,10 @@ func (t *Block) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Prefix: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Prefix: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -449,10 +449,10 @@ func (t *Block) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Data: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Data: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } diff --git a/storage/cbor_gen.go b/storage/cbor_gen.go index aa1eabd47..92a5999d4 100644 --- a/storage/cbor_gen.go +++ b/storage/cbor_gen.go @@ -67,10 +67,10 @@ func (t *SealTicket) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.TicketBytes: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.TicketBytes: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -136,10 +136,10 @@ func (t *SealSeed) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.TicketBytes: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.TicketBytes: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -220,10 +220,10 @@ func (t *Piece) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.CommP: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.CommP: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -388,7 +388,8 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { + + if extra > cbg.MaxLength { return fmt.Errorf("t.Pieces: array too large (%d)", extra) } @@ -414,10 +415,10 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Pad0: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Pad0: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -431,10 +432,10 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.CommD: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.CommD: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -448,10 +449,10 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.CommR: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.CommR: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -465,10 +466,10 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Pad1: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Pad1: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } @@ -482,10 +483,10 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { if err != nil { return err } - if extra > 8192 { - return fmt.Errorf("t.Proof: array too large (%d)", extra) - } + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Proof: byte array too large (%d)", extra) + } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } From 1a2401a8161c43e9d9fb0d1b3737195fcb48d56f Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 9 Dec 2019 16:49:46 +0100 Subject: [PATCH 09/14] Add short tests License: MIT Signed-off-by: Jakub Sztandera --- .circleci/config.yml | 10 +++++++--- lib/sectorbuilder/sectorbuilder_test.go | 9 +++++++++ node/node_test.go | 3 +++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ad761862..c5bdeaa1b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -76,7 +76,7 @@ jobs: - store_artifacts: path: lotus-storage-miner - test: + test: &test description: | Run tests with gotestsum. parameters: @@ -154,6 +154,9 @@ jobs: - "~/go/src/github.com" - "~/go/src/golang.org" + test-short: + <<: *test + build_macos: description: build darwin lotus binary macos: @@ -254,8 +257,9 @@ workflows: jobs: - lint-changes: args: "--new-from-rev origin/master" - - test: - codecov-upload: true + - test + - test-short: + go-test-flags: "--timeout 10m --short" - mod-tidy-check - build-all - build_macos: diff --git a/lib/sectorbuilder/sectorbuilder_test.go b/lib/sectorbuilder/sectorbuilder_test.go index 13c80f0fc..5aa03b447 100644 --- a/lib/sectorbuilder/sectorbuilder_test.go +++ b/lib/sectorbuilder/sectorbuilder_test.go @@ -124,6 +124,9 @@ func post(t *testing.T, sb *sectorbuilder.SectorBuilder, seals ...seal) time.Tim } func TestSealAndVerify(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware t.Skip("this is slow") } @@ -193,6 +196,9 @@ func TestSealAndVerify(t *testing.T) { } func TestSealPoStNoCommit(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware t.Skip("this is slow") } @@ -259,6 +265,9 @@ func TestSealPoStNoCommit(t *testing.T) { } func TestSealAndVerify2(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } if runtime.NumCPU() < 10 && os.Getenv("CI") == "" { // don't bother on slow hardware t.Skip("this is slow") } diff --git a/node/node_test.go b/node/node_test.go index 28d66a013..0fe2ac7f9 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -292,5 +292,8 @@ func TestAPIRPC(t *testing.T) { } func TestAPIDealFlow(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } test.TestDealFlow(t, builder) } From c49dc28a04c56b3761e507dacb1f0d81757c24e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 9 Dec 2019 18:08:32 +0100 Subject: [PATCH 10/14] Move api struct to a seprate pkg --- api/api_common.go | 2 + api/{ => apistruct}/permissioned.go | 30 +++---- api/{ => apistruct}/struct.go | 125 ++++++++++++++-------------- api/client/client.go | 7 +- cli/auth.go | 8 +- cmd/lotus-storage-miner/run.go | 3 +- cmd/lotus/rpc.go | 5 +- lib/auth/handler.go | 6 +- node/impl/common.go | 1 - node/impl/storminer.go | 3 +- node/modules/core.go | 4 +- 11 files changed, 101 insertions(+), 93 deletions(-) rename api/{ => apistruct}/permissioned.go (68%) rename api/{ => apistruct}/struct.go (86%) diff --git a/api/api_common.go b/api/api_common.go index c83a4c260..ee99f6d76 100644 --- a/api/api_common.go +++ b/api/api_common.go @@ -10,6 +10,8 @@ import ( "github.com/filecoin-project/lotus/build" ) +type Permission = string + type Common interface { // Auth AuthVerify(ctx context.Context, token string) ([]Permission, error) diff --git a/api/permissioned.go b/api/apistruct/permissioned.go similarity index 68% rename from api/permissioned.go rename to api/apistruct/permissioned.go index 58d027278..4c29f6688 100644 --- a/api/permissioned.go +++ b/api/apistruct/permissioned.go @@ -1,50 +1,50 @@ -package api +package apistruct import ( "context" "reflect" "golang.org/x/xerrors" + + "github.com/filecoin-project/lotus/api" ) type permKey int var permCtxKey permKey -type Permission = string - const ( // When changing these, update docs/API.md too - PermRead Permission = "read" // default - PermWrite Permission = "write" - PermSign Permission = "sign" // Use wallet keys for signing - PermAdmin Permission = "admin" // Manage permissions + PermRead api.Permission = "read" // default + PermWrite api.Permission = "write" + PermSign api.Permission = "sign" // Use wallet keys for signing + PermAdmin api.Permission = "admin" // Manage permissions ) -var AllPermissions = []Permission{PermRead, PermWrite, PermSign, PermAdmin} -var defaultPerms = []Permission{PermRead} +var AllPermissions = []api.Permission{PermRead, PermWrite, PermSign, PermAdmin} +var defaultPerms = []api.Permission{PermRead} -func WithPerm(ctx context.Context, perms []Permission) context.Context { +func WithPerm(ctx context.Context, perms []api.Permission) context.Context { return context.WithValue(ctx, permCtxKey, perms) } -func PermissionedStorMinerAPI(a StorageMiner) StorageMiner { +func PermissionedStorMinerAPI(a api.StorageMiner) api.StorageMiner { var out StorageMinerStruct permissionedAny(a, &out.Internal) permissionedAny(a, &out.CommonStruct.Internal) return &out } -func PermissionedFullAPI(a FullNode) FullNode { +func PermissionedFullAPI(a api.FullNode) api.FullNode { var out FullNodeStruct permissionedAny(a, &out.Internal) permissionedAny(a, &out.CommonStruct.Internal) return &out } -func HasPerm(ctx context.Context, perm Permission) bool { - callerPerms, ok := ctx.Value(permCtxKey).([]Permission) +func HasPerm(ctx context.Context, perm api.Permission) bool { + callerPerms, ok := ctx.Value(permCtxKey).([]api.Permission) if !ok { callerPerms = defaultPerms } @@ -63,7 +63,7 @@ func permissionedAny(in interface{}, out interface{}) { for f := 0; f < rint.NumField(); f++ { field := rint.Type().Field(f) - requiredPerm := Permission(field.Tag.Get("perm")) + requiredPerm := api.Permission(field.Tag.Get("perm")) if requiredPerm == "" { panic("missing 'perm' tag on " + field.Name) // ok } diff --git a/api/struct.go b/api/apistruct/struct.go similarity index 86% rename from api/struct.go rename to api/apistruct/struct.go index c0ab86713..d2ea6bb8e 100644 --- a/api/struct.go +++ b/api/apistruct/struct.go @@ -1,4 +1,4 @@ -package api +package apistruct import ( "context" @@ -9,6 +9,7 @@ import ( "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/address" "github.com/filecoin-project/lotus/chain/store" @@ -20,8 +21,8 @@ var _ = AllPermissions type CommonStruct struct { Internal struct { - AuthVerify func(ctx context.Context, token string) ([]Permission, error) `perm:"read"` - AuthNew func(ctx context.Context, perms []Permission) ([]byte, error) `perm:"admin"` + AuthVerify func(ctx context.Context, token string) ([]api.Permission, error) `perm:"read"` + AuthNew func(ctx context.Context, perms []api.Permission) ([]byte, error) `perm:"admin"` NetConnectedness func(context.Context, peer.ID) (network.Connectedness, error) `perm:"read"` NetPeers func(context.Context) ([]peer.AddrInfo, error) `perm:"read"` @@ -29,8 +30,8 @@ type CommonStruct struct { NetAddrsListen func(context.Context) (peer.AddrInfo, error) `perm:"read"` NetDisconnect func(context.Context, peer.ID) error `perm:"write"` - ID func(context.Context) (peer.ID, error) `perm:"read"` - Version func(context.Context) (Version, error) `perm:"read"` + ID func(context.Context) (peer.ID, error) `perm:"read"` + Version func(context.Context) (api.Version, error) `perm:"read"` } } @@ -44,16 +45,16 @@ type FullNodeStruct struct { ChainGetRandomness func(context.Context, types.TipSetKey, int64) ([]byte, error) `perm:"read"` ChainGetBlock func(context.Context, cid.Cid) (*types.BlockHeader, error) `perm:"read"` ChainGetTipSet func(context.Context, types.TipSetKey) (*types.TipSet, error) `perm:"read"` - ChainGetBlockMessages func(context.Context, cid.Cid) (*BlockMessages, error) `perm:"read"` + ChainGetBlockMessages func(context.Context, cid.Cid) (*api.BlockMessages, error) `perm:"read"` ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error) `perm:"read"` - ChainGetParentMessages func(context.Context, cid.Cid) ([]Message, error) `perm:"read"` + ChainGetParentMessages func(context.Context, cid.Cid) ([]api.Message, error) `perm:"read"` ChainGetTipSetByHeight func(context.Context, uint64, *types.TipSet) (*types.TipSet, error) `perm:"read"` ChainReadObj func(context.Context, cid.Cid) ([]byte, error) `perm:"read"` ChainSetHead func(context.Context, *types.TipSet) error `perm:"admin"` ChainGetGenesis func(context.Context) (*types.TipSet, error) `perm:"read"` ChainTipSetWeight func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` - SyncState func(context.Context) (*SyncState, error) `perm:"read"` + SyncState func(context.Context) (*api.SyncState, error) `perm:"read"` SyncSubmitBlock func(ctx context.Context, blk *types.BlockMsg) error `perm:"write"` SyncIncomingBlocks func(ctx context.Context) (<-chan *types.BlockHeader, error) `perm:"read"` @@ -61,7 +62,7 @@ type FullNodeStruct struct { MpoolPush func(context.Context, *types.SignedMessage) error `perm:"write"` MpoolPushMessage func(context.Context, *types.Message) (*types.SignedMessage, error) `perm:"sign"` MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"` - MpoolSub func(context.Context) (<-chan MpoolUpdate, error) `perm:"read"` + MpoolSub func(context.Context) (<-chan api.MpoolUpdate, error) `perm:"read"` MinerCreateBlock func(context.Context, address.Address, *types.TipSet, *types.Ticket, *types.EPostProof, []*types.SignedMessage, uint64, uint64) (*types.BlockMsg, error) `perm:"write"` @@ -77,28 +78,28 @@ type FullNodeStruct struct { WalletImport func(context.Context, *types.KeyInfo) (address.Address, error) `perm:"admin"` ClientImport func(ctx context.Context, path string) (cid.Cid, error) `perm:"admin"` - ClientListImports func(ctx context.Context) ([]Import, error) `perm:"write"` + ClientListImports func(ctx context.Context) ([]api.Import, error) `perm:"write"` ClientHasLocal func(ctx context.Context, root cid.Cid) (bool, error) `perm:"write"` - ClientFindData func(ctx context.Context, root cid.Cid) ([]QueryOffer, error) `perm:"read"` + ClientFindData func(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) `perm:"read"` ClientStartDeal func(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) `perm:"admin"` - ClientGetDealInfo func(context.Context, cid.Cid) (*DealInfo, error) `perm:"read"` - ClientListDeals func(ctx context.Context) ([]DealInfo, error) `perm:"write"` - ClientRetrieve func(ctx context.Context, order RetrievalOrder, path string) error `perm:"admin"` + ClientGetDealInfo func(context.Context, cid.Cid) (*api.DealInfo, error) `perm:"read"` + ClientListDeals func(ctx context.Context) ([]api.DealInfo, error) `perm:"write"` + ClientRetrieve func(ctx context.Context, order api.RetrievalOrder, path string) error `perm:"admin"` ClientQueryAsk func(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) `perm:"read"` - StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"` - StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*ChainSectorInfo, error) `perm:"read"` - StateMinerPower func(context.Context, address.Address, *types.TipSet) (MinerPower, error) `perm:"read"` + StateMinerSectors func(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) `perm:"read"` + StateMinerProvingSet func(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error) `perm:"read"` + StateMinerPower func(context.Context, address.Address, *types.TipSet) (api.MinerPower, error) `perm:"read"` StateMinerWorker func(context.Context, address.Address, *types.TipSet) (address.Address, error) `perm:"read"` StateMinerPeerID func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error) `perm:"read"` StateMinerElectionPeriodStart func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error) `perm:"read"` StateMinerSectorSize func(context.Context, address.Address, *types.TipSet) (uint64, error) `perm:"read"` StateCall func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error) `perm:"read"` - StateReplay func(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error) `perm:"read"` + StateReplay func(context.Context, *types.TipSet, cid.Cid) (*api.ReplayResults, error) `perm:"read"` StateGetActor func(context.Context, address.Address, *types.TipSet) (*types.Actor, error) `perm:"read"` - StateReadState func(context.Context, *types.Actor, *types.TipSet) (*ActorState, error) `perm:"read"` + StateReadState func(context.Context, *types.Actor, *types.TipSet) (*api.ActorState, error) `perm:"read"` StatePledgeCollateral func(context.Context, *types.TipSet) (types.BigInt, error) `perm:"read"` - StateWaitMsg func(context.Context, cid.Cid) (*MsgWait, error) `perm:"read"` + StateWaitMsg func(context.Context, cid.Cid) (*api.MsgWait, error) `perm:"read"` StateListMiners func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` StateListActors func(context.Context, *types.TipSet) ([]address.Address, error) `perm:"read"` StateMarketBalance func(context.Context, address.Address, *types.TipSet) (actors.StorageParticipantBalance, error) `perm:"read"` @@ -111,19 +112,19 @@ type FullNodeStruct struct { MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"` - PaychGet func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) `perm:"sign"` - PaychList func(context.Context) ([]address.Address, error) `perm:"read"` - PaychStatus func(context.Context, address.Address) (*PaychStatus, error) `perm:"read"` - PaychClose func(context.Context, address.Address) (cid.Cid, error) `perm:"sign"` - PaychAllocateLane func(context.Context, address.Address) (uint64, error) `perm:"sign"` - PaychNewPayment func(ctx context.Context, from, to address.Address, vouchers []VoucherSpec) (*PaymentInfo, error) `perm:"sign"` - PaychVoucherCheck func(context.Context, *types.SignedVoucher) error `perm:"read"` - PaychVoucherCheckValid func(context.Context, address.Address, *types.SignedVoucher) error `perm:"read"` - PaychVoucherCheckSpendable func(context.Context, address.Address, *types.SignedVoucher, []byte, []byte) (bool, error) `perm:"read"` - PaychVoucherAdd func(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) `perm:"write"` + PaychGet func(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error) `perm:"sign"` + PaychList func(context.Context) ([]address.Address, error) `perm:"read"` + PaychStatus func(context.Context, address.Address) (*api.PaychStatus, error) `perm:"read"` + PaychClose func(context.Context, address.Address) (cid.Cid, error) `perm:"sign"` + PaychAllocateLane func(context.Context, address.Address) (uint64, error) `perm:"sign"` + PaychNewPayment func(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) `perm:"sign"` + PaychVoucherCheck func(context.Context, *types.SignedVoucher) error `perm:"read"` + PaychVoucherCheckValid func(context.Context, address.Address, *types.SignedVoucher) error `perm:"read"` + PaychVoucherCheckSpendable func(context.Context, address.Address, *types.SignedVoucher, []byte, []byte) (bool, error) `perm:"read"` + PaychVoucherAdd func(context.Context, address.Address, *types.SignedVoucher, []byte, types.BigInt) (types.BigInt, error) `perm:"write"` PaychVoucherCreate func(context.Context, address.Address, types.BigInt, uint64) (*types.SignedVoucher, error) `perm:"sign"` PaychVoucherList func(context.Context, address.Address) ([]*types.SignedVoucher, error) `perm:"write"` - PaychVoucherSubmit func(context.Context, address.Address, *types.SignedVoucher) (cid.Cid, error) `perm:"sign"` + PaychVoucherSubmit func(context.Context, address.Address, *types.SignedVoucher) (cid.Cid, error) `perm:"sign"` } } @@ -136,10 +137,10 @@ type StorageMinerStruct struct { PledgeSector func(context.Context) error `perm:"write"` - SectorsStatus func(context.Context, uint64) (SectorInfo, error) `perm:"read"` - SectorsList func(context.Context) ([]uint64, error) `perm:"read"` - SectorsRefs func(context.Context) (map[string][]SealedRef, error) `perm:"read"` - SectorsUpdate func(context.Context, uint64, SectorState) error `perm:"write"` + SectorsStatus func(context.Context, uint64) (api.SectorInfo, error) `perm:"read"` + SectorsList func(context.Context) ([]uint64, error) `perm:"read"` + SectorsRefs func(context.Context) (map[string][]api.SealedRef, error) `perm:"read"` + SectorsUpdate func(context.Context, uint64, api.SectorState) error `perm:"write"` WorkerStats func(context.Context) (sectorbuilder.WorkerStats, error) `perm:"read"` @@ -148,11 +149,11 @@ type StorageMinerStruct struct { } } -func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]Permission, error) { +func (c *CommonStruct) AuthVerify(ctx context.Context, token string) ([]api.Permission, error) { return c.Internal.AuthVerify(ctx, token) } -func (c *CommonStruct) AuthNew(ctx context.Context, perms []Permission) ([]byte, error) { +func (c *CommonStruct) AuthNew(ctx context.Context, perms []api.Permission) ([]byte, error) { return c.Internal.AuthNew(ctx, perms) } @@ -182,11 +183,11 @@ func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) { } // Version implements API.Version -func (c *CommonStruct) Version(ctx context.Context) (Version, error) { +func (c *CommonStruct) Version(ctx context.Context) (api.Version, error) { return c.Internal.Version(ctx) } -func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]Import, error) { +func (c *FullNodeStruct) ClientListImports(ctx context.Context) ([]api.Import, error) { return c.Internal.ClientListImports(ctx) } @@ -198,22 +199,22 @@ func (c *FullNodeStruct) ClientHasLocal(ctx context.Context, root cid.Cid) (bool return c.Internal.ClientHasLocal(ctx, root) } -func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]QueryOffer, error) { +func (c *FullNodeStruct) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffer, error) { return c.Internal.ClientFindData(ctx, root) } func (c *FullNodeStruct) ClientStartDeal(ctx context.Context, data cid.Cid, miner address.Address, price types.BigInt, blocksDuration uint64) (*cid.Cid, error) { return c.Internal.ClientStartDeal(ctx, data, miner, price, blocksDuration) } -func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*DealInfo, error) { +func (c *FullNodeStruct) ClientGetDealInfo(ctx context.Context, deal cid.Cid) (*api.DealInfo, error) { return c.Internal.ClientGetDealInfo(ctx, deal) } -func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]DealInfo, error) { +func (c *FullNodeStruct) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) { return c.Internal.ClientListDeals(ctx) } -func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order RetrievalOrder, path string) error { +func (c *FullNodeStruct) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path string) error { return c.Internal.ClientRetrieve(ctx, order, path) } @@ -233,7 +234,7 @@ func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Messag return c.Internal.MpoolPushMessage(ctx, msg) } -func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan MpoolUpdate, error) { +func (c *FullNodeStruct) MpoolSub(ctx context.Context) (<-chan api.MpoolUpdate, error) { return c.Internal.MpoolSub(ctx) } @@ -305,7 +306,7 @@ func (c *FullNodeStruct) ChainGetTipSet(ctx context.Context, key types.TipSetKey return c.Internal.ChainGetTipSet(ctx, key) } -func (c *FullNodeStruct) ChainGetBlockMessages(ctx context.Context, b cid.Cid) (*BlockMessages, error) { +func (c *FullNodeStruct) ChainGetBlockMessages(ctx context.Context, b cid.Cid) (*api.BlockMessages, error) { return c.Internal.ChainGetBlockMessages(ctx, b) } @@ -313,7 +314,7 @@ func (c *FullNodeStruct) ChainGetParentReceipts(ctx context.Context, b cid.Cid) return c.Internal.ChainGetParentReceipts(ctx, b) } -func (c *FullNodeStruct) ChainGetParentMessages(ctx context.Context, b cid.Cid) ([]Message, error) { +func (c *FullNodeStruct) ChainGetParentMessages(ctx context.Context, b cid.Cid) ([]api.Message, error) { return c.Internal.ChainGetParentMessages(ctx, b) } @@ -337,7 +338,7 @@ func (c *FullNodeStruct) ChainTipSetWeight(ctx context.Context, ts *types.TipSet return c.Internal.ChainTipSetWeight(ctx, ts) } -func (c *FullNodeStruct) SyncState(ctx context.Context) (*SyncState, error) { +func (c *FullNodeStruct) SyncState(ctx context.Context) (*api.SyncState, error) { return c.Internal.SyncState(ctx) } @@ -349,15 +350,15 @@ func (c *FullNodeStruct) SyncIncomingBlocks(ctx context.Context) (<-chan *types. return c.Internal.SyncIncomingBlocks(ctx) } -func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) { +func (c *FullNodeStruct) StateMinerSectors(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) { return c.Internal.StateMinerSectors(ctx, addr, ts) } -func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*ChainSectorInfo, error) { +func (c *FullNodeStruct) StateMinerProvingSet(ctx context.Context, addr address.Address, ts *types.TipSet) ([]*api.ChainSectorInfo, error) { return c.Internal.StateMinerProvingSet(ctx, addr, ts) } -func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, ts *types.TipSet) (MinerPower, error) { +func (c *FullNodeStruct) StateMinerPower(ctx context.Context, a address.Address, ts *types.TipSet) (api.MinerPower, error) { return c.Internal.StateMinerPower(ctx, a, ts) } @@ -381,7 +382,7 @@ func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, ts * return c.Internal.StateCall(ctx, msg, ts) } -func (c *FullNodeStruct) StateReplay(ctx context.Context, ts *types.TipSet, mc cid.Cid) (*ReplayResults, error) { +func (c *FullNodeStruct) StateReplay(ctx context.Context, ts *types.TipSet, mc cid.Cid) (*api.ReplayResults, error) { return c.Internal.StateReplay(ctx, ts, mc) } @@ -389,7 +390,7 @@ func (c *FullNodeStruct) StateGetActor(ctx context.Context, actor address.Addres return c.Internal.StateGetActor(ctx, actor, ts) } -func (c *FullNodeStruct) StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*ActorState, error) { +func (c *FullNodeStruct) StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*api.ActorState, error) { return c.Internal.StateReadState(ctx, act, ts) } @@ -397,7 +398,7 @@ func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, ts *types.Ti return c.Internal.StatePledgeCollateral(ctx, ts) } -func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid) (*MsgWait, error) { +func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid) (*api.MsgWait, error) { return c.Internal.StateWaitMsg(ctx, msgc) } func (c *FullNodeStruct) StateListMiners(ctx context.Context, ts *types.TipSet) ([]address.Address, error) { @@ -440,7 +441,7 @@ func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr address return c.Internal.MarketEnsureAvailable(ctx, addr, amt) } -func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*ChannelInfo, error) { +func (c *FullNodeStruct) PaychGet(ctx context.Context, from, to address.Address, ensureFunds types.BigInt) (*api.ChannelInfo, error) { return c.Internal.PaychGet(ctx, from, to, ensureFunds) } @@ -448,7 +449,7 @@ func (c *FullNodeStruct) PaychList(ctx context.Context) ([]address.Address, erro return c.Internal.PaychList(ctx) } -func (c *FullNodeStruct) PaychStatus(ctx context.Context, pch address.Address) (*PaychStatus, error) { +func (c *FullNodeStruct) PaychStatus(ctx context.Context, pch address.Address) (*api.PaychStatus, error) { return c.Internal.PaychStatus(ctx, pch) } @@ -480,7 +481,7 @@ func (c *FullNodeStruct) PaychAllocateLane(ctx context.Context, ch address.Addre return c.Internal.PaychAllocateLane(ctx, ch) } -func (c *FullNodeStruct) PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []VoucherSpec) (*PaymentInfo, error) { +func (c *FullNodeStruct) PaychNewPayment(ctx context.Context, from, to address.Address, vouchers []api.VoucherSpec) (*api.PaymentInfo, error) { return c.Internal.PaychNewPayment(ctx, from, to, vouchers) } @@ -501,7 +502,7 @@ func (c *StorageMinerStruct) PledgeSector(ctx context.Context) error { } // Get the status of a given sector by ID -func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (SectorInfo, error) { +func (c *StorageMinerStruct) SectorsStatus(ctx context.Context, sid uint64) (api.SectorInfo, error) { return c.Internal.SectorsStatus(ctx, sid) } @@ -510,11 +511,11 @@ func (c *StorageMinerStruct) SectorsList(ctx context.Context) ([]uint64, error) return c.Internal.SectorsList(ctx) } -func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]SealedRef, error) { +func (c *StorageMinerStruct) SectorsRefs(ctx context.Context) (map[string][]api.SealedRef, error) { return c.Internal.SectorsRefs(ctx) } -func (c *StorageMinerStruct) SectorsUpdate(ctx context.Context, id uint64, state SectorState) error { +func (c *StorageMinerStruct) SectorsUpdate(ctx context.Context, id uint64, state api.SectorState) error { return c.Internal.SectorsUpdate(ctx, id, state) } @@ -530,6 +531,6 @@ func (c *StorageMinerStruct) WorkerDone(ctx context.Context, task uint64, res se return c.Internal.WorkerDone(ctx, task, res) } -var _ Common = &CommonStruct{} -var _ FullNode = &FullNodeStruct{} -var _ StorageMiner = &StorageMinerStruct{} +var _ api.Common = &CommonStruct{} +var _ api.FullNode = &FullNodeStruct{} +var _ api.StorageMiner = &StorageMinerStruct{} diff --git a/api/client/client.go b/api/client/client.go index ffca93363..0e19f65c2 100644 --- a/api/client/client.go +++ b/api/client/client.go @@ -1,6 +1,7 @@ package client import ( + "github.com/filecoin-project/lotus/api/apistruct" "net/http" "github.com/filecoin-project/lotus/api" @@ -9,7 +10,7 @@ import ( // NewCommonRPC creates a new http jsonrpc client. func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.ClientCloser, error) { - var res api.CommonStruct + var res apistruct.CommonStruct closer, err := jsonrpc.NewMergeClient(addr, "Filecoin", []interface{}{ &res.Internal, @@ -20,7 +21,7 @@ func NewCommonRPC(addr string, requestHeader http.Header) (api.Common, jsonrpc.C // NewFullNodeRPC creates a new http jsonrpc client. func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonrpc.ClientCloser, error) { - var res api.FullNodeStruct + var res apistruct.FullNodeStruct closer, err := jsonrpc.NewMergeClient(addr, "Filecoin", []interface{}{ &res.CommonStruct.Internal, @@ -32,7 +33,7 @@ func NewFullNodeRPC(addr string, requestHeader http.Header) (api.FullNode, jsonr // NewStorageMinerRPC creates a new http jsonrpc client for storage miner func NewStorageMinerRPC(addr string, requestHeader http.Header) (api.StorageMiner, jsonrpc.ClientCloser, error) { - var res api.StorageMinerStruct + var res apistruct.StorageMinerStruct closer, err := jsonrpc.NewMergeClient(addr, "Filecoin", []interface{}{ &res.CommonStruct.Internal, diff --git a/cli/auth.go b/cli/auth.go index 121f39463..d957881b6 100644 --- a/cli/auth.go +++ b/cli/auth.go @@ -6,7 +6,7 @@ import ( "gopkg.in/urfave/cli.v2" - "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apistruct" ) var authCmd = &cli.Command{ @@ -42,18 +42,18 @@ var authCreateAdminToken = &cli.Command{ perm := cctx.String("perm") idx := 0 - for i, p := range api.AllPermissions { + for i, p := range apistruct.AllPermissions { if perm == p { idx = i + 1 } } if idx == 0 { - return fmt.Errorf("--perm flag has to be one of: %s", api.AllPermissions) + return fmt.Errorf("--perm flag has to be one of: %s", apistruct.AllPermissions) } // slice on [:idx] so for example: 'sign' gives you [read, write, sign] - token, err := napi.AuthNew(ctx, api.AllPermissions[:idx]) + token, err := napi.AuthNew(ctx, apistruct.AllPermissions[:idx]) if err != nil { return err } diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 814e93686..72826d585 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -15,6 +15,7 @@ import ( "gopkg.in/urfave/cli.v2" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/build" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/auth" @@ -131,7 +132,7 @@ var runCmd = &cli.Command{ mux := mux.NewRouter() rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", api.PermissionedStorMinerAPI(minerapi)) + rpcServer.Register("Filecoin", apistruct.PermissionedStorMinerAPI(minerapi)) mux.Handle("/rpc/v0", rpcServer) mux.PathPrefix("/remote").HandlerFunc(minerapi.(*impl.StorageMinerAPI).ServeRemote) diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index 3b203fff2..5178e1b6c 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "github.com/filecoin-project/lotus/api/apistruct" "net/http" _ "net/http/pprof" "os" @@ -26,7 +27,7 @@ var log = logging.Logger("main") func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr) error { rpcServer := jsonrpc.NewServer() - rpcServer.Register("Filecoin", api.PermissionedFullAPI(a)) + rpcServer.Register("Filecoin", apistruct.PermissionedFullAPI(a)) ah := &auth.Handler{ Verify: a.AuthVerify, @@ -70,7 +71,7 @@ func handleImport(a *impl.FullNodeAPI) func(w http.ResponseWriter, r *http.Reque w.WriteHeader(404) return } - if !api.HasPerm(r.Context(), api.PermWrite) { + if !apistruct.HasPerm(r.Context(), apistruct.PermWrite) { w.WriteHeader(401) json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"}) return diff --git a/lib/auth/handler.go b/lib/auth/handler.go index 67fc1ac9b..b6112ce8c 100644 --- a/lib/auth/handler.go +++ b/lib/auth/handler.go @@ -5,8 +5,10 @@ import ( "net/http" "strings" - "github.com/filecoin-project/lotus/api" logging "github.com/ipfs/go-log" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/apistruct" ) var log = logging.Logger("auth") @@ -42,7 +44,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - ctx = api.WithPerm(ctx, allow) + ctx = apistruct.WithPerm(ctx, allow) } h.Next(w, r.WithContext(ctx)) diff --git a/node/impl/common.go b/node/impl/common.go index f8037ca1d..990a46a8a 100644 --- a/node/impl/common.go +++ b/node/impl/common.go @@ -2,7 +2,6 @@ package impl import ( "context" - "github.com/gbrlsnchs/jwt/v3" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 2b3a0352e..afec53486 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -3,6 +3,7 @@ package impl import ( "context" "encoding/json" + "github.com/filecoin-project/lotus/api/apistruct" "io" "mime" "net/http" @@ -33,7 +34,7 @@ type StorageMinerAPI struct { } func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) { - if !api.HasPerm(r.Context(), api.PermAdmin) { + if !apistruct.HasPerm(r.Context(), apistruct.PermAdmin) { w.WriteHeader(401) json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing write permission"}) return diff --git a/node/modules/core.go b/node/modules/core.go index 5eab490eb..186ad9936 100644 --- a/node/modules/core.go +++ b/node/modules/core.go @@ -3,10 +3,10 @@ package modules import ( "context" "crypto/rand" + "github.com/filecoin-project/lotus/api/apistruct" "io" "io/ioutil" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/lib/addrutil" @@ -57,7 +57,7 @@ func APISecret(keystore types.KeyStore, lr repo.LockedRepo) (*dtypes.APIAlg, err // TODO: make this configurable p := jwtPayload{ - Allow: api.AllPermissions, + Allow: apistruct.AllPermissions, } cliToken, err := jwt.Sign(&p, jwt.NewHS256(key.PrivateKey)) From 9e363f9266e5ffe062622c202ffeb4a03a3a6e07 Mon Sep 17 00:00:00 2001 From: Henri S Date: Mon, 9 Dec 2019 15:15:25 +0100 Subject: [PATCH 11/14] fix target for ePoSt IsTicketWinner fn License: MIT Signed-off-by: Jakub Sztandera --- chain/gen/gen.go | 4 +++- chain/sync.go | 6 ++++-- chain/types/blockheader.go | 22 +++++++++++++++++++--- lib/sectorbuilder/sectorbuilder.go | 10 +++------- lib/sectorbuilder/simple.go | 3 ++- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/chain/gen/gen.go b/chain/gen/gen.go index 866a75357..8c82895f9 100644 --- a/chain/gen/gen.go +++ b/chain/gen/gen.go @@ -526,9 +526,11 @@ func IsRoundWinner(ctx context.Context, ts *types.TipSet, round int64, miner add return nil, xerrors.Errorf("failed to look up miners sector size: %w", err) } + snum := types.BigDiv(pow.MinerPower, types.NewInt(ssize)) + var winners []sectorbuilder.EPostCandidate for _, c := range candidates { - if types.IsTicketWinner(c.PartialTicket[:], ssize, pow.TotalPower) { + if types.IsTicketWinner(c.PartialTicket[:], ssize, snum.Uint64(), pow.TotalPower) { winners = append(winners, c) } } diff --git a/chain/sync.go b/chain/sync.go index c632c80a0..e594749e9 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -501,7 +501,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err return xerrors.Errorf("received block was from miner slashed at height %d", slashedAt) } - _, tpow, err := stmgr.GetPower(ctx, syncer.sm, baseTs, h.Miner) + mpow, tpow, err := stmgr.GetPower(ctx, syncer.sm, baseTs, h.Miner) if err != nil { return xerrors.Errorf("failed getting power: %w", err) } @@ -511,8 +511,10 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err return xerrors.Errorf("failed to get sector size for block miner: %w", err) } + snum := types.BigDiv(mpow, types.NewInt(ssize)) + for _, t := range h.EPostProof.Candidates { - if !types.IsTicketWinner(t.Partial, ssize, tpow) { + if !types.IsTicketWinner(t.Partial, ssize, snum.Uint64(), tpow) { return xerrors.Errorf("miner created a block but was not a winner") } } diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 7ba5ef436..9d3c0a4b3 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -174,9 +174,9 @@ var blocksPerEpoch = NewInt(build.BlocksPerEpoch) const sha256bits = 256 -func IsTicketWinner(partialTicket []byte, ssizeI uint64, totpow BigInt) bool { +func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow BigInt) bool { ssize := NewInt(ssizeI) - + ssampled := ElectionPostChallengeCount(snum) /* Need to check that (h(vrfout) + 1) / (max(h) + 1) <= e * sectorSize / totalPower @@ -185,23 +185,39 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, totpow BigInt) bool { (h(vrfout) + 1) * totalPower <= e * sectorSize * 2^256 in 2^256 space, it is equivalent to: h(vrfout) * totalPower < e * sectorSize * 2^256 + + Because of SectorChallengeRatioDiv sampling for proofs + we need to scale this appropriately. + + Let c = ceil(numSectors/SectorChallengeRatioDiv) + (c is the number of tickets a miner requests) + Accordingly we check + (h(vrfout) + 1) / 2^256 <= e * sectorSize / totalPower * snum / c + or + h(vrfout) * totalPower * c < e * sectorSize * 2^256 * snum */ h := sha256.Sum256(partialTicket) lhs := BigFromBytes(h[:]).Int lhs = lhs.Mul(lhs, totpow.Int) + lhs = lhs.Mul(lhs, new(big.Int).SetUint64(ssampled)) // rhs = sectorSize * 2^256 // rhs = sectorSize << 256 rhs := new(big.Int).Lsh(ssize.Int, sha256bits) - rhs = rhs.Mul(rhs, big.NewInt(build.SectorChallengeRatioDiv)) + rhs = rhs.Mul(rhs, new(big.Int).SetUint64(snum)) rhs = rhs.Mul(rhs, blocksPerEpoch.Int) // h(vrfout) * totalPower < e * sectorSize * 2^256? return lhs.Cmp(rhs) < 0 } +func ElectionPostChallengeCount(sectors uint64) uint64 { + // ceil(sectors / build.SectorChallengeRatioDiv) + return (sectors + build.SectorChallengeRatioDiv - 1) / build.SectorChallengeRatioDiv +} + func (t *Ticket) Equals(ot *Ticket) bool { return bytes.Equal(t.VRFProof, ot.VRFProof) } diff --git a/lib/sectorbuilder/sectorbuilder.go b/lib/sectorbuilder/sectorbuilder.go index 7a5d744ab..c4fca12d4 100644 --- a/lib/sectorbuilder/sectorbuilder.go +++ b/lib/sectorbuilder/sectorbuilder.go @@ -16,6 +16,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules/dtypes" ) @@ -623,7 +624,7 @@ func (sb *SectorBuilder) GenerateEPostCandidates(sectorInfo SortedPublicSectorIn return nil, err } - challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo.Values()))) + challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values()))) proverID := addressToProverID(sb.Miner) return sectorbuilder.GenerateCandidates(sb.ssize, proverID, challengeSeed, challengeCount, privsectors) @@ -674,13 +675,8 @@ func (sb *SectorBuilder) Stop() { close(sb.stopping) } -func ElectionPostChallengeCount(sectors uint64) uint64 { - // ceil(sectors / build.SectorChallengeRatioDiv) - return (sectors + build.SectorChallengeRatioDiv - 1) / build.SectorChallengeRatioDiv -} - func fallbackPostChallengeCount(sectors uint64) uint64 { - challengeCount := ElectionPostChallengeCount(sectors) + challengeCount := types.ElectionPostChallengeCount(sectors) if challengeCount > build.MaxFallbackPostChallengeCount { return build.MaxFallbackPostChallengeCount } diff --git a/lib/sectorbuilder/simple.go b/lib/sectorbuilder/simple.go index 0c9231092..457c4b2e1 100644 --- a/lib/sectorbuilder/simple.go +++ b/lib/sectorbuilder/simple.go @@ -8,6 +8,7 @@ import ( "go.opencensus.io/trace" "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/types" ) func (sb *SectorBuilder) SectorSize() uint64 { @@ -36,7 +37,7 @@ func NewSortedPublicSectorInfo(sectors []sectorbuilder.PublicSectorInfo) SortedP } func VerifyElectionPost(ctx context.Context, sectorSize uint64, sectorInfo SortedPublicSectorInfo, challengeSeed []byte, proof []byte, candidates []EPostCandidate, proverID address.Address) (bool, error) { - challengeCount := ElectionPostChallengeCount(uint64(len(sectorInfo.Values()))) + challengeCount := types.ElectionPostChallengeCount(uint64(len(sectorInfo.Values()))) return verifyPost(ctx, sectorSize, sectorInfo, challengeCount, challengeSeed, proof, candidates, proverID) } From 887a34135bc1c7d9801462ec76cba941b427b368 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Mon, 9 Dec 2019 17:41:38 +0100 Subject: [PATCH 12/14] Make ElectionPostChallengeCount math overflow safe License: MIT Signed-off-by: Jakub Sztandera --- chain/types/blockheader.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/chain/types/blockheader.go b/chain/types/blockheader.go index 9d3c0a4b3..e44ca54b4 100644 --- a/chain/types/blockheader.go +++ b/chain/types/blockheader.go @@ -214,8 +214,11 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow Big } func ElectionPostChallengeCount(sectors uint64) uint64 { + if sectors == 0 { + return 0 + } // ceil(sectors / build.SectorChallengeRatioDiv) - return (sectors + build.SectorChallengeRatioDiv - 1) / build.SectorChallengeRatioDiv + return (sectors-1)/build.SectorChallengeRatioDiv + 1 } func (t *Ticket) Equals(ot *Ticket) bool { From d2448912a62dfb45d832b338794af216fbaa310c Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 6 Dec 2019 15:06:42 +0100 Subject: [PATCH 13/14] Implement basic fault handling WIP: miner actor tests working miner actor test rebase and fix tests --- api/api_storage.go | 9 ++ chain/actors/actor_miner.go | 104 +++++++++++--------- chain/actors/actor_miner_test.go | 157 +++++++++++++++++++++++++++++++ chain/actors/actor_paych_test.go | 2 +- chain/actors/cbor_gen.go | 70 +++++++++++--- chain/actors/harness2_test.go | 30 +++++- chain/types/bitfield.go | 53 ++++++++++- chain/types/vmcontext.go | 5 + chain/vm/mkactor.go | 2 +- chain/vm/syscalls.go | 14 +++ chain/vm/vm.go | 10 ++ gen/main.go | 1 + storage/cbor_gen.go | 36 +++++++ storage/sector_states.go | 54 +++++++++++ storage/sector_types.go | 3 + storage/sectors.go | 6 ++ 16 files changed, 493 insertions(+), 63 deletions(-) create mode 100644 chain/actors/actor_miner_test.go create mode 100644 chain/vm/syscalls.go diff --git a/api/api_storage.go b/api/api_storage.go index 816ad74bf..5b3c200ce 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -29,6 +29,10 @@ const ( CommitFailed FailedUnrecoverable + + Faulty // sector is corrupted or gone for some reason + FaultReported // sector has been declared as a fault on chain + FaultedFinal // fault declared on chain ) var SectorStates = []string{ @@ -39,6 +43,7 @@ var SectorStates = []string{ PreCommitting: "PreCommitting", PreCommitted: "PreCommitted", Committing: "Committing", + CommitWait: "CommitWait", Proving: "Proving", SealFailed: "SealFailed", @@ -47,6 +52,10 @@ var SectorStates = []string{ CommitFailed: "CommitFailed", FailedUnrecoverable: "FailedUnrecoverable", + + Faulty: "Faulty", + FaultReported: "FaultReported", + FaultedFinal: "FaultedFinal", } // StorageMiner is a low-level interface to the Filecoin network storage miner node diff --git a/chain/actors/actor_miner.go b/chain/actors/actor_miner.go index 6c775eac0..4e1ba5570 100644 --- a/chain/actors/actor_miner.go +++ b/chain/actors/actor_miner.go @@ -53,14 +53,10 @@ type StorageMinerActorState struct { // Contains mostly static info about this miner Info cid.Cid - // Faulty sectors reported since last SubmitPost, - // up to the current proving period's challenge time. - CurrentFaultSet types.BitField + // Faulty sectors reported since last SubmitPost + FaultSet types.BitField - // Faults submitted after the current proving period's challenge time, - // but before the PoSt for that period is submitted. - // These become the currentFaultSet when a PoSt is submitted. - NextFaultSet types.BitField + LastFaultSubmission uint64 // Amount of power this miner has. Power types.BigInt @@ -340,7 +336,7 @@ func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMC return nil, aerrors.Wrapf(err, "failed to compute data commitment (sector %d, deals: %v)", params.SectorID, params.DealIDs) } - if ok, err := ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { + if ok, err := vmctx.Sys().ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil { return nil, err } else if !ok { return nil, aerrors.Newf(2, "porep proof was invalid (t:%x; s:%x(%d); p:%s)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, truncateHexPrint(params.Proof)) @@ -463,8 +459,17 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.HandleExternalError(lerr, "could not load proving set node") } + faults, nerr := self.FaultSet.AllMap() + if nerr != nil { + return nil, aerrors.Absorb(err, 5, "RLE+ invalid") + } + var sectorInfos []ffi.PublicSectorInfo if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error { + if faults[id] { + return nil + } + var comms [][]byte if err := cbor.DecodeInto(v.Raw, &comms); err != nil { return xerrors.New("could not decode comms") @@ -485,12 +490,6 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM return nil, aerrors.Absorb(err, 3, "could not decode sectorset") } - faults, nerr := self.CurrentFaultSet.All() - if nerr != nil { - return nil, aerrors.Absorb(err, 5, "RLE+ invalid") - } - _ = faults - proverID := vmctx.Message().To // TODO: normalize to ID address var candidates []sectorbuilder.EPostCandidate @@ -598,6 +597,25 @@ func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID return true, comms[0], comms[1], nil } +func RemoveFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, ids []uint64) (cid.Cid, aerrors.ActorError) { + + ssr, err := amt.LoadAMT(types.WrapStorage(s), ss) + if err != nil { + return cid.Undef, aerrors.HandleExternalError(err, "could not load sector set node") + } + + if err := ssr.BatchDelete(ids); err != nil { + return cid.Undef, aerrors.HandleExternalError(err, "failed to delete from sector set") + } + + ncid, err := ssr.Flush() + if err != nil { + return cid.Undef, aerrors.HandleExternalError(err, "failed to flush sector set") + } + + return ncid, nil +} + func ValidatePoRep(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, ActorError) { _, span := trace.StartSpan(ctx, "ValidatePoRep") defer span.End() @@ -787,34 +805,28 @@ type DeclareFaultsParams struct { } func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) { - /* - oldstate, self, aerr := loadState(vmctx) - if aerr != nil { - return nil, aerr - } + oldstate, self, aerr := loadState(vmctx) + if aerr != nil { + return nil, aerr + } - challengeHeight := self.ProvingPeriodEnd - build.PoStChallangeTime + nfaults, err := types.MergeBitFields(params.Faults, self.FaultSet) + if err != nil { + return nil, aerrors.Absorb(err, 1, "failed to merge bitfields") + } - if vmctx.BlockHeight() < challengeHeight { - // TODO: optimized bitfield methods - for _, v := range params.Faults.All() { - self.CurrentFaultSet.Set(v) - } - } else { - for _, v := range params.Faults.All() { - self.NextFaultSet.Set(v) - } - } + self.FaultSet = nfaults - nstate, err := vmctx.Storage().Put(self) - if err != nil { - return nil, err - } - if err := vmctx.Storage().Commit(oldstate, nstate); err != nil { - return nil, err - } + self.LastFaultSubmission = vmctx.BlockHeight() + + nstate, aerr := vmctx.Storage().Put(self) + if err != nil { + return nil, aerr + } + if err := vmctx.Storage().Commit(oldstate, nstate); err != nil { + return nil, err + } - */ return nil, nil } @@ -906,10 +918,12 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerro return aerrors.HandleExternalError(nerr, "failed to load proving set") } - self.CurrentFaultSet = self.NextFaultSet - self.NextFaultSet = types.NewBitField() + faults, nerr := self.FaultSet.All() + if nerr != nil { + return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)") + } - faults := []uint64{} // TODO + self.FaultSet = types.NewBitField() oldPower := self.Power self.Power = types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), @@ -941,7 +955,13 @@ func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerro return err } - self.ProvingSet = self.Sectors + ncid, err := RemoveFromSectorSet(vmctx.Context(), vmctx.Storage(), self.Sectors, faults) + if err != nil { + return err + } + + self.Sectors = ncid + self.ProvingSet = ncid self.ElectionPeriodStart = vmctx.BlockHeight() return nil } diff --git a/chain/actors/actor_miner_test.go b/chain/actors/actor_miner_test.go new file mode 100644 index 000000000..28a5ce046 --- /dev/null +++ b/chain/actors/actor_miner_test.go @@ -0,0 +1,157 @@ +package actors_test + +import ( + "bytes" + "context" + "math/rand" + "testing" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/actors/aerrors" + "github.com/filecoin-project/lotus/chain/address" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/sectorbuilder" + hamt "github.com/ipfs/go-hamt-ipld" + blockstore "github.com/ipfs/go-ipfs-blockstore" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func TestMinerCommitSectors(t *testing.T) { + var worker, client address.Address + var minerAddr address.Address + opts := []HarnessOpt{ + HarnessAddr(&worker, 1000000), + HarnessAddr(&client, 1000000), + HarnessActor(&minerAddr, &worker, actors.StorageMinerCodeCid, + func() cbg.CBORMarshaler { + return &actors.StorageMinerConstructorParams{ + Owner: worker, + Worker: worker, + SectorSize: 1024, + PeerID: "fakepeerid", + } + }), + } + + h := NewHarness(t, opts...) + h.vm.Syscalls.ValidatePoRep = func(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, aerrors.ActorError) { + // all proofs are valid + return true, nil + } + + ret, _ := h.SendFunds(t, worker, minerAddr, types.NewInt(100000)) + ApplyOK(t, ret) + + ret, _ = h.InvokeWithValue(t, client, actors.StorageMarketAddress, actors.SMAMethods.AddBalance, types.NewInt(2000), nil) + ApplyOK(t, ret) + + addSectorToMiner(h, t, minerAddr, worker, client, 1) + + assertSectorIDs(h, t, minerAddr, []uint64{1}) +} + +func addSectorToMiner(h *Harness, t *testing.T, minerAddr, worker, client address.Address, sid uint64) { + t.Helper() + s := sectorbuilder.UserBytesForSectorSize(1024) + deal := h.makeFakeDeal(t, minerAddr, worker, client, s) + ret, _ := h.Invoke(t, worker, actors.StorageMarketAddress, actors.SMAMethods.PublishStorageDeals, + &actors.PublishStorageDealsParams{ + Deals: []actors.StorageDealProposal{*deal}, + }) + ApplyOK(t, ret) + var dealIds actors.PublishStorageDealResponse + if err := dealIds.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil { + t.Fatal(err) + } + + dealid := dealIds.DealIDs[0] + + ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.PreCommitSector, + &actors.SectorPreCommitInfo{ + SectorNumber: sid, + CommR: []byte("cats"), + SealEpoch: 10, + DealIDs: []uint64{dealid}, + }) + ApplyOK(t, ret) + + h.BlockHeight += 100 + ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.ProveCommitSector, + &actors.SectorProveCommitInfo{ + Proof: []byte("prooofy"), + SectorID: sid, + DealIDs: []uint64{dealid}, // TODO: weird that i have to pass this again + }) + ApplyOK(t, ret) +} + +func assertSectorIDs(h *Harness, t *testing.T, maddr address.Address, ids []uint64) { + t.Helper() + sectors, err := getMinerSectorSet(context.TODO(), h.vm.StateTree(), h.bs, maddr) + if err != nil { + t.Fatal(err) + } + + if len(sectors) != len(ids) { + t.Fatal("miner has wrong number of sectors in their sector set") + } + + all := make(map[uint64]bool) + for _, s := range sectors { + all[s.SectorID] = true + } + + for _, id := range ids { + if !all[id] { + t.Fatal("expected to find sector ID: ", id) + } + } +} + +func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) { + mact, err := st.GetActor(maddr) + if err != nil { + return nil, err + } + + cst := hamt.CSTFromBstore(bs) + + var mstate actors.StorageMinerActorState + if err := cst.Get(ctx, mact.Head, &mstate); err != nil { + return nil, err + } + + return stmgr.LoadSectorsFromSet(ctx, bs, mstate.Sectors) +} + +func (h *Harness) makeFakeDeal(t *testing.T, miner, worker, client address.Address, size uint64) *actors.StorageDealProposal { + data := make([]byte, size) + rand.Read(data) + commP, err := sectorbuilder.GeneratePieceCommitment(bytes.NewReader(data), size) + if err != nil { + t.Fatal(err) + } + + prop := actors.StorageDealProposal{ + PieceRef: commP[:], + PieceSize: size, + //PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid + + Client: client, + Provider: miner, + + ProposalExpiration: 10000, + Duration: 150, + + StoragePricePerEpoch: types.NewInt(1), + StorageCollateral: types.NewInt(0), + } + + if err := api.SignWith(context.TODO(), h.w.Sign, client, &prop); err != nil { + t.Fatal(err) + } + + return &prop +} diff --git a/chain/actors/actor_paych_test.go b/chain/actors/actor_paych_test.go index e23acdf4a..55de306c0 100644 --- a/chain/actors/actor_paych_test.go +++ b/chain/actors/actor_paych_test.go @@ -83,7 +83,7 @@ func TestPaychUpdate(t *testing.T) { ApplyOK(t, ret) // now we have to 'wait' for the chain to advance. - h.vm.SetBlockHeight(1000) + h.BlockHeight = 1000 ret, _ = h.Invoke(t, targetAddr, pch, actors.PCAMethods.Collect, nil) ApplyOK(t, ret) diff --git a/chain/actors/cbor_gen.go b/chain/actors/cbor_gen.go index 475add9da..bdace2e38 100644 --- a/chain/actors/cbor_gen.go +++ b/chain/actors/cbor_gen.go @@ -248,13 +248,13 @@ func (t *StorageMinerActorState) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.Info: %w", err) } - // t.t.CurrentFaultSet (types.BitField) (struct) - if err := t.CurrentFaultSet.MarshalCBOR(w); err != nil { + // t.t.FaultSet (types.BitField) (struct) + if err := t.FaultSet.MarshalCBOR(w); err != nil { return err } - // t.t.NextFaultSet (types.BitField) (struct) - if err := t.NextFaultSet.MarshalCBOR(w); err != nil { + // t.t.LastFaultSubmission (uint64) (uint64) + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.LastFaultSubmission))); err != nil { return err } @@ -384,24 +384,25 @@ func (t *StorageMinerActorState) UnmarshalCBOR(r io.Reader) error { t.Info = c } - // t.t.CurrentFaultSet (types.BitField) (struct) + // t.t.FaultSet (types.BitField) (struct) { - if err := t.CurrentFaultSet.UnmarshalCBOR(br); err != nil { + if err := t.FaultSet.UnmarshalCBOR(br); err != nil { return err } } - // t.t.NextFaultSet (types.BitField) (struct) - - { - - if err := t.NextFaultSet.UnmarshalCBOR(br); err != nil { - return err - } + // t.t.LastFaultSubmission (uint64) (uint64) + maj, extra, err = cbg.CborReadHeader(br) + if err != nil { + return err } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.LastFaultSubmission = uint64(extra) // t.t.Power (types.BigInt) (struct) { @@ -1031,6 +1032,49 @@ func (t *UpdatePeerIDParams) UnmarshalCBOR(r io.Reader) error { return nil } +func (t *DeclareFaultsParams) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{129}); err != nil { + return err + } + + // t.t.Faults (types.BitField) (struct) + if err := t.Faults.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *DeclareFaultsParams) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 1 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.t.Faults (types.BitField) (struct) + + { + + if err := t.Faults.UnmarshalCBOR(br); err != nil { + return err + } + + } + return nil +} + func (t *MultiSigActorState) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) diff --git a/chain/actors/harness2_test.go b/chain/actors/harness2_test.go index 0ffc02b1b..6f1da8d58 100644 --- a/chain/actors/harness2_test.go +++ b/chain/actors/harness2_test.go @@ -3,6 +3,7 @@ package actors_test import ( "bytes" "context" + "math/rand" "testing" "github.com/ipfs/go-cid" @@ -41,10 +42,12 @@ const ( type HarnessOpt func(testing.TB, *Harness) error type Harness struct { - HI HarnessInit - Stage HarnessStage - Nonces map[address.Address]uint64 - GasCharges map[address.Address]types.BigInt + HI HarnessInit + Stage HarnessStage + Nonces map[address.Address]uint64 + GasCharges map[address.Address]types.BigInt + Rand vm.Rand + BlockHeight uint64 lastBalanceCheck map[address.Address]types.BigInt @@ -127,6 +130,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness { h := &Harness{ Stage: HarnessPreInit, Nonces: make(map[address.Address]uint64), + Rand: &fakeRand{}, HI: HarnessInit{ NAddrs: 1, Miner: blsaddr(0), @@ -140,6 +144,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness { w: w, ctx: context.Background(), bs: bstore.NewBlockstore(dstore.NewMapDatastore()), + BlockHeight: 0, } for _, opt := range options { err := opt(t, h) @@ -157,8 +162,14 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness { if err != nil { t.Fatal(err) } + + stateroot, err = gen.SetupStorageMarketActor(h.bs, stateroot, nil) + if err != nil { + t.Fatal(err) + } + h.cs = store.NewChainStore(h.bs, nil) - h.vm, err = vm.NewVM(stateroot, 1, nil, h.HI.Miner, h.cs.Blockstore()) + h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore()) if err != nil { t.Fatal(err) } @@ -246,6 +257,7 @@ func (h *Harness) Invoke(t testing.TB, from address.Address, to address.Address, func (h *Harness) InvokeWithValue(t testing.TB, from address.Address, to address.Address, method uint64, value types.BigInt, params cbg.CBORMarshaler) (*vm.ApplyRet, *state.StateTree) { t.Helper() + h.vm.SetBlockHeight(h.BlockHeight) return h.Apply(t, types.Message{ To: to, From: from, @@ -315,3 +327,11 @@ func DumpObject(t testing.TB, obj cbg.CBORMarshaler) []byte { } return b.Bytes() } + +type fakeRand struct{} + +func (fr *fakeRand) GetRandomness(ctx context.Context, h int64) ([]byte, error) { + out := make([]byte, 32) + rand.New(rand.NewSource(h)).Read(out) + return out, nil +} diff --git a/chain/types/bitfield.go b/chain/types/bitfield.go index b4687f5e8..20e63dd95 100644 --- a/chain/types/bitfield.go +++ b/chain/types/bitfield.go @@ -34,6 +34,38 @@ func BitFieldFromSet(setBits []uint64) BitField { return res } +func MergeBitFields(a, b BitField) (BitField, error) { + ra, err := a.rle.RunIterator() + if err != nil { + return BitField{}, err + } + + rb, err := b.rle.RunIterator() + if err != nil { + return BitField{}, err + } + + merge, err := rlepluslazy.Sum(ra, rb) + if err != nil { + return BitField{}, err + } + + mergebytes, err := rlepluslazy.EncodeRuns(merge, nil) + if err != nil { + return BitField{}, err + } + + rle, err := rlepluslazy.FromBuf(mergebytes) + if err != nil { + return BitField{}, err + } + + return BitField{ + rle: rle, + bits: make(map[uint64]struct{}), + }, nil +} + func (bf BitField) sum() (rlepluslazy.RunIterator, error) { if len(bf.bits) == 0 { return bf.rle.RunIterator() @@ -86,7 +118,26 @@ func (bf BitField) All() ([]uint64, error) { return nil, err } - return res, err + return res, nil +} + +func (bf BitField) AllMap() (map[uint64]bool, error) { + + runs, err := bf.sum() + if err != nil { + return nil, err + } + + res, err := rlepluslazy.SliceFromRuns(runs) + if err != nil { + return nil, err + } + + out := make(map[uint64]bool) + for _, i := range res { + out[i] = true + } + return out, nil } func (bf BitField) MarshalCBOR(w io.Writer) error { diff --git a/chain/types/vmcontext.go b/chain/types/vmcontext.go index 5e2963483..39667d2a4 100644 --- a/chain/types/vmcontext.go +++ b/chain/types/vmcontext.go @@ -40,10 +40,15 @@ type VMContext interface { ChargeGas(uint64) aerrors.ActorError GetRandomness(height uint64) ([]byte, aerrors.ActorError) GetBalance(address.Address) (BigInt, aerrors.ActorError) + Sys() *VMSyscalls Context() context.Context } +type VMSyscalls struct { + ValidatePoRep func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError) +} + type storageWrapper struct { s Storage } diff --git a/chain/vm/mkactor.go b/chain/vm/mkactor.go index 1e1885825..3419f9f3c 100644 --- a/chain/vm/mkactor.go +++ b/chain/vm/mkactor.go @@ -48,7 +48,7 @@ func makeActor(st *state.StateTree, addr address.Address) (*types.Actor, aerrors case address.SECP256K1: return NewSecp256k1AccountActor(st, addr) case address.ID: - return nil, aerrors.New(1, "no actor with given ID") + return nil, aerrors.Newf(1, "no actor with given ID: %s", addr) case address.Actor: return nil, aerrors.Newf(1, "no such actor: %s", addr) default: diff --git a/chain/vm/syscalls.go b/chain/vm/syscalls.go new file mode 100644 index 000000000..92f9ea269 --- /dev/null +++ b/chain/vm/syscalls.go @@ -0,0 +1,14 @@ +package vm + +import ( + "github.com/filecoin-project/lotus/chain/actors" + "github.com/filecoin-project/lotus/chain/types" +) + +// Actual type is defined in chain/types/vmcontext.go because the VMContext interface is there + +func DefaultSyscalls() *types.VMSyscalls { + return &types.VMSyscalls{ + ValidatePoRep: actors.ValidatePoRep, + } +} diff --git a/chain/vm/vm.go b/chain/vm/vm.go index 1eed0f895..7c1c5a802 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -54,6 +54,8 @@ type VMContext struct { gasAvailable types.BigInt gasUsed types.BigInt + sys *types.VMSyscalls + // root cid of the state of the actor this invocation will be on sroot cid.Cid @@ -75,6 +77,10 @@ func (vmc *VMContext) GetRandomness(height uint64) ([]byte, aerrors.ActorError) return res, nil } +func (vmc *VMContext) Sys() *types.VMSyscalls { + return vmc.sys +} + // Storage interface func (vmc *VMContext) Put(i cbg.CBORMarshaler) (cid.Cid, aerrors.ActorError) { @@ -284,6 +290,7 @@ func (vm *VM) makeVMContext(ctx context.Context, sroot cid.Cid, msg *types.Messa msg: msg, origin: origin, height: vm.blockHeight, + sys: vm.Syscalls, gasUsed: usedGas, gasAvailable: msg.GasLimit, @@ -304,6 +311,8 @@ type VM struct { blockMiner address.Address inv *invoker rand Rand + + Syscalls *types.VMSyscalls } func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore) (*VM, error) { @@ -323,6 +332,7 @@ func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs block blockMiner: maddr, inv: newInvoker(), rand: r, + Syscalls: DefaultSyscalls(), }, nil } diff --git a/gen/main.go b/gen/main.go index 9356f5a8b..35bee00b4 100644 --- a/gen/main.go +++ b/gen/main.go @@ -96,6 +96,7 @@ func main() { actors.SubmitFallbackPoStParams{}, actors.PaymentVerifyParams{}, actors.UpdatePeerIDParams{}, + actors.DeclareFaultsParams{}, actors.MultiSigActorState{}, actors.MultiSigConstructorParams{}, actors.MultiSigProposeParams{}, diff --git a/storage/cbor_gen.go b/storage/cbor_gen.go index d7a84430c..340479941 100644 --- a/storage/cbor_gen.go +++ b/storage/cbor_gen.go @@ -342,6 +342,18 @@ func (t *SectorInfo) MarshalCBOR(w io.Writer) error { } } + // t.t.FaultReportMsg (cid.Cid) (struct) + + if t.FaultReportMsg == nil { + if _, err := w.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(w, *t.FaultReportMsg); err != nil { + return xerrors.Errorf("failed to write cid field t.FaultReportMsg: %w", err) + } + } + // t.t.LastErr (string) (string) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.LastErr)))); err != nil { return err @@ -574,6 +586,30 @@ func (t *SectorInfo) UnmarshalCBOR(r io.Reader) error { t.CommitMessage = &c } + } + // t.t.FaultReportMsg (cid.Cid) (struct) + + { + + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.FaultReportMsg: %w", err) + } + + t.FaultReportMsg = &c + } + } // t.t.LastErr (string) (string) diff --git a/storage/sector_states.go b/storage/sector_states.go index 00c070a04..0addd1376 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -233,3 +233,57 @@ func (m *Miner) handleCommitWait(ctx context.Context, sector SectorInfo) *sector return sector.upd().to(api.Proving).state(func(info *SectorInfo) { }) } + +func (m *Miner) handleFaulty(ctx context.Context, sector SectorInfo) *sectorUpdate { + // TODO: check if the fault has already been reported, and that this sector is even valid + + // TODO: coalesce faulty sector reporting + bf := types.NewBitField() + bf.Set(sector.SectorID) + + fp := &actors.DeclareFaultsParams{bf} + _ = fp + enc, aerr := actors.SerializeParams(nil) + if aerr != nil { + return sector.upd().to(api.FailedUnrecoverable).error(xerrors.Errorf("failed to serialize declare fault params: %w", aerr)) + } + + msg := &types.Message{ + To: m.maddr, + From: m.worker, + Method: actors.MAMethods.DeclareFaults, + Params: enc, + Value: types.NewInt(0), // TODO: need to ensure sufficient collateral + GasLimit: types.NewInt(1000000 /* i dont know help */), + GasPrice: types.NewInt(1), + } + + smsg, err := m.api.MpoolPushMessage(ctx, msg) + if err != nil { + return sector.upd().to(api.FailedUnrecoverable).error(xerrors.Errorf("failed to push declare faults message to network: %w", err)) + } + + return sector.upd().to(api.FaultReported).state(func(info *SectorInfo) { + c := smsg.Cid() + info.FaultReportMsg = &c + }) +} + +func (m *Miner) handleFaultReported(ctx context.Context, sector SectorInfo) *sectorUpdate { + if sector.FaultReportMsg == nil { + return sector.upd().to(api.FailedUnrecoverable).error(xerrors.Errorf("entered fault reported state without a FaultReportMsg cid")) + } + + mw, err := m.api.StateWaitMsg(ctx, *sector.FaultReportMsg) + if err != nil { + return sector.upd().to(api.CommitFailed).error(xerrors.Errorf("failed to wait for fault declaration: %w", err)) + } + + if mw.Receipt.ExitCode != 0 { + log.Errorf("UNHANDLED: declaring sector fault failed (exit=%d, msg=%s) (id: %d)", mw.Receipt.ExitCode, *sector.FaultReportMsg, sector.SectorID) + return sector.upd().fatal(xerrors.Errorf("UNHANDLED: submitting fault declaration failed (exit %d)", mw.Receipt.ExitCode)) + } + + return sector.upd().to(api.FaultedFinal).state(func(info *SectorInfo) {}) + +} diff --git a/storage/sector_types.go b/storage/sector_types.go index dccc885dc..11a3120a7 100644 --- a/storage/sector_types.go +++ b/storage/sector_types.go @@ -71,6 +71,9 @@ type SectorInfo struct { // Committing CommitMessage *cid.Cid + // Faults + FaultReportMsg *cid.Cid + // Debug LastErr string } diff --git a/storage/sectors.go b/storage/sectors.go index 251c10331..973869418 100644 --- a/storage/sectors.go +++ b/storage/sectors.go @@ -255,6 +255,12 @@ func (m *Miner) onSectorUpdated(ctx context.Context, update sectorUpdate) { case api.CommitFailed: log.Warn("sector %d entered unimplemented state 'CommitFailed'", update.id) + // Faults + case api.Faulty: + m.handleSectorUpdate(ctx, sector, m.handleFaulty) + case api.FaultReported: + m.handleSectorUpdate(ctx, sector, m.handleFaultReported) + // Fatal errors case api.UndefinedSectorState: log.Error("sector update with undefined state!") From 2f2a5824b327c7425eea9f63795dd3ca5f8b42e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 9 Dec 2019 21:19:46 +0100 Subject: [PATCH 14/14] storage: Use fatal instead of .to.error --- storage/sector_states.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/sector_states.go b/storage/sector_states.go index 0addd1376..ffad0119e 100644 --- a/storage/sector_states.go +++ b/storage/sector_states.go @@ -245,7 +245,7 @@ func (m *Miner) handleFaulty(ctx context.Context, sector SectorInfo) *sectorUpda _ = fp enc, aerr := actors.SerializeParams(nil) if aerr != nil { - return sector.upd().to(api.FailedUnrecoverable).error(xerrors.Errorf("failed to serialize declare fault params: %w", aerr)) + return sector.upd().fatal(xerrors.Errorf("failed to serialize declare fault params: %w", aerr)) } msg := &types.Message{