From 60a031a7134b673f4ebb31f61e3536969bc54634 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 14 Sep 2020 00:25:54 -0400 Subject: [PATCH 01/54] Lower devnet block time --- build/params_2k.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/params_2k.go b/build/params_2k.go index 0ef1d9b34..9913374e1 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -29,7 +29,7 @@ func init() { BuildType |= Build2k } -const BlockDelaySecs = uint64(30) +const BlockDelaySecs = uint64(4) const PropagationDelaySecs = uint64(1) From d1850ea27d1488990f2395656a4154eabe0c9608 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Sun, 13 Sep 2020 01:26:44 -0400 Subject: [PATCH 02/54] Add an API to get detailed gas costs for a message --- api/api_full.go | 13 ++++++++ api/apistruct/struct.go | 5 +++ chain/stmgr/stmgr.go | 4 +-- chain/vm/vm.go | 35 +++++++++++---------- cli/state.go | 55 +++++++++++++++++++++++++++++++++ documentation/en/api-methods.md | 35 +++++++++++++++++++++ node/impl/full/state.go | 40 ++++++++++++++++++++++++ 7 files changed, 169 insertions(+), 18 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 23226443a..588e567a8 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -354,6 +354,8 @@ type FullNode interface { StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*SectorLocation, error) // StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) + // StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip + StateMsgGasCost(context.Context, cid.Cid, types.TipSetKey) (*MsgGasCost, error) // StateWaitMsg looks back in the chain for a message. If not found, it blocks until the // message arrives on chain, and gets to the indicated confidence depth. StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) @@ -531,6 +533,17 @@ type MsgLookup struct { Height abi.ChainEpoch } +type MsgGasCost struct { + Message cid.Cid // Can be different than requested, in case it was replaced, but only gas values changed + GasUsed abi.TokenAmount + BaseFeeBurn abi.TokenAmount + OverEstimationBurn abi.TokenAmount + MinerPenalty abi.TokenAmount + MinerTip abi.TokenAmount + Refund abi.TokenAmount + TotalCost abi.TokenAmount +} + type BlockMessages struct { BlsMessages []*types.Message SecpkMessages []*types.SignedMessage diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index fdb9843ff..9f2b7bc4b 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -183,6 +183,7 @@ type FullNodeStruct struct { StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` + StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` @@ -816,6 +817,10 @@ func (c *FullNodeStruct) StateReadState(ctx context.Context, addr address.Addres return c.Internal.StateReadState(ctx, addr, tsk) } +func (c *FullNodeStruct) StateMsgGasCost(ctx context.Context, msgc cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { + return c.Internal.StateMsgGasCost(ctx, msgc, tsk) +} + func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) { return c.Internal.StateWaitMsg(ctx, msgc, confidence) } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index e6103e2b3..a7f94e6ee 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -233,8 +233,8 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } receipts = append(receipts, &r.MessageReceipt) - gasReward = big.Add(gasReward, r.MinerTip) - penalty = big.Add(penalty, r.Penalty) + gasReward = big.Add(gasReward, r.GasCosts.MinerTip) + penalty = big.Add(penalty, r.GasCosts.MinerPenalty) if cb != nil { if err := cb(cm.Cid(), m, r); err != nil { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index e389a3531..eb1ef70ce 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -201,10 +201,9 @@ type Rand interface { type ApplyRet struct { types.MessageReceipt ActorErr aerrors.ActorError - Penalty types.BigInt - MinerTip types.BigInt ExecutionTrace types.ExecutionTrace Duration time.Duration + GasCosts GasOutputs } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, @@ -328,8 +327,7 @@ func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*Ap }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - Penalty: types.NewInt(0), - MinerTip: types.NewInt(0), + GasCosts: GasOutputs{}, Duration: time.Since(start), }, actorErr } @@ -362,9 +360,10 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ExitCode: exitcode.SysErrOutOfGas, GasUsed: 0, }, - Penalty: types.BigMul(vm.baseFee, abi.NewTokenAmount(msgGasCost)), + GasCosts: GasOutputs{ + MinerPenalty: types.BigMul(vm.baseFee, abi.NewTokenAmount(msgGasCost)), + }, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } @@ -381,9 +380,10 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "actor not found: %s", msg.From), - Penalty: minerPenaltyAmount, + GasCosts: GasOutputs{ + MinerPenalty: minerPenaltyAmount, + }, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } return nil, xerrors.Errorf("failed to look up from actor: %w", err) @@ -397,9 +397,10 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "send from not account actor: %s", fromActor.Code), - Penalty: minerPenaltyAmount, + GasCosts: GasOutputs{ + MinerPenalty: minerPenaltyAmount, + }, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } @@ -411,9 +412,11 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor nonce invalid: msg:%d != state:%d", msg.Nonce, fromActor.Nonce), - Penalty: minerPenaltyAmount, + + GasCosts: GasOutputs{ + MinerPenalty: minerPenaltyAmount, + }, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } @@ -426,9 +429,10 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor balance less than needed: %s < %s", types.FIL(fromActor.Balance), types.FIL(gascost)), - Penalty: minerPenaltyAmount, + GasCosts: GasOutputs{ + MinerPenalty: minerPenaltyAmount, + }, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } @@ -521,8 +525,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - Penalty: gasOutputs.MinerPenalty, - MinerTip: gasOutputs.MinerTip, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } diff --git a/cli/state.go b/cli/state.go index 1036e8fe5..7729e729a 100644 --- a/cli/state.go +++ b/cli/state.go @@ -65,6 +65,7 @@ var stateCmd = &cli.Command{ stateGetDealSetCmd, stateWaitMsgCmd, stateSearchMsgCmd, + stateMsgCostCmd, stateMinerInfo, stateMarketCmd, }, @@ -1304,6 +1305,60 @@ var stateSearchMsgCmd = &cli.Command{ }, } +var stateMsgCostCmd = &cli.Command{ + Name: "msg-cost", + Usage: "Get the detailed gas costs of a message", + ArgsUsage: "[messageCid]", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return fmt.Errorf("must specify message cid to get gas costs for") + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + msg, err := cid.Decode(cctx.Args().First()) + if err != nil { + return err + } + + tsk := types.EmptyTSK + + ts, err := LoadTipSet(ctx, cctx, api) + if err != nil { + return err + } + + if ts != nil { + tsk = ts.Key() + } + + mgc, err := api.StateMsgGasCost(ctx, msg, tsk) + if err != nil { + return err + } + + if mgc != nil { + fmt.Printf("\nMessage CID: %d", mgc.Message) + fmt.Printf("\nGas Used: %d", mgc.GasUsed) + fmt.Printf("\nBase Fee Burn: %d", mgc.BaseFeeBurn) + fmt.Printf("\nOverestimation Burn: %d", mgc.OverEstimationBurn) + fmt.Printf("\nMiner Tip: %d", mgc.MinerTip) + fmt.Printf("\nRefundd: %d", mgc.Refund) + fmt.Printf("\nTotal Cost: %d", mgc.TotalCost) + fmt.Printf("\nMiner Penalty: %d", mgc.MinerPenalty) + } else { + fmt.Print("message was not found on chain") + } + return nil + }, +} + var stateCallCmd = &cli.Command{ Name: "call", Usage: "Invoke a method on an actor locally", diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 27875eca1..ec298d635 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -148,6 +148,7 @@ * [StateMinerRecoveries](#StateMinerRecoveries) * [StateMinerSectorCount](#StateMinerSectorCount) * [StateMinerSectors](#StateMinerSectors) + * [StateMsgGasCost](#StateMsgGasCost) * [StateNetworkName](#StateNetworkName) * [StateReadState](#StateReadState) * [StateReplay](#StateReplay) @@ -3731,6 +3732,40 @@ Inputs: Response: `null` +### StateMsgGasCost +StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "Receipt": { + "ExitCode": 0, + "Return": "Ynl0ZSBhcnJheQ==", + "GasUsed": 9 + }, + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0" +} +``` + ### StateNetworkName StateNetworkName returns the name of the network the node is synced to diff --git a/node/impl/full/state.go b/node/impl/full/state.go index e183fafa4..1fb078001 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1226,3 +1226,43 @@ func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetK return a.StateManager.GetCirculatingSupplyDetailed(ctx, ts.Height(), sTree) } + +func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { + var msg cid.Cid + var ts *types.TipSet + var err error + if tsk != types.EmptyTSK { + msg = inputMsg + ts, err = a.Chain.LoadTipSet(tsk) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) + } + } else { + msg, err := a.StateSearchMsg(ctx, inputMsg) + if err != nil { + return nil, xerrors.Errorf("searching for msg %s: %w", inputMsg, err) + } + + ts, err = a.Chain.GetTipSetFromKey(msg.TipSet) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", msg.TipSet, err) + } + } + + m, r, err := a.StateManager.Replay(ctx, ts, msg) + if err != nil { + return nil, err + } + + gasSpent := big.Sub(big.NewInt(m.GasLimit), r.GasCosts.Refund) + return &api.MsgGasCost{ + Message: msg, + GasUsed: big.NewInt(r.GasUsed), + BaseFeeBurn: r.GasCosts.BaseFeeBurn, + OverEstimationBurn: r.GasCosts.OverEstimationBurn, + MinerPenalty: r.GasCosts.MinerPenalty, + MinerTip: r.GasCosts.MinerTip, + Refund: r.GasCosts.Refund, + TotalCost: big.Mul(gasSpent, m.GasFeeCap), + }, nil +} From 0edcae9e046346217c482b7d1c093d5a6dc77ac8 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Mon, 14 Sep 2020 01:23:29 -0400 Subject: [PATCH 03/54] Fixup new command --- cli/state.go | 4 ++-- documentation/en/api-methods.md | 19 ++++++++++++------- extern/storage-sealing/upgrade_queue.go | 2 ++ node/impl/full/state.go | 23 ++++++++++++++++------- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/cli/state.go b/cli/state.go index 7729e729a..62d751204 100644 --- a/cli/state.go +++ b/cli/state.go @@ -1344,12 +1344,12 @@ var stateMsgCostCmd = &cli.Command{ } if mgc != nil { - fmt.Printf("\nMessage CID: %d", mgc.Message) + fmt.Printf("Message CID: %s", mgc.Message) fmt.Printf("\nGas Used: %d", mgc.GasUsed) fmt.Printf("\nBase Fee Burn: %d", mgc.BaseFeeBurn) fmt.Printf("\nOverestimation Burn: %d", mgc.OverEstimationBurn) fmt.Printf("\nMiner Tip: %d", mgc.MinerTip) - fmt.Printf("\nRefundd: %d", mgc.Refund) + fmt.Printf("\nRefund: %d", mgc.Refund) fmt.Printf("\nTotal Cost: %d", mgc.TotalCost) fmt.Printf("\nMiner Penalty: %d", mgc.MinerPenalty) } else { diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index ec298d635..5bba832df 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -3743,7 +3743,15 @@ Inputs: [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] ] ``` @@ -3753,16 +3761,13 @@ Response: "Message": { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" }, - "Receipt": { - "ExitCode": 0, - "Return": "Ynl0ZSBhcnJheQ==", - "GasUsed": 9 - }, + "GasUsed": "0", "BaseFeeBurn": "0", "OverEstimationBurn": "0", "MinerPenalty": "0", "MinerTip": "0", - "Refund": "0" + "Refund": "0", + "TotalCost": "0" } ``` diff --git a/extern/storage-sealing/upgrade_queue.go b/extern/storage-sealing/upgrade_queue.go index 650fdc83d..af2d1827d 100644 --- a/extern/storage-sealing/upgrade_queue.go +++ b/extern/storage-sealing/upgrade_queue.go @@ -67,6 +67,8 @@ func (m *Sealing) tryUpgradeSector(ctx context.Context, params *miner.SectorPreC params.ReplaceSectorDeadline = loc.Deadline params.ReplaceSectorPartition = loc.Partition + log.Infof("replacing sector %d with %d", *replace, params.SectorNumber) + ri, err := m.api.StateSectorGetInfo(ctx, m.maddr, *replace, nil) if err != nil { log.Errorf("error calling StateSectorGetInfo for replaced sector: %+v", err) diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 1fb078001..c37ab9c3d 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1238,15 +1238,25 @@ func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk ty return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } } else { - msg, err := a.StateSearchMsg(ctx, inputMsg) + mlkp, err := a.StateSearchMsg(ctx, inputMsg) if err != nil { return nil, xerrors.Errorf("searching for msg %s: %w", inputMsg, err) } - - ts, err = a.Chain.GetTipSetFromKey(msg.TipSet) - if err != nil { - return nil, xerrors.Errorf("loading tipset %s: %w", msg.TipSet, err) + if mlkp == nil { + return nil, xerrors.Errorf("didn't find msg %s", inputMsg) } + + executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) + } + + ts, err = a.Chain.LoadTipSet(executionTs.Parents()) + if err != nil { + return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) + } + + msg = mlkp.Message } m, r, err := a.StateManager.Replay(ctx, ts, msg) @@ -1254,7 +1264,6 @@ func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk ty return nil, err } - gasSpent := big.Sub(big.NewInt(m.GasLimit), r.GasCosts.Refund) return &api.MsgGasCost{ Message: msg, GasUsed: big.NewInt(r.GasUsed), @@ -1263,6 +1272,6 @@ func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk ty MinerPenalty: r.GasCosts.MinerPenalty, MinerTip: r.GasCosts.MinerTip, Refund: r.GasCosts.Refund, - TotalCost: big.Mul(gasSpent, m.GasFeeCap), + TotalCost: big.Sub(m.RequiredFunds(), r.GasCosts.Refund), }, nil } From ae326ce65de2ad3973ac625226d80dac0f1ccf8f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 15:43:37 +0100 Subject: [PATCH 04/54] feat: add inspect runtime method to chaos actor --- conformance/chaos/actor.go | 32 ++++++ conformance/chaos/actor_test.go | 26 +++++ conformance/chaos/cbor_gen.go | 198 +++++++++++++++++++++++++++++++- conformance/chaos/gen/gen.go | 1 + 4 files changed, 254 insertions(+), 3 deletions(-) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index daff26694..526dac64c 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -64,6 +64,9 @@ const ( // MethodAbortWith is the identifier for the method that panics optionally with // a passed exit code. MethodAbortWith + // MethodInspectRuntime is the identifier for the method that returns the + // current runtime values. + MethodInspectRuntime ) // Exports defines the methods this actor exposes publicly. @@ -77,6 +80,7 @@ func (a Actor) Exports() []interface{} { MethodSend: a.Send, MethodMutateState: a.MutateState, MethodAbortWith: a.AbortWith, + MethodInspectRuntime: a.InspectRuntime, } } @@ -247,3 +251,31 @@ func (a Actor) AbortWith(rt runtime.Runtime, args *AbortWithArgs) *abi.EmptyValu } return nil } + +// InspectRuntimeReturn is the return value for the Actor.InspectRuntime method. +type InspectRuntimeReturn struct { + NetworkVersion int64 + Caller address.Address + Receiver address.Address + ValueReceived abi.TokenAmount + CurrEpoch abi.ChainEpoch + CurrentBalance abi.TokenAmount + State State + TotalFilCircSupply abi.TokenAmount +} + +// InspectRuntime returns a copy of the serializable values available in the Runtime. +func (a Actor) InspectRuntime(rt runtime.Runtime, _ *abi.EmptyValue) *InspectRuntimeReturn { + rt.ValidateImmediateCallerAcceptAny() + var st State + rt.StateReadonly(&st) + return &InspectRuntimeReturn{ + NetworkVersion: int64(rt.NetworkVersion()), + Caller: rt.Caller(), + Receiver: rt.Receiver(), + ValueReceived: rt.ValueReceived(), + CurrentBalance: rt.CurrentBalance(), + State: st, + TotalFilCircSupply: rt.TotalFilCircSupply(), + } +} diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index 67ced2899..2e899980d 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -6,6 +6,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/support/mock" atesting "github.com/filecoin-project/specs-actors/support/testing" ) @@ -151,3 +152,28 @@ func TestAbortWithUncontrolled(t *testing.T) { }) rt.Verify() } + +func TestInspectRuntime(t *testing.T) { + caller := atesting.NewIDAddr(t, 100) + receiver := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + rt.SetCaller(caller, builtin.AccountActorCodeID) + rt.StateCreate(&State{}) + var a Actor + + rt.ExpectValidateCallerAny() + ret := rt.Call(a.InspectRuntime, abi.Empty) + rtr, ok := ret.(*InspectRuntimeReturn) + if !ok { + t.Fatal("invalid return value") + } + if rtr.Caller != caller { + t.Fatal("unexpected runtime caller") + } + if rtr.Receiver != receiver { + t.Fatal("unexpected runtime receiver") + } + rt.Verify() +} diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 710b84b93..96b707386 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -6,10 +6,10 @@ import ( "fmt" "io" - abi "github.com/filecoin-project/go-state-types/abi" - exitcode "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/exitcode" cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" + "golang.org/x/xerrors" ) var _ = xerrors.Errorf @@ -730,3 +730,195 @@ func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error { } return nil } + +var lengthBufInspectRuntimeReturn = []byte{136} + +func (t *InspectRuntimeReturn) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufInspectRuntimeReturn); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.NetworkVersion (int64) (int64) + if t.NetworkVersion >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.NetworkVersion)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.NetworkVersion-1)); err != nil { + return err + } + } + + // t.Caller (address.Address) (struct) + if err := t.Caller.MarshalCBOR(w); err != nil { + return err + } + + // t.Receiver (address.Address) (struct) + if err := t.Receiver.MarshalCBOR(w); err != nil { + return err + } + + // t.ValueReceived (big.Int) (struct) + if err := t.ValueReceived.MarshalCBOR(w); err != nil { + return err + } + + // t.CurrEpoch (abi.ChainEpoch) (int64) + if t.CurrEpoch >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.CurrEpoch)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.CurrEpoch-1)); err != nil { + return err + } + } + + // t.CurrentBalance (big.Int) (struct) + if err := t.CurrentBalance.MarshalCBOR(w); err != nil { + return err + } + + // t.State (chaos.State) (struct) + if err := t.State.MarshalCBOR(w); err != nil { + return err + } + + // t.TotalFilCircSupply (big.Int) (struct) + if err := t.TotalFilCircSupply.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { + *t = InspectRuntimeReturn{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 8 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.NetworkVersion (int64) (int64) + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.NetworkVersion = int64(extraI) + } + // t.Caller (address.Address) (struct) + + { + + if err := t.Caller.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.Caller: %w", err) + } + + } + // t.Receiver (address.Address) (struct) + + { + + if err := t.Receiver.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.Receiver: %w", err) + } + + } + // t.ValueReceived (big.Int) (struct) + + { + + if err := t.ValueReceived.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.ValueReceived: %w", err) + } + + } + // t.CurrEpoch (abi.ChainEpoch) (int64) + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.CurrEpoch = abi.ChainEpoch(extraI) + } + // t.CurrentBalance (big.Int) (struct) + + { + + if err := t.CurrentBalance.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.CurrentBalance: %w", err) + } + + } + // t.State (chaos.State) (struct) + + { + + if err := t.State.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.State: %w", err) + } + + } + // t.TotalFilCircSupply (big.Int) (struct) + + { + + if err := t.TotalFilCircSupply.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.TotalFilCircSupply: %w", err) + } + + } + return nil +} diff --git a/conformance/chaos/gen/gen.go b/conformance/chaos/gen/gen.go index 97ea98dc8..496cc3d35 100644 --- a/conformance/chaos/gen/gen.go +++ b/conformance/chaos/gen/gen.go @@ -15,6 +15,7 @@ func main() { chaos.SendReturn{}, chaos.MutateStateArgs{}, chaos.AbortWithArgs{}, + chaos.InspectRuntimeReturn{}, ); err != nil { panic(err) } From 24756ac0568e2d43682a12aa419a9bdeb1cac72e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 15 Sep 2020 11:55:54 -0700 Subject: [PATCH 05/54] allow specification of repo directory in lotus-bench --- cmd/lotus-bench/import.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 7fe63eae1..f2845ba20 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -56,6 +56,10 @@ var importBenchCmd = &cli.Command{ Usage: "set the parallelism factor for batch seal verification", Value: runtime.NumCPU(), }, + &cli.StringFlag{ + Name: "repodir", + Usage: "set the repo directory for the lotus bench run (defaults to /tmp)", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -70,9 +74,15 @@ var importBenchCmd = &cli.Command{ } defer cfi.Close() //nolint:errcheck // read only file - tdir, err := ioutil.TempDir("", "lotus-import-bench") - if err != nil { - return err + var tdir string + if rdir := cctx.String("repodir"); rdir != "" { + tdir = rdir + } else { + tmp, err := ioutil.TempDir("", "lotus-import-bench") + if err != nil { + return err + } + tdir = tmp } bds, err := badger.NewDatastore(tdir, nil) From ed4caac9bff516ffd16544641c5fcc3b17c9c88f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 22:24:09 +0100 Subject: [PATCH 06/54] fix: remove network version and total fil --- conformance/chaos/actor.go | 27 ++++++++---------- conformance/chaos/cbor_gen.go | 54 ++--------------------------------- 2 files changed, 14 insertions(+), 67 deletions(-) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index 526dac64c..005a06f0d 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -254,14 +254,12 @@ func (a Actor) AbortWith(rt runtime.Runtime, args *AbortWithArgs) *abi.EmptyValu // InspectRuntimeReturn is the return value for the Actor.InspectRuntime method. type InspectRuntimeReturn struct { - NetworkVersion int64 - Caller address.Address - Receiver address.Address - ValueReceived abi.TokenAmount - CurrEpoch abi.ChainEpoch - CurrentBalance abi.TokenAmount - State State - TotalFilCircSupply abi.TokenAmount + Caller address.Address + Receiver address.Address + ValueReceived abi.TokenAmount + CurrEpoch abi.ChainEpoch + CurrentBalance abi.TokenAmount + State State } // InspectRuntime returns a copy of the serializable values available in the Runtime. @@ -270,12 +268,11 @@ func (a Actor) InspectRuntime(rt runtime.Runtime, _ *abi.EmptyValue) *InspectRun var st State rt.StateReadonly(&st) return &InspectRuntimeReturn{ - NetworkVersion: int64(rt.NetworkVersion()), - Caller: rt.Caller(), - Receiver: rt.Receiver(), - ValueReceived: rt.ValueReceived(), - CurrentBalance: rt.CurrentBalance(), - State: st, - TotalFilCircSupply: rt.TotalFilCircSupply(), + Caller: rt.Caller(), + Receiver: rt.Receiver(), + ValueReceived: rt.ValueReceived(), + CurrEpoch: rt.CurrEpoch(), + CurrentBalance: rt.CurrentBalance(), + State: st, } } diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 96b707386..2d9deec93 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -731,7 +731,7 @@ func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error { return nil } -var lengthBufInspectRuntimeReturn = []byte{136} +var lengthBufInspectRuntimeReturn = []byte{134} func (t *InspectRuntimeReturn) MarshalCBOR(w io.Writer) error { if t == nil { @@ -744,17 +744,6 @@ func (t *InspectRuntimeReturn) MarshalCBOR(w io.Writer) error { scratch := make([]byte, 9) - // t.NetworkVersion (int64) (int64) - if t.NetworkVersion >= 0 { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.NetworkVersion)); err != nil { - return err - } - } else { - if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.NetworkVersion-1)); err != nil { - return err - } - } - // t.Caller (address.Address) (struct) if err := t.Caller.MarshalCBOR(w); err != nil { return err @@ -790,11 +779,6 @@ func (t *InspectRuntimeReturn) MarshalCBOR(w io.Writer) error { if err := t.State.MarshalCBOR(w); err != nil { return err } - - // t.TotalFilCircSupply (big.Int) (struct) - if err := t.TotalFilCircSupply.MarshalCBOR(w); err != nil { - return err - } return nil } @@ -812,35 +796,10 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("cbor input should be of type array") } - if extra != 8 { + if extra != 6 { return fmt.Errorf("cbor input had wrong number of fields") } - // t.NetworkVersion (int64) (int64) - { - maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) - var extraI int64 - if err != nil { - return err - } - switch maj { - case cbg.MajUnsignedInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 positive overflow") - } - case cbg.MajNegativeInt: - extraI = int64(extra) - if extraI < 0 { - return fmt.Errorf("int64 negative oveflow") - } - extraI = -1 - extraI - default: - return fmt.Errorf("wrong type for int64 field: %d", maj) - } - - t.NetworkVersion = int64(extraI) - } // t.Caller (address.Address) (struct) { @@ -910,15 +869,6 @@ func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { return xerrors.Errorf("unmarshaling t.State: %w", err) } - } - // t.TotalFilCircSupply (big.Int) (struct) - - { - - if err := t.TotalFilCircSupply.UnmarshalCBOR(br); err != nil { - return xerrors.Errorf("unmarshaling t.TotalFilCircSupply: %w", err) - } - } return nil } From 952f2c2f1e2659ebe4fb4f5940832d02e5da6c4e Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 9 Sep 2020 05:13:51 -0400 Subject: [PATCH 07/54] Sever chainstore's dependence on the state tree --- chain/store/store.go | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 20a7e3031..ad30e0324 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -18,7 +18,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/journal" bstore "github.com/filecoin-project/lotus/lib/blockstore" @@ -767,32 +766,16 @@ type BlockMessages struct { func (cs *ChainStore) BlockMsgsForTipset(ts *types.TipSet) ([]BlockMessages, error) { applied := make(map[address.Address]uint64) - cst := cbor.NewCborStore(cs.bs) - st, err := state.LoadStateTree(cst, ts.Blocks()[0].ParentStateRoot) - if err != nil { - return nil, xerrors.Errorf("failed to load state tree") - } - - preloadAddr := func(a address.Address) error { - if _, ok := applied[a]; !ok { - act, err := st.GetActor(a) - if err != nil { - return err - } - - applied[a] = act.Nonce - } - return nil - } - selectMsg := func(m *types.Message) (bool, error) { - if err := preloadAddr(m.From); err != nil { - return false, err + // The first match for a sender is guaranteed to have correct nonce -- the block isn't valid otherwise + if _, ok := applied[m.From]; !ok { + applied[m.From] = m.Nonce } if applied[m.From] != m.Nonce { return false, nil } + applied[m.From]++ return true, nil From 937ff4e9aef0273c7ec4b9edadd3d3dea4610f82 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 15 Sep 2020 20:55:11 -0400 Subject: [PATCH 08/54] Add a sync test around blocks with bad nonce messages --- chain/sync_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/chain/sync_test.go b/chain/sync_test.go index 0b0d1ed00..f4c83341f 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -662,6 +662,49 @@ func TestDuplicateNonce(t *testing.T) { require.Equal(t, includedMsg, mft[0].VMMessage().Cid(), "messages for tipset didn't contain expected message") } +// This test asserts that a block that includes a message with bad nonce can't be synced. A nonce is "bad" if it can't +// be applied on the parent state. +func TestBadNonce(t *testing.T) { + H := 10 + tu := prepSyncTest(t, H) + + base := tu.g.CurTipset + + // Produce a message from the banker with a bad nonce + makeBadMsg := func() *types.SignedMessage { + + ba, err := tu.nds[0].StateGetActor(context.TODO(), tu.g.Banker(), base.TipSet().Key()) + require.NoError(t, err) + msg := types.Message{ + To: tu.g.Banker(), + From: tu.g.Banker(), + + Nonce: ba.Nonce + 5, + + Value: types.NewInt(1), + + Method: 0, + + GasLimit: 100_000_000, + GasFeeCap: types.NewInt(0), + GasPremium: types.NewInt(0), + } + + sig, err := tu.g.Wallet().Sign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) + require.NoError(t, err) + + return &types.SignedMessage{ + Message: msg, + Signature: *sig, + } + } + + msgs := make([][]*types.SignedMessage, 1) + msgs[0] = []*types.SignedMessage{makeBadMsg()} + + tu.mineOnBlock(base, 0, []int{0}, true, true, msgs) +} + func BenchmarkSyncBasic(b *testing.B) { for i := 0; i < b.N; i++ { runSyncBenchLength(b, 100) From fa9be9a627385da6a6c4a36d94f9b0b9bd811a43 Mon Sep 17 00:00:00 2001 From: zgfzgf <1901989065@qq.com> Date: Wed, 16 Sep 2020 09:14:56 +0800 Subject: [PATCH 09/54] Replace fx.Extract with fx.Populate --- node/builder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/builder.go b/node/builder.go index 8c7674ffd..c37a5db58 100644 --- a/node/builder.go +++ b/node/builder.go @@ -382,7 +382,7 @@ func StorageMiner(out *api.StorageMiner) Option { func(s *Settings) error { resAPI := &impl.StorageMinerAPI{} - s.invokes[ExtractApiKey] = fx.Extract(resAPI) + s.invokes[ExtractApiKey] = fx.Populate(resAPI) *out = resAPI return nil }, @@ -509,7 +509,7 @@ func FullAPI(out *api.FullNode) Option { }, func(s *Settings) error { resAPI := &impl.FullNodeAPI{} - s.invokes[ExtractApiKey] = fx.Extract(resAPI) + s.invokes[ExtractApiKey] = fx.Populate(resAPI) *out = resAPI return nil }, From 5c69249ba36caf130c1fa5aa1c6937c7d86e34ae Mon Sep 17 00:00:00 2001 From: jennijuju Date: Tue, 15 Sep 2020 21:19:27 -0400 Subject: [PATCH 10/54] Use `window post` for window PoSt related log messages. --- chain/stmgr/utils.go | 2 +- chain/sync.go | 2 +- cmd/lotus-bench/main.go | 4 ++-- storage/addresses.go | 2 +- storage/wdpost_run.go | 20 ++++++++++---------- storage/wdpost_sched.go | 14 +++++++------- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 27b71f358..c09f2b22a 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -554,7 +554,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule sectors, err := GetSectorsForWinningPoSt(ctx, pv, sm, lbst, maddr, prand) if err != nil { - return nil, xerrors.Errorf("getting wpost proving set: %w", err) + return nil, xerrors.Errorf("getting winning post proving set: %w", err) } if len(sectors) == 0 { diff --git a/chain/sync.go b/chain/sync.go index f7530f556..b5716a343 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -991,7 +991,7 @@ func (syncer *Syncer) VerifyWinningPoStProof(ctx context.Context, h *types.Block rand, err := store.DrawRandomness(rbase.Data, crypto.DomainSeparationTag_WinningPoStChallengeSeed, h.Height, buf.Bytes()) if err != nil { - return xerrors.Errorf("failed to get randomness for verifying winningPost proof: %w", err) + return xerrors.Errorf("failed to get randomness for verifying winning post proof: %w", err) } mid, err := address.IDFromAddress(h.Miner) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 431bfdd44..7da754415 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -387,7 +387,7 @@ var sealBenchCmd = &cli.Command{ return err } if !ok { - log.Error("post verification failed") + log.Error("window post verification failed") } verifyWindowpost1 := time.Now() @@ -403,7 +403,7 @@ var sealBenchCmd = &cli.Command{ return err } if !ok { - log.Error("post verification failed") + log.Error("window post verification failed") } verifyWindowpost2 := time.Now() diff --git a/storage/addresses.go b/storage/addresses.go index bef845367..2388b9fca 100644 --- a/storage/addresses.go +++ b/storage/addresses.go @@ -60,7 +60,7 @@ func AddressFor(ctx context.Context, a addrSelectApi, mi api.MinerInfo, use Addr return addr, nil } - log.Warnw("control address didn't have enough funds for PoSt message", "address", addr, "required", types.FIL(minFunds), "balance", types.FIL(b)) + log.Warnw("control address didn't have enough funds for window post message", "address", addr, "required", types.FIL(minFunds), "balance", types.FIL(b)) } // Try to use the owner account if we can, fallback to worker if we can't diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 7aa90bbba..17b567440 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -78,7 +78,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, posts, err := s.runPost(ctx, *deadline, ts) if err != nil { - log.Errorf("runPost failed: %+v", err) + log.Errorf("run window post failed: %+v", err) s.failPost(err, deadline) return } @@ -92,7 +92,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, post := &posts[i] sm, err := s.submitPost(ctx, post) if err != nil { - log.Errorf("submitPost failed: %+v", err) + log.Errorf("submit window post failed: %+v", err) s.failPost(err, deadline) } else { recordProofsEvent(post.Partitions, sm.Cid()) @@ -397,7 +397,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty rand, err := s.api.ChainGetRandomnessFromBeacon(ctx, ts.Key(), crypto.DomainSeparationTag_WindowedPoStChallengeSeed, di.Challenge, buf.Bytes()) if err != nil { - return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), di, err) + return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err) } // Get the partitions for the given deadline @@ -490,7 +490,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } // Generate proof - log.Infow("running windowPost", + log.Infow("running window post", "chain-random", rand, "deadline", di, "height", ts.Height(), @@ -507,7 +507,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty postOut, ps, err = s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) elapsed := time.Since(tsStart) - log.Infow("computing window PoSt", "batch", batchIdx, "elapsed", elapsed) + log.Infow("computing window post", "batch", batchIdx, "elapsed", elapsed) if err == nil { // Proof generation successful, stop retrying @@ -517,10 +517,10 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty // Proof generation failed, so retry if len(ps) == 0 { - return nil, xerrors.Errorf("running post failed: %w", err) + return nil, xerrors.Errorf("running window post failed: %w", err) } - log.Warnw("generate window PoSt skipped sectors", "sectors", ps, "error", err, "try", retries) + log.Warnw("generate window post skipped sectors", "sectors", ps, "error", err, "try", retries) skipCount += uint64(len(ps)) for _, sector := range ps { @@ -547,7 +547,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty commEpoch := di.Open commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) if err != nil { - return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) + return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) } for i := range posts { @@ -644,7 +644,7 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi enc, aerr := actors.SerializeParams(proof) if aerr != nil { - return nil, xerrors.Errorf("could not serialize submit post parameters: %w", aerr) + return nil, xerrors.Errorf("could not serialize submit window post parameters: %w", aerr) } msg := &types.Message{ @@ -705,7 +705,7 @@ func (s *WindowPoStScheduler) setSender(ctx context.Context, msg *types.Message, pa, err := AddressFor(ctx, s.api, mi, PoStAddr, minFunds) if err != nil { - log.Errorw("error selecting address for post", "error", err) + log.Errorw("error selecting address for window post", "error", err) msg.From = s.worker return } diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 037f4e481..d130d801c 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -110,7 +110,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { select { case changes, ok := <-notifs: if !ok { - log.Warn("WindowPoStScheduler notifs channel closed") + log.Warn("window post scheduler notifs channel closed") notifs = nil continue } @@ -151,10 +151,10 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { } if err := s.revert(ctx, lowest); err != nil { - log.Error("handling head reverts in windowPost sched: %+v", err) + log.Error("handling head reverts in window post sched: %+v", err) } if err := s.update(ctx, highest); err != nil { - log.Error("handling head updates in windowPost sched: %+v", err) + log.Error("handling head updates in window post sched: %+v", err) } span.End() @@ -184,7 +184,7 @@ func (s *WindowPoStScheduler) revert(ctx context.Context, newLowest *types.TipSe func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) error { if new == nil { - return xerrors.Errorf("no new tipset in WindowPoStScheduler.update") + return xerrors.Errorf("no new tipset in window post sched update") } di, err := s.api.StateMinerProvingDeadline(ctx, s.actor, new.Key()) @@ -206,7 +206,7 @@ func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) err // (Need to get correct deadline above, which is tricky) if di.Open+StartConfidence >= new.Height() { - log.Info("not starting windowPost yet, waiting for startconfidence", di.Open, di.Open+StartConfidence, new.Height()) + log.Info("not starting window post yet, waiting for startconfidence", di.Open, di.Open+StartConfidence, new.Height()) return nil } @@ -216,7 +216,7 @@ func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) err s.activeEPS = 0 } s.failLk.Unlock()*/ - log.Infof("at %d, doPost for P %d, dd %d", new.Height(), di.PeriodStart, di.Index) + log.Infof("at %d, do window post for P %d, dd %d", new.Height(), di.PeriodStart, di.Index) s.doPost(ctx, di, new) @@ -238,7 +238,7 @@ func (s *WindowPoStScheduler) abortActivePoSt() { } }) - log.Warnf("Aborting Window PoSt (Deadline: %+v)", s.activeDeadline) + log.Warnf("Aborting window post (Deadline: %+v)", s.activeDeadline) } s.activeDeadline = nil From e2cf0c4d4c0659931960982205c349e75e65c9c4 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Tue, 15 Sep 2020 21:24:30 -0400 Subject: [PATCH 11/54] Make interactive deal CLI reject deals shorter than minimum duration --- cli/client.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cli/client.go b/cli/client.go index e68f98791..d9ce9251e 100644 --- a/cli/client.go +++ b/cli/client.go @@ -12,6 +12,8 @@ import ( "text/tabwriter" "time" + "github.com/filecoin-project/specs-actors/actors/builtin" + tm "github.com/buger/goterm" "github.com/docker/go-units" "github.com/fatih/color" @@ -527,6 +529,11 @@ func interactiveDeal(cctx *cli.Context) error { continue } + if days < int(build.MinDealDuration/builtin.EpochsInDay) { + printErr(xerrors.Errorf("minimum duration is %d days", int(build.MinDealDuration/builtin.EpochsInDay))) + continue + } + state = "miner" case "miner": fmt.Print("Miner Address (t0..): ") From 808051d34ed5cdde0cb12e94a4f188952eb7c137 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 16 Sep 2020 02:06:40 +0000 Subject: [PATCH 12/54] break out of mining loop when stop is called during niceSleep --- miner/miner.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/miner/miner.go b/miner/miner.go index bcfdb46c1..555fe2e19 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -139,6 +139,7 @@ func (m *Miner) niceSleep(d time.Duration) bool { case <-build.Clock.After(d): return true case <-m.stop: + log.Infow("recieved interrupt while trying to sleep in mining cycle") return false } } @@ -169,7 +170,9 @@ func (m *Miner) mine(ctx context.Context) { prebase, err := m.GetBestMiningCandidate(ctx) if err != nil { log.Errorf("failed to get best mining candidate: %s", err) - m.niceSleep(time.Second * 5) + if !m.niceSleep(time.Second * 5) { + break + } continue } @@ -199,7 +202,9 @@ func (m *Miner) mine(ctx context.Context) { _, err = m.api.BeaconGetEntry(ctx, prebase.TipSet.Height()+prebase.NullRounds+1) if err != nil { log.Errorf("failed getting beacon entry: %s", err) - m.niceSleep(time.Second) + if !m.niceSleep(time.Second) { + break + } continue } @@ -208,7 +213,9 @@ func (m *Miner) mine(ctx context.Context) { if base.TipSet.Equals(lastBase.TipSet) && lastBase.NullRounds == base.NullRounds { log.Warnf("BestMiningCandidate from the previous round: %s (nulls:%d)", lastBase.TipSet.Cids(), lastBase.NullRounds) - m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) + if !m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) { + break + } continue } @@ -217,7 +224,9 @@ func (m *Miner) mine(ctx context.Context) { b, err := m.mineOne(ctx, base) if err != nil { log.Errorf("mining block failed: %+v", err) - m.niceSleep(time.Second) + if !m.niceSleep(time.Second) { + break + } onDone(false, 0, err) continue } From 362fc180ec5ea053047a04ad39984cbfdcc5cc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Wed, 16 Sep 2020 10:53:15 +0100 Subject: [PATCH 13/54] fix a regression caused by #3800. --- chain/vm/burn.go | 19 ++++++++++++------- chain/vm/vm.go | 30 +++++++++++++++--------------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/chain/vm/burn.go b/chain/vm/burn.go index eb0611349..9f9b95755 100644 --- a/chain/vm/burn.go +++ b/chain/vm/burn.go @@ -22,6 +22,17 @@ type GasOutputs struct { GasBurned int64 } +// ZeroGasOutputs returns a logically zeroed GasOutputs. +func ZeroGasOutputs() GasOutputs { + return GasOutputs{ + BaseFeeBurn: big.Zero(), + OverEstimationBurn: big.Zero(), + MinerPenalty: big.Zero(), + MinerTip: big.Zero(), + Refund: big.Zero(), + } +} + // ComputeGasOverestimationBurn computes amount of gas to be refunded and amount of gas to be burned // Result is (refund, burn) func ComputeGasOverestimationBurn(gasUsed, gasLimit int64) (int64, int64) { @@ -58,13 +69,7 @@ func ComputeGasOverestimationBurn(gasUsed, gasLimit int64) (int64, int64) { func ComputeGasOutputs(gasUsed, gasLimit int64, baseFee, feeCap, gasPremium abi.TokenAmount) GasOutputs { gasUsedBig := big.NewInt(gasUsed) - out := GasOutputs{ - BaseFeeBurn: big.Zero(), - OverEstimationBurn: big.Zero(), - MinerPenalty: big.Zero(), - MinerTip: big.Zero(), - Refund: big.Zero(), - } + out := ZeroGasOutputs() baseFeeToPay := baseFee if baseFee.Cmp(feeCap.Int) > 0 { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index f245bb2b0..a90856a1b 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -355,14 +355,14 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, msgGasCost := msgGas.Total() // this should never happen, but is currently still exercised by some tests if msgGasCost > msg.GasLimit { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = types.BigMul(vm.baseFee, abi.NewTokenAmount(msgGasCost)) return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrOutOfGas, GasUsed: 0, }, - GasCosts: GasOutputs{ - MinerPenalty: types.BigMul(vm.baseFee, abi.NewTokenAmount(msgGasCost)), - }, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } @@ -374,15 +374,15 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, // this should never happen, but is currently still exercised by some tests if err != nil { if xerrors.Is(err, types.ErrActorNotFound) { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderInvalid, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "actor not found: %s", msg.From), - GasCosts: GasOutputs{ - MinerPenalty: minerPenaltyAmount, - }, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } @@ -391,20 +391,22 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, // this should never happen, but is currently still exercised by some tests if !fromActor.Code.Equals(builtin.AccountActorCodeID) { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderInvalid, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "send from not account actor: %s", fromActor.Code), - GasCosts: GasOutputs{ - MinerPenalty: minerPenaltyAmount, - }, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } if msg.Nonce != fromActor.Nonce { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderStateInvalid, @@ -413,15 +415,15 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor nonce invalid: msg:%d != state:%d", msg.Nonce, fromActor.Nonce), - GasCosts: GasOutputs{ - MinerPenalty: minerPenaltyAmount, - }, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } gascost := types.BigMul(types.NewInt(uint64(msg.GasLimit)), msg.GasFeeCap) if fromActor.Balance.LessThan(gascost) { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderStateInvalid, @@ -429,9 +431,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor balance less than needed: %s < %s", types.FIL(fromActor.Balance), types.FIL(gascost)), - GasCosts: GasOutputs{ - MinerPenalty: minerPenaltyAmount, - }, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } From d870c74a5e22115517f8fb74905001b014f97ea8 Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 16 Sep 2020 19:08:47 +0800 Subject: [PATCH 14/54] add lotus-pcr to ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 940f37b3f..05f762d8d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ /lotus-stats /lotus-bench /lotus-gateway +/lotus-pcr /bench.json /lotuspond/front/node_modules /lotuspond/front/build From b4115021c5063ec5c12a44f3b4d62c055f79d8e5 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 15 Sep 2020 15:02:14 +0200 Subject: [PATCH 15/54] make sure lotus compiles with +testground build flag --- tools/stats/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index ae79d9273..e50ac953f 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -189,7 +189,7 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti pl.AddPoint(p) } { - blks := len(cids) + blks := int64(len(cids)) p = NewPoint("chain.gas_fill_ratio", float64(totalGasLimit)/float64(blks*build.BlockGasTarget)) pl.AddPoint(p) p = NewPoint("chain.gas_capacity_ratio", float64(totalUniqGasLimit)/float64(blks*build.BlockGasTarget)) From ed74091c204c417a1f82ec3f6260e32f8042bb9f Mon Sep 17 00:00:00 2001 From: Frank Date: Wed, 16 Sep 2020 19:49:45 +0800 Subject: [PATCH 16/54] add exist sector state check --- cmd/lotus-storage-miner/sectors.go | 6 +++++ extern/storage-sealing/sector_state.go | 33 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 06f09fe20..c08d6ca14 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -17,6 +17,8 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" lcli "github.com/filecoin-project/lotus/cli" ) @@ -420,6 +422,10 @@ var sectorsUpdateCmd = &cli.Command{ return xerrors.Errorf("could not parse sector number: %w", err) } + if _, ok := sealing.ExistSectorStateList[sealing.SectorState(cctx.Args().Get(1))]; !ok { + return xerrors.Errorf("Not existing sector state") + } + return nodeApi.SectorsUpdate(ctx, abi.SectorNumber(id), api.SectorState(cctx.Args().Get(1))) }, } diff --git a/extern/storage-sealing/sector_state.go b/extern/storage-sealing/sector_state.go index 4e674603d..3ed891064 100644 --- a/extern/storage-sealing/sector_state.go +++ b/extern/storage-sealing/sector_state.go @@ -2,6 +2,8 @@ package sealing type SectorState string +var ExistSectorStateList = make(map[SectorState]struct{}) + const ( UndefinedSectorState SectorState = "" @@ -39,6 +41,37 @@ const ( RemoveFailed SectorState = "RemoveFailed" Removed SectorState = "Removed" ) +func init() { + ExistSectorStateList[Empty] = struct{}{} + ExistSectorStateList[WaitDeals] = struct{}{} + ExistSectorStateList[Packing] = struct{}{} + ExistSectorStateList[PreCommit1] = struct{}{} + ExistSectorStateList[PreCommit2] = struct{}{} + ExistSectorStateList[PreCommitting] = struct{}{} + ExistSectorStateList[PreCommitWait] = struct{}{} + ExistSectorStateList[WaitSeed] = struct{}{} + ExistSectorStateList[Committing] = struct{}{} + ExistSectorStateList[SubmitCommit] = struct{}{} + ExistSectorStateList[CommitWait] = struct{}{} + ExistSectorStateList[FinalizeSector] = struct{}{} + ExistSectorStateList[Proving] = struct{}{} + ExistSectorStateList[FailedUnrecoverable] = struct{}{} + ExistSectorStateList[SealPreCommit1Failed] = struct{}{} + ExistSectorStateList[SealPreCommit2Failed] = struct{}{} + ExistSectorStateList[PreCommitFailed] = struct{}{} + ExistSectorStateList[ComputeProofFailed] = struct{}{} + ExistSectorStateList[CommitFailed] = struct{}{} + ExistSectorStateList[PackingFailed] = struct{}{} + ExistSectorStateList[FinalizeFailed] = struct{}{} + ExistSectorStateList[DealsExpired] = struct{}{} + ExistSectorStateList[RecoverDealIDs] = struct{}{} + ExistSectorStateList[Faulty] = struct{}{} + ExistSectorStateList[FaultReported] = struct{}{} + ExistSectorStateList[FaultedFinal] = struct{}{} + ExistSectorStateList[Removing] = struct{}{} + ExistSectorStateList[RemoveFailed] = struct{}{} + ExistSectorStateList[Removed] = struct{}{} +} func toStatState(st SectorState) statSectorState { switch st { From 3f67c4524e8bbdae123e1ee50cf603899ce5cde0 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 15 Sep 2020 11:42:52 +0200 Subject: [PATCH 17/54] add filecoin-project oni as submodule and compile lotus-soup --- .circleci/config.yml | 19 +++++++++++++++++++ .gitmodules | 3 +++ extern/oni | 1 + 3 files changed, 23 insertions(+) create mode 160000 extern/oni diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fdb3e38f..5e2ef6eab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -254,6 +254,25 @@ jobs: path: /tmp/test-reports - store_artifacts: path: /tmp/test-artifacts/conformance-coverage.html + test-lotus-soup: + description: | + Compile `lotus-soup` Testground test plan using the current version of Lotus. + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: cd extern/oni && git submodule sync + - run: cd extern/oni && git submodule update --init + - run: cd extern/oni/extra/filecoin-ffi && make + - run: + name: "update lotus-soup dependency to lotus" + command: pushd extern/oni/lotus-soup && echo 'replace github.com/filecoin-project/lotus => ../../../' >> go.mod + - run: + name: "build lotus-soup testplan" + command: pushd extern/oni/lotus-soup && go build -tags=testground . + build-macos: description: build darwin lotus binary diff --git a/.gitmodules b/.gitmodules index 4b450aaf3..35f5a3d3f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,3 +11,6 @@ [submodule "extern/fil-blst"] path = extern/fil-blst url = https://github.com/filecoin-project/fil-blst.git +[submodule "extern/oni"] + path = extern/oni + url = https://github.com/filecoin-project/oni diff --git a/extern/oni b/extern/oni new file mode 160000 index 000000000..d9bcdfe61 --- /dev/null +++ b/extern/oni @@ -0,0 +1 @@ +Subproject commit d9bcdfe61b89f00bef07f6d3995e36e277930acd From f4f80b25d11d53ca69ceb285dfefb748f72a2b41 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 16 Sep 2020 12:40:08 +0200 Subject: [PATCH 18/54] add test-lotus-soup to workflows --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e2ef6eab..b2340c836 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -447,6 +447,7 @@ workflows: test-suite-name: conformance-bleeding-edge packages: "./conformance" vectors-branch: master + - test-lotus-soup - build-debug - build-all: requires: From b0677ab38e675f66eb05213aa6a98bb56fc7c45b Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 16 Sep 2020 12:49:45 +0200 Subject: [PATCH 19/54] replace lotus and filecoin-ffi deps --- .circleci/config.yml | 6 +++--- extern/oni | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b2340c836..468812ccc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -265,10 +265,10 @@ jobs: - prepare - run: cd extern/oni && git submodule sync - run: cd extern/oni && git submodule update --init - - run: cd extern/oni/extra/filecoin-ffi && make + - run: cd extern/filecoin-ffi && make - run: - name: "update lotus-soup dependency to lotus" - command: pushd extern/oni/lotus-soup && echo 'replace github.com/filecoin-project/lotus => ../../../' >> go.mod + name: "replace lotus and filecoin-ffi deps" + command: pushd extern/oni/lotus-soup && go mod edit -replace github.com/filecoin-project/lotus=../../../ && go mod edit -replace github.com/filecoin-project/filecoin-ffi=../../filecoin-ffi - run: name: "build lotus-soup testplan" command: pushd extern/oni/lotus-soup && go build -tags=testground . diff --git a/extern/oni b/extern/oni index d9bcdfe61..097d82535 160000 --- a/extern/oni +++ b/extern/oni @@ -1 +1 @@ -Subproject commit d9bcdfe61b89f00bef07f6d3995e36e277930acd +Subproject commit 097d82535de690160fb84e97d0baf1bd04e175c6 From 0ff5a71114622a0109f682d048b81f6f7fcc4461 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 16 Sep 2020 15:13:28 +0200 Subject: [PATCH 20/54] update oni to master --- extern/oni | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/oni b/extern/oni index 097d82535..8b7e7d438 160000 --- a/extern/oni +++ b/extern/oni @@ -1 +1 @@ -Subproject commit 097d82535de690160fb84e97d0baf1bd04e175c6 +Subproject commit 8b7e7d438c4cc38a0d2d671876d4590ad20655b3 From 56f223bfc3ee736212a4125cfd8c36b56af03e09 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 16 Sep 2020 15:16:15 +0200 Subject: [PATCH 21/54] add go mod edit for blst deps --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 468812ccc..fff35d7a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -267,8 +267,8 @@ jobs: - run: cd extern/oni && git submodule update --init - run: cd extern/filecoin-ffi && make - run: - name: "replace lotus and filecoin-ffi deps" - command: pushd extern/oni/lotus-soup && go mod edit -replace github.com/filecoin-project/lotus=../../../ && go mod edit -replace github.com/filecoin-project/filecoin-ffi=../../filecoin-ffi + name: "replace lotus, filecoin-ffi, blst and fil-blst deps" + command: cd extern/oni/lotus-soup && go mod edit -replace github.com/filecoin-project/lotus=../../../ && go mod edit -replace github.com/filecoin-project/filecoin-ffi=../../filecoin-ffi && go mod edit -replace github.com/supranational/blst=../../fil-blst/blst && go mod edit -replace github.com/filecoin-project/fil-blst=../../fil-blst - run: name: "build lotus-soup testplan" command: pushd extern/oni/lotus-soup && go build -tags=testground . From 6656e5a6c836fdd762fc198772bba99bdd7cff46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 16 Sep 2020 22:47:06 +0200 Subject: [PATCH 22/54] circle: test-lotus-soup -> build-lotus-soup --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fff35d7a8..acd447f69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -254,7 +254,7 @@ jobs: path: /tmp/test-reports - store_artifacts: path: /tmp/test-artifacts/conformance-coverage.html - test-lotus-soup: + build-lotus-soup: description: | Compile `lotus-soup` Testground test plan using the current version of Lotus. parameters: @@ -447,7 +447,7 @@ workflows: test-suite-name: conformance-bleeding-edge packages: "./conformance" vectors-branch: master - - test-lotus-soup + - build-lotus-soup - build-debug - build-all: requires: From a6b7791d325fec21798540aee420e6511fda2734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 16 Sep 2020 22:56:04 +0200 Subject: [PATCH 23/54] lint --- miner/miner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/miner/miner.go b/miner/miner.go index 555fe2e19..2d8591992 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -139,7 +139,7 @@ func (m *Miner) niceSleep(d time.Duration) bool { case <-build.Clock.After(d): return true case <-m.stop: - log.Infow("recieved interrupt while trying to sleep in mining cycle") + log.Infow("received interrupt while trying to sleep in mining cycle") return false } } From 665d23cb8e72f661d55beb1f390216e4bcf3c7f4 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Tue, 15 Sep 2020 12:10:22 -0700 Subject: [PATCH 24/54] Add set-ask, get-ask retrieval CLI --- cmd/lotus-storage-miner/retrieval-deals.go | 114 +++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/cmd/lotus-storage-miner/retrieval-deals.go b/cmd/lotus-storage-miner/retrieval-deals.go index df194978d..0f15f19f2 100644 --- a/cmd/lotus-storage-miner/retrieval-deals.go +++ b/cmd/lotus-storage-miner/retrieval-deals.go @@ -5,9 +5,12 @@ import ( "os" "text/tabwriter" + "github.com/docker/go-units" "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-state-types/abi" "github.com/urfave/cli/v2" + "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) @@ -17,6 +20,8 @@ var retrievalDealsCmd = &cli.Command{ Subcommands: []*cli.Command{ retrievalDealSelectionCmd, retrievalDealsListCmd, + retrievalSetAskCmd, + retrievalGetAskCmd, }, } @@ -154,3 +159,112 @@ var retrievalDealsListCmd = &cli.Command{ return w.Flush() }, } + +var retrievalSetAskCmd = &cli.Command{ + Name: "set-ask", + Usage: "Configure the provider's retrieval ask", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "price", + Usage: "Set the price of the ask for retrievals (per byte)", + }, + &cli.StringFlag{ + Name: "unseal-price", + Usage: "Set the price to unseal", + }, + &cli.StringFlag{ + Name: "payment-interval", + Usage: "Set the payment interval (in bytes) for retrieval", + DefaultText: "1Mb", + }, + &cli.StringFlag{ + Name: "payment-interval-increase", + Usage: "Set the payment interval increase (in bytes) for retrieval", + DefaultText: "1Mb", + }, + }, + Action: func(cctx *cli.Context) error { + ctx := lcli.DaemonContext(cctx) + + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + ask, err := api.MarketGetRetrievalAsk(ctx) + if err != nil { + return err + } + + if cctx.IsSet("price") { + v, err := types.ParseFIL(cctx.String("price")) + if err != nil { + return err + } + ask.PricePerByte = abi.TokenAmount(v) + } + + if cctx.IsSet("unseal-price") { + v, err := types.ParseFIL(cctx.String("unseal-price")) + if err != nil { + return err + } + ask.UnsealPrice = abi.TokenAmount(v) + } + + if cctx.IsSet("payment-interval") { + v, err := units.RAMInBytes(cctx.String("payment-interval")) + if err != nil { + return err + } + ask.PaymentInterval = uint64(v) + } + + if cctx.IsSet("payment-interval-increase") { + v, err := units.RAMInBytes(cctx.String("payment-interval-increase")) + if err != nil { + return err + } + ask.PaymentIntervalIncrease = uint64(v) + } + + return api.MarketSetRetrievalAsk(ctx, ask) + }, +} + +var retrievalGetAskCmd = &cli.Command{ + Name: "get-ask", + Usage: "Get the provider's current retrieval ask", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + ctx := lcli.DaemonContext(cctx) + + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + ask, err := api.MarketGetRetrievalAsk(ctx) + if err != nil { + return err + } + + w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) + fmt.Fprintf(w, "Price per Byte\tUnseal Price\tPayment Interval\tPayment Interval Increase\n") + if ask == nil { + fmt.Fprintf(w, "\n") + return w.Flush() + } + + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", + types.FIL(ask.PricePerByte), + types.FIL(ask.UnsealPrice), + units.BytesSize(float64(ask.PaymentInterval)), + units.BytesSize(float64(ask.PaymentIntervalIncrease)), + ) + return w.Flush() + + }, +} From 2aef7f7c07445d64b0b4c7b9bf7b7b10898531f7 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Wed, 16 Sep 2020 13:56:34 -0700 Subject: [PATCH 25/54] Clarify help text units --- cmd/lotus-storage-miner/retrieval-deals.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-storage-miner/retrieval-deals.go b/cmd/lotus-storage-miner/retrieval-deals.go index 0f15f19f2..08a0fea51 100644 --- a/cmd/lotus-storage-miner/retrieval-deals.go +++ b/cmd/lotus-storage-miner/retrieval-deals.go @@ -170,17 +170,17 @@ var retrievalSetAskCmd = &cli.Command{ }, &cli.StringFlag{ Name: "unseal-price", - Usage: "Set the price to unseal", + Usage: "Set the price to unseal (FIL/GiB)", }, &cli.StringFlag{ Name: "payment-interval", Usage: "Set the payment interval (in bytes) for retrieval", - DefaultText: "1Mb", + DefaultText: "1MiB", }, &cli.StringFlag{ Name: "payment-interval-increase", Usage: "Set the payment interval increase (in bytes) for retrieval", - DefaultText: "1Mb", + DefaultText: "1MiB", }, }, Action: func(cctx *cli.Context) error { From f56602c5887864d88328df4374cb4a894e1c53f9 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Wed, 16 Sep 2020 14:30:04 -0700 Subject: [PATCH 26/54] Specify retrieval price in FIL/GiB --- cmd/lotus-storage-miner/retrieval-deals.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-storage-miner/retrieval-deals.go b/cmd/lotus-storage-miner/retrieval-deals.go index 08a0fea51..03d397852 100644 --- a/cmd/lotus-storage-miner/retrieval-deals.go +++ b/cmd/lotus-storage-miner/retrieval-deals.go @@ -166,11 +166,11 @@ var retrievalSetAskCmd = &cli.Command{ Flags: []cli.Flag{ &cli.StringFlag{ Name: "price", - Usage: "Set the price of the ask for retrievals (per byte)", + Usage: "Set the price of the ask for retrievals (FIL/GiB)", }, &cli.StringFlag{ Name: "unseal-price", - Usage: "Set the price to unseal (FIL/GiB)", + Usage: "Set the price to unseal", }, &cli.StringFlag{ Name: "payment-interval", @@ -202,7 +202,7 @@ var retrievalSetAskCmd = &cli.Command{ if err != nil { return err } - ask.PricePerByte = abi.TokenAmount(v) + ask.PricePerByte = types.BigDiv(types.BigInt(v), types.NewInt(1<<30)) } if cctx.IsSet("unseal-price") { From d5af25b76cba60d0aceb94ab395d5df9ebee1428 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 17 Sep 2020 10:38:07 +0800 Subject: [PATCH 27/54] update init sector state list --- cmd/lotus-storage-miner/sectors.go | 4 +- extern/storage-sealing/sector_state.go | 63 +++++++++++++------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index c08d6ca14..1b5cfb676 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -17,9 +17,9 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" - - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" + lcli "github.com/filecoin-project/lotus/cli" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" ) var sectorsCmd = &cli.Command{ diff --git a/extern/storage-sealing/sector_state.go b/extern/storage-sealing/sector_state.go index 3ed891064..10b96e504 100644 --- a/extern/storage-sealing/sector_state.go +++ b/extern/storage-sealing/sector_state.go @@ -2,7 +2,37 @@ package sealing type SectorState string -var ExistSectorStateList = make(map[SectorState]struct{}) +var ExistSectorStateList = map[SectorState]struct{}{ + Empty: {}, + WaitDeals: {}, + Packing: {}, + PreCommit1: {}, + PreCommit2: {}, + PreCommitting: {}, + PreCommitWait: {}, + WaitSeed: {}, + Committing: {}, + SubmitCommit: {}, + CommitWait: {}, + FinalizeSector: {}, + Proving: {}, + FailedUnrecoverable: {}, + SealPreCommit1Failed: {}, + SealPreCommit2Failed: {}, + PreCommitFailed: {}, + ComputeProofFailed: {}, + CommitFailed: {}, + PackingFailed: {}, + FinalizeFailed: {}, + DealsExpired: {}, + RecoverDealIDs: {}, + Faulty: {}, + FaultReported: {}, + FaultedFinal: {}, + Removing: {}, + RemoveFailed: {}, + Removed: {}, +} const ( UndefinedSectorState SectorState = "" @@ -41,37 +71,6 @@ const ( RemoveFailed SectorState = "RemoveFailed" Removed SectorState = "Removed" ) -func init() { - ExistSectorStateList[Empty] = struct{}{} - ExistSectorStateList[WaitDeals] = struct{}{} - ExistSectorStateList[Packing] = struct{}{} - ExistSectorStateList[PreCommit1] = struct{}{} - ExistSectorStateList[PreCommit2] = struct{}{} - ExistSectorStateList[PreCommitting] = struct{}{} - ExistSectorStateList[PreCommitWait] = struct{}{} - ExistSectorStateList[WaitSeed] = struct{}{} - ExistSectorStateList[Committing] = struct{}{} - ExistSectorStateList[SubmitCommit] = struct{}{} - ExistSectorStateList[CommitWait] = struct{}{} - ExistSectorStateList[FinalizeSector] = struct{}{} - ExistSectorStateList[Proving] = struct{}{} - ExistSectorStateList[FailedUnrecoverable] = struct{}{} - ExistSectorStateList[SealPreCommit1Failed] = struct{}{} - ExistSectorStateList[SealPreCommit2Failed] = struct{}{} - ExistSectorStateList[PreCommitFailed] = struct{}{} - ExistSectorStateList[ComputeProofFailed] = struct{}{} - ExistSectorStateList[CommitFailed] = struct{}{} - ExistSectorStateList[PackingFailed] = struct{}{} - ExistSectorStateList[FinalizeFailed] = struct{}{} - ExistSectorStateList[DealsExpired] = struct{}{} - ExistSectorStateList[RecoverDealIDs] = struct{}{} - ExistSectorStateList[Faulty] = struct{}{} - ExistSectorStateList[FaultReported] = struct{}{} - ExistSectorStateList[FaultedFinal] = struct{}{} - ExistSectorStateList[Removing] = struct{}{} - ExistSectorStateList[RemoveFailed] = struct{}{} - ExistSectorStateList[Removed] = struct{}{} -} func toStatState(st SectorState) statSectorState { switch st { From bf2e3baa8fd2e192cfcca15c05c66ced1bc12eb9 Mon Sep 17 00:00:00 2001 From: Frank Date: Thu, 17 Sep 2020 11:49:23 +0800 Subject: [PATCH 28/54] fix conformance gen --- conformance/chaos/cbor_gen.go | 6 +++--- conformance/chaos/gen/gen.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 2d9deec93..61e36e661 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -6,10 +6,10 @@ import ( "fmt" "io" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/exitcode" + abi "github.com/filecoin-project/go-state-types/abi" + exitcode "github.com/filecoin-project/go-state-types/exitcode" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" + xerrors "golang.org/x/xerrors" ) var _ = xerrors.Errorf diff --git a/conformance/chaos/gen/gen.go b/conformance/chaos/gen/gen.go index 496cc3d35..ea04aee21 100644 --- a/conformance/chaos/gen/gen.go +++ b/conformance/chaos/gen/gen.go @@ -7,7 +7,7 @@ import ( ) func main() { - if err := gen.WriteTupleEncodersToFile("../cbor_gen.go", "chaos", + if err := gen.WriteTupleEncodersToFile("./cbor_gen.go", "chaos", chaos.State{}, chaos.CreateActorArgs{}, chaos.ResolveAddressResponse{}, From 12e76bc1afd9a33a8e10be62c7d332e83d4852bc Mon Sep 17 00:00:00 2001 From: Marius Darila <3396463+kenshyx@users.noreply.github.com> Date: Thu, 17 Sep 2020 12:17:18 +0200 Subject: [PATCH 29/54] replace Requires with Wants Prevent lotus-miner shutdown when daemon restarts. --- scripts/lotus-miner.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lotus-miner.service b/scripts/lotus-miner.service index c079f44a9..54c48d411 100644 --- a/scripts/lotus-miner.service +++ b/scripts/lotus-miner.service @@ -2,7 +2,7 @@ Description=Lotus Miner After=network.target After=lotus-daemon.service -Requires=lotus-daemon.service +Wants=lotus-daemon.service [Service] ExecStart=/usr/local/bin/lotus-miner run From 14a4acec8cefd4e64ba7919b7b0539f49c0e9ef7 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Wed, 16 Sep 2020 17:43:55 -0400 Subject: [PATCH 30/54] Add an option to hide sectors in Removed for `sectors list`. --- cmd/lotus-storage-miner/sectors.go | 33 +++++++++++++++++++----------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 06f09fe20..0e6fe8435 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -2,6 +2,7 @@ package main import ( "fmt" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "os" "sort" "strconv" @@ -136,6 +137,12 @@ var sectorsStatusCmd = &cli.Command{ var sectorsListCmd = &cli.Command{ Name: "list", Usage: "List sectors", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "hide-removed", + Usage: "to hide Removed sectors", + }, + }, Action: func(cctx *cli.Context) error { nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { @@ -192,19 +199,21 @@ var sectorsListCmd = &cli.Command{ continue } - _, inSSet := commitedIDs[s] - _, inASet := activeIDs[s] + if !cctx.Bool("hide-removed") || st.State != api.SectorState(sealing.Removed) { + _, inSSet := commitedIDs[s] + _, inASet := activeIDs[s] - fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", - s, - st.State, - yesno(inSSet), - yesno(inASet), - st.Ticket.Epoch, - st.Seed.Epoch, - st.Deals, - st.ToUpgrade, - ) + fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", + s, + st.State, + yesno(inSSet), + yesno(inASet), + st.Ticket.Epoch, + st.Seed.Epoch, + st.Deals, + st.ToUpgrade, + ) + } } return w.Flush() From b85b50afbcc8ad24fda710369fbcd3ad4a001c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 17 Sep 2020 19:57:42 +0200 Subject: [PATCH 31/54] shed: bitfield intersect command --- cmd/lotus-shed/bitfield.go | 115 +++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 31 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 79ce214ee..78f6f02fe 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -28,6 +28,7 @@ var bitFieldCmd = &cli.Command{ bitFieldRunsCmd, bitFieldStatCmd, bitFieldDecodeCmd, + bitFieldIntersectCmd, }, } @@ -200,38 +201,9 @@ var bitFieldDecodeCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - var val string - if cctx.Args().Present() { - val = cctx.Args().Get(0) - } else { - b, err := ioutil.ReadAll(os.Stdin) - if err != nil { - return err - } - val = string(b) - } - - var dec []byte - switch cctx.String("enc") { - case "base64": - d, err := base64.StdEncoding.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding base64 value: %w", err) - } - dec = d - case "hex": - d, err := hex.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding hex value: %w", err) - } - dec = d - default: - return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) - } - - rle, err := bitfield.NewFromBytes(dec) + rle, err := decode(cctx, 0) if err != nil { - return xerrors.Errorf("failed to parse bitfield: %w", err) + return err } vals, err := rle.All(100000000000) @@ -243,3 +215,84 @@ var bitFieldDecodeCmd = &cli.Command{ return nil }, } + +var bitFieldIntersectCmd = &cli.Command{ + Name: "intersect", + Description: "intersect 2 bitfields and print the resulting bitfield as base64", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + b, err := decode(cctx, 1) + if err != nil { + return err + } + + a, err := decode(cctx, 0) + if err != nil { + return err + } + + o, err := bitfield.IntersectBitField(a, b) + if err != nil { + return xerrors.Errorf("intersect: %w", err) + } + + s, err := o.RunIterator() + if err != nil { + return err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + + return nil + }, +} + +func decode(cctx *cli.Context, a int) (bitfield.BitField, error) { + var val string + if cctx.Args().Present() { + if a >= cctx.NArg() { + return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a) + } + val = cctx.Args().Get(a) + } else { + if a > 0 { + return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a) + } + b, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return bitfield.BitField{}, err + } + val = string(b) + } + + var dec []byte + switch cctx.String("enc") { + case "base64": + d, err := base64.StdEncoding.DecodeString(val) + if err != nil { + return bitfield.BitField{}, fmt.Errorf("decoding base64 value: %w", err) + } + dec = d + case "hex": + d, err := hex.DecodeString(val) + if err != nil { + return bitfield.BitField{}, fmt.Errorf("decoding hex value: %w", err) + } + dec = d + default: + return bitfield.BitField{}, fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) + } + + return bitfield.NewFromBytes(dec) +} From a894c1e8e626ca933b72a3864026ca05bef6a74c Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 17 Sep 2020 12:30:26 -0700 Subject: [PATCH 32/54] feat(markets): upgrade markets v0.6.1 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 65c2addb8..41bc727b3 100644 --- a/go.mod +++ b/go.mod @@ -25,9 +25,9 @@ require ( github.com/filecoin-project/go-bitfield v0.2.0 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.3 + github.com/filecoin-project/go-data-transfer v0.6.4 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c + github.com/filecoin-project/go-fil-markets v0.6.1 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index 77b556fb2..11d607034 100644 --- a/go.sum +++ b/go.sum @@ -222,12 +222,12 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.3 h1:7TLwm8nuodHYD/uiwJjKc/PGRR+LwqM8jmlZqgWuUfY= -github.com/filecoin-project/go-data-transfer v0.6.3/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM= +github.com/filecoin-project/go-data-transfer v0.6.4 h1:Q08ABa+cOTOLoAyHeA94fPLcwu53p6eeAaxMxQb0m0A= +github.com/filecoin-project/go-data-transfer v0.6.4/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c h1:YGoyYmELQ0LHwDj/WcOvY3oYt+3iM0wdrAhqJQUAIy4= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A= +github.com/filecoin-project/go-fil-markets v0.6.1 h1:qCFLcVkUCbxwEfH/6EcqTuQvibXt/TXZr+vh8tWv/BQ= +github.com/filecoin-project/go-fil-markets v0.6.1/go.mod h1:dBJl59dAyl8+cGVb/ONPlEQW4+YzhjI3d6bxLfHVpX0= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0= From dcc56434732783a400a679c3adcd59b295586a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 02:38:30 +0200 Subject: [PATCH 33/54] shed: Even more bitfield utils --- cmd/lotus-shed/bitfield.go | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 78f6f02fe..e1e06573c 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "io" "io/ioutil" "os" @@ -29,6 +30,8 @@ var bitFieldCmd = &cli.Command{ bitFieldStatCmd, bitFieldDecodeCmd, bitFieldIntersectCmd, + bitFieldEncodeCmd, + bitFieldSubCmd, }, } @@ -258,6 +261,93 @@ var bitFieldIntersectCmd = &cli.Command{ }, } + +var bitFieldSubCmd = &cli.Command{ + Name: "sub", + Description: "subtract 2 bitfields and print the resulting bitfield as base64", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + b, err := decode(cctx, 1) + if err != nil { + return err + } + + a, err := decode(cctx, 0) + if err != nil { + return err + } + + o, err := bitfield.SubtractBitField(a, b) + if err != nil { + return xerrors.Errorf("intersect: %w", err) + } + + s, err := o.RunIterator() + if err != nil { + return err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + + return nil + }, +} + +var bitFieldEncodeCmd = &cli.Command{ + Name: "encode", + Description: "encode a series of decimal numbers into a bitfield", + ArgsUsage: "[infile]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + f, err := os.Open(cctx.Args().First()) + if err != nil { + return err + } + defer f.Close() + + out := bitfield.New() + for { + var i uint64 + _, err := fmt.Fscan(f, &i) + if err == io.EOF { + break + } + out.Set(i) + } + + s, err := out.RunIterator() + if err != nil { + return err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + + return nil + }, +} + func decode(cctx *cli.Context, a int) (bitfield.BitField, error) { var val string if cctx.Args().Present() { From 8419dda84b1cbfa194a15a50529a5cc4cf862546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 02:39:08 +0200 Subject: [PATCH 34/54] wdpost: Only add proven partitions to message params --- storage/wdpost_run.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 17b567440..532d7144b 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -431,7 +431,9 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty postSkipped := bitfield.New() var postOut []proof.PoStProof somethingToProve := true + for retries := 0; retries < 5; retries++ { + var partitions []miner.PoStPartition var sinfos []proof.SectorInfo for partIdx, partition := range batch { // TODO: Can do this in parallel @@ -477,7 +479,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } sinfos = append(sinfos, ssi...) - params.Partitions = append(params.Partitions, miner.PoStPartition{ + partitions = append(params.Partitions, miner.PoStPartition{ Index: uint64(batchPartitionStartIdx + partIdx), Skipped: skipped, }) @@ -511,6 +513,8 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty if err == nil { // Proof generation successful, stop retrying + params.Partitions = append(params.Partitions, partitions...) + break } From 57c1eac37a29a1c39b85e4d245fdd1b0632f5069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 02:40:01 +0200 Subject: [PATCH 35/54] gofmt --- cmd/lotus-shed/bitfield.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index e1e06573c..9cc632283 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -261,7 +261,6 @@ var bitFieldIntersectCmd = &cli.Command{ }, } - var bitFieldSubCmd = &cli.Command{ Name: "sub", Description: "subtract 2 bitfields and print the resulting bitfield as base64", @@ -307,7 +306,7 @@ var bitFieldSubCmd = &cli.Command{ var bitFieldEncodeCmd = &cli.Command{ Name: "encode", Description: "encode a series of decimal numbers into a bitfield", - ArgsUsage: "[infile]", + ArgsUsage: "[infile]", Flags: []cli.Flag{ &cli.StringFlag{ Name: "enc", From 26bbd9fb9d13207ed6f32d769bfb8ebfb548f795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Sep 2020 03:00:31 +0200 Subject: [PATCH 36/54] wdpost: append is hard --- cmd/lotus-shed/bitfield.go | 2 +- storage/wdpost_run.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 9cc632283..442cbef48 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -319,7 +319,7 @@ var bitFieldEncodeCmd = &cli.Command{ if err != nil { return err } - defer f.Close() + defer f.Close() // nolint out := bitfield.New() for { diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 532d7144b..463c7a3c5 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -479,7 +479,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } sinfos = append(sinfos, ssi...) - partitions = append(params.Partitions, miner.PoStPartition{ + partitions = append(partitions, miner.PoStPartition{ Index: uint64(batchPartitionStartIdx + partIdx), Skipped: skipped, }) From aa081dbc6d018f3bbfcaf25c2374b4917fb2de1c Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 17 Sep 2020 15:17:25 -0400 Subject: [PATCH 37/54] Lotus version 0.7.1 --- CHANGELOG.md | 24 ++++++++++++++++++++++++ build/version.go | 4 ++-- documentation/en/api-methods.md | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8617554a9..06c0959b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Lotus changelog +# 0.7.1 / 2020-09-17 + +This optional release of Lotus introduces some critical fixes to the window PoSt process. It also upgrades some core dependencies, and introduces many improvements to the mining process, deal-making cycle, and overall User Experience. + +## Changes + +#### Some notable improvements: + +- Correctly construct params for `SubmitWindowedPoSt` messages (https://github.com/filecoin-project/lotus/pull/3909) +- Skip sectors correctly for Window PoSt (https://github.com/filecoin-project/lotus/pull/3839) +- Split window PoST submission into multiple messages (https://github.com/filecoin-project/lotus/pull/3689) +- Improve journal coverage (https://github.com/filecoin-project/lotus/pull/2455) +- Allow retrievals while sealing (https://github.com/filecoin-project/lotus/pull/3778) +- Don't prune locally published messages (https://github.com/filecoin-project/lotus/pull/3772) +- Add get-ask, set-ask retrieval commands (https://github.com/filecoin-project/lotus/pull/3886) +- Consistently name winning and window post in logs (https://github.com/filecoin-project/lotus/pull/3873)) +- Add auto flag to mpool replace (https://github.com/filecoin-project/lotus/pull/3752)) + +#### Dependencies + +- Upgrade markets to `v0.6.1` (https://github.com/filecoin-project/lotus/pull/3906) +- Upgrade specs-actors to `v0.9.10` (https://github.com/filecoin-project/lotus/pull/3846) +- Upgrade badger (https://github.com/filecoin-project/lotus/pull/3739) + # 0.7.0 / 2020-09-10 This consensus-breaking release of Lotus is designed to test a network upgrade on the space race testnet. The changes that break consensus are: diff --git a/build/version.go b/build/version.go index 2466d8023..933cfca70 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.7.0" +const BuildVersion = "0.7.1" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit @@ -83,7 +83,7 @@ func VersionForType(nodeType NodeType) (Version, error) { // semver versions of the rpc api exposed var ( - FullAPIVersion = newVer(0, 14, 0) + FullAPIVersion = newVer(0, 15, 0) MinerAPIVersion = newVer(0, 14, 0) WorkerAPIVersion = newVer(0, 14, 0) ) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 2f3164bb7..84d72ab67 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -212,7 +212,7 @@ Response: ```json { "Version": "string value", - "APIVersion": 3584, + "APIVersion": 3840, "BlockDelay": 42 } ``` From db2a20da6cb8343a2a8c7a45f3d6e91b48928c01 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Thu, 17 Sep 2020 23:09:12 +0000 Subject: [PATCH 38/54] lotus-shed: add keyinfo verify --- cmd/lotus-shed/keyinfo.go | 90 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index f397bd4c7..6090b7a0f 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -9,16 +9,22 @@ import ( "io" "io/ioutil" "os" + "path" "strings" "text/template" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/multiformats/go-base32" + "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/lp2p" "github.com/filecoin-project/lotus/node/repo" @@ -43,6 +49,90 @@ var keyinfoCmd = &cli.Command{ keyinfoNewCmd, keyinfoInfoCmd, keyinfoImportCmd, + keyinfoVerifyCmd, + }, +} + +var keyinfoVerifyCmd = &cli.Command{ + Name: "verify", + Usage: "verify the filename of a keystore object on disk with it's contents", + Description: `Keystore objects are base32 enocded strings, with wallets being dynamically named via + the wallet address. This command can ensure that the naming of these keystore objects are correct`, + Action: func(cctx *cli.Context) error { + filePath := cctx.Args().First() + fileName := path.Base(filePath) + + inputFile, err := os.Open(filePath) + if err != nil { + return err + } + defer inputFile.Close() //nolint:errcheck + input := bufio.NewReader(inputFile) + + keyContent, err := ioutil.ReadAll(input) + if err != nil { + return err + } + + var keyInfo types.KeyInfo + if err := json.Unmarshal(keyContent, &keyInfo); err != nil { + return err + } + + switch keyInfo.Type { + case lp2p.KTLibp2pHost: + name, err := base32.RawStdEncoding.DecodeString(fileName) + if err != nil { + return xerrors.Errorf("decoding key: '%s': %w", fileName, err) + } + + if string(name) != keyInfo.Type { + return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type) + } + case modules.KTJwtHmacSecret: + name, err := base32.RawStdEncoding.DecodeString(fileName) + if err != nil { + return xerrors.Errorf("decoding key: '%s': %w", fileName, err) + } + + if string(name) != modules.JWTSecretName { + return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type) + } + case wallet.KTSecp256k1, wallet.KTBLS: + keystore := wallet.NewMemKeyStore() + w, err := wallet.NewWallet(keystore) + if err != nil { + return err + } + + if _, err := w.Import(&keyInfo); err != nil { + return err + } + + list, err := keystore.List() + if err != nil { + return err + } + + if len(list) != 1 { + return fmt.Errorf("Unexpected number of keys, expected 1, found %d", len(list)) + } + + name, err := base32.RawStdEncoding.DecodeString(fileName) + if err != nil { + return xerrors.Errorf("decoding key: '%s': %w", fileName, err) + } + + if string(name) != list[0] { + return fmt.Errorf("%s of type %s; file is named for %s, but key is actually %s", fileName, keyInfo.Type, string(name), list[0]) + } + + break + default: + return fmt.Errorf("Unknown keytype %s", keyInfo.Type) + } + + return nil }, } From ac7007d1d010d31a970b5f4706045976e43ab796 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Fri, 18 Sep 2020 00:40:46 +0000 Subject: [PATCH 39/54] lotus-shed: add jwt token command --- cmd/lotus-shed/jwt.go | 99 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/cmd/lotus-shed/jwt.go b/cmd/lotus-shed/jwt.go index d37359f71..7fa1a18dd 100644 --- a/cmd/lotus-shed/jwt.go +++ b/cmd/lotus-shed/jwt.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "crypto/rand" "encoding/hex" "encoding/json" @@ -8,10 +9,12 @@ import ( "io" "io/ioutil" "os" + "strings" "github.com/gbrlsnchs/jwt/v3" "github.com/urfave/cli/v2" + "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules" @@ -24,6 +27,102 @@ var jwtCmd = &cli.Command{ having to run the lotus daemon.`, Subcommands: []*cli.Command{ jwtNewCmd, + jwtTokenCmd, + }, +} + +var jwtTokenCmd = &cli.Command{ + Name: "token", + Usage: "create a token for a given jwt secret", + ArgsUsage: "", + Description: `The jwt tokens have four different levels of permissions that provide some ability + to control access to what methods can be invoked by the holder of the token. + + This command only works on jwt secrets that are base16 encoded files, such as those produced by the + sibling 'new' command. + `, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "output", + Value: "token", + Usage: "specify a name", + }, + &cli.BoolFlag{ + Name: "read", + Value: false, + Usage: "add read permissions to the token", + }, + &cli.BoolFlag{ + Name: "write", + Value: false, + Usage: "add write permissions to the token", + }, + &cli.BoolFlag{ + Name: "sign", + Value: false, + Usage: "add sign permissions to the token", + }, + &cli.BoolFlag{ + Name: "admin", + Value: false, + Usage: "add admin permissions to the token", + }, + }, + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return fmt.Errorf("please specify a name") + } + + inputFile, err := os.Open(cctx.Args().First()) + if err != nil { + return err + } + defer inputFile.Close() //nolint:errcheck + input := bufio.NewReader(inputFile) + + encoded, err := ioutil.ReadAll(input) + if err != nil { + return err + } + + decoded, err := hex.DecodeString(strings.TrimSpace(string(encoded))) + if err != nil { + return err + } + + var keyInfo types.KeyInfo + if err := json.Unmarshal(decoded, &keyInfo); err != nil { + return err + } + + perms := []auth.Permission{} + + if cctx.Bool("read") { + perms = append(perms, apistruct.PermRead) + } + + if cctx.Bool("write") { + perms = append(perms, apistruct.PermWrite) + } + + if cctx.Bool("sign") { + perms = append(perms, apistruct.PermSign) + } + + if cctx.Bool("admin") { + perms = append(perms, apistruct.PermAdmin) + } + + p := modules.JwtPayload{ + Allow: perms, + } + + token, err := jwt.Sign(&p, jwt.NewHS256(keyInfo.PrivateKey)) + if err != nil { + return err + } + + return ioutil.WriteFile(cctx.String("output"), token, 0600) }, } From 37108ebf8fa9f5fc6f78e9fe4b50ee1d72b4e17a Mon Sep 17 00:00:00 2001 From: Jennifer <42981373+jennijuju@users.noreply.github.com> Date: Fri, 18 Sep 2020 04:36:27 -0400 Subject: [PATCH 40/54] Update build and setup instruction link Update the link since we deprecated lotus.sh and migrates it to doc.Filecoin --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6c1e23efa..766317e4f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more ## Building & Documentation -For instructions on how to build lotus from source, please visit [https://lotu.sh](https://lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation). +For instructions on how to build lotus from source, please visit [Lotus build and setup instruction](https://docs.filecoin.io/get-started/lotus/installation/#minimal-requirements) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation). ## Reporting a Vulnerability From eadc61c37a81ca999ff1c2a021352a82f4df067a Mon Sep 17 00:00:00 2001 From: Travis Person Date: Fri, 18 Sep 2020 18:07:58 +0000 Subject: [PATCH 41/54] log shutdown method for lotus daemon and miner --- cmd/lotus-storage-miner/run.go | 4 +++- cmd/lotus/rpc.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 83d78172e..a5d996f78 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -163,8 +163,10 @@ var runCmd = &cli.Command{ sigChan := make(chan os.Signal, 2) go func() { select { - case <-sigChan: + case sig := <-sigChan: + log.Warnw("received shutdown", "signal", sig) case <-shutdownChan: + log.Warn("received shutdown") } log.Warn("Shutting down...") diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index fbe05e938..9718deb3a 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -66,8 +66,10 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr, shut shutdownDone := make(chan struct{}) go func() { select { - case <-sigCh: + case sig := <-sigCh: + log.Warnw("received shutdown", "signal", sig) case <-shutdownCh: + log.Warn("received shutdown") } log.Warn("Shutting down...") From 6b40e1e30dabc87ddecfa84861b77aaf0046100e Mon Sep 17 00:00:00 2001 From: shepf Date: Sat, 19 Sep 2020 14:04:48 +0800 Subject: [PATCH 42/54] sync wait add heightDiff maybe useful! --- cli/sync.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cli/sync.go b/cli/sync.go index bff34960e..3b4e2e9fb 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -249,14 +249,24 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { ss := state.ActiveSyncs[working] + var baseHeight abi.ChainEpoch var target []cid.Cid var theight abi.ChainEpoch + var heightDiff int64 + + if ss.Base != nil { + baseHeight = ss.Base.Height() + heightDiff = int64(ss.Base.Height()) + } if ss.Target != nil { target = ss.Target.Cids() theight = ss.Target.Height() + heightDiff = int64(ss.Target.Height()) - heightDiff + } else { + heightDiff = 0 } - fmt.Printf("\r\x1b[2KWorker %d: Target Height: %d\tTarget: %s\tState: %s\tHeight: %d", working, theight, target, ss.Stage, ss.Height) + fmt.Printf("\r\x1b[2KWorker %d: Base Height: %d\tTarget Height: %d\t Height diff: %d\tTarget: %s\tState: %s\tHeight: %d", working, baseHeight, theight, heightDiff, target, ss.Stage, ss.Height) if time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) { fmt.Println("\nDone!") From c401d2ec4ec58dd7ddf5e12598c04dcdcd8bf309 Mon Sep 17 00:00:00 2001 From: Jerry <1032246642@qq.com> Date: Sat, 19 Sep 2020 15:48:02 +0800 Subject: [PATCH 43/54] fix storage find error --- cmd/lotus-storage-miner/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 71d88de6d..8a3687877 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -370,7 +370,7 @@ var storageFindCmd = &cli.Command{ } fmt.Printf("In %s (%s)\n", info.id, types[:len(types)-2]) - fmt.Printf("\tSealing: %t; Storage: %t\n", info.store.CanSeal, info.store.CanSeal) + fmt.Printf("\tSealing: %t; Storage: %t\n", info.store.CanSeal, info.store.CanStore) if localPath, ok := local[info.id]; ok { fmt.Printf("\tLocal (%s)\n", localPath) } else { From b3d0a5fb4a240500b8088dcbc068737fa27c3323 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Sun, 20 Sep 2020 20:16:20 +0000 Subject: [PATCH 44/54] lotus-shed: add consensus check command --- cmd/lotus-shed/consensus.go | 286 ++++++++++++++++++++++++++++++++++++ cmd/lotus-shed/main.go | 8 + 2 files changed, 294 insertions(+) create mode 100644 cmd/lotus-shed/consensus.go diff --git a/cmd/lotus-shed/consensus.go b/cmd/lotus-shed/consensus.go new file mode 100644 index 000000000..59d9555df --- /dev/null +++ b/cmd/lotus-shed/consensus.go @@ -0,0 +1,286 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "os" + "strconv" + "strings" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/multiformats/go-multiaddr" + "github.com/urfave/cli/v2" +) + +var consensusCmd = &cli.Command{ + Name: "consensus", + Usage: "tools for gathering information about consensus between nodes", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + consensusCheckCmd, + }, +} + +type consensusItem struct { + multiaddr multiaddr.Multiaddr + genesisTipset *types.TipSet + targetTipset *types.TipSet + headTipset *types.TipSet + peerID peer.ID + version api.Version + api api.FullNode +} + +var consensusCheckCmd = &cli.Command{ + Name: "check", + Usage: "verify if all nodes agree upon a common tipset for a given tipset height", + Description: `Consensus check verifies that all nodes share a common tipset for a given + height. + + The height flag specifies a chain height to start a comparison from. There are two special + arguments for this flag. All other expected values should be chain tipset heights. + + @common - Use the maximum common chain height between all nodes + @expected - Use the current time and the genesis timestamp to determine a height + + Examples + + Find the highest common tipset and look back 10 tipsets + lotus-shed consensus check --height @common --lookback 10 + + Calculate the expected tipset height and look back 10 tipsets + lotus-shed consensus check --height @expected --lookback 10 + + Check if nodes all share a common genesis + lotus-shed consensus check --height 0 + + Check that all nodes agree upon the tipset for 1day post genesis + lotus-shed consensus check --height 2880 --lookback 0 + `, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "height", + Value: "@common", + Usage: "height of tipset to start check from", + }, + &cli.IntFlag{ + Name: "lookback", + Value: int(build.MessageConfidence * 2), + Usage: "number of tipsets behind to look back when comparing nodes", + }, + }, + Action: func(cctx *cli.Context) error { + filePath := cctx.Args().First() + + var input *bufio.Reader + if cctx.Args().Len() == 0 { + input = bufio.NewReader(os.Stdin) + } else { + var err error + inputFile, err := os.Open(filePath) + if err != nil { + return err + } + defer inputFile.Close() //nolint:errcheck + input = bufio.NewReader(inputFile) + } + + var nodes []*consensusItem + ctx := lcli.ReqContext(cctx) + + for { + strma, errR := input.ReadString('\n') + strma = strings.TrimSpace(strma) + + if len(strma) == 0 { + if errR == io.EOF { + break + } + continue + } + + apima, err := multiaddr.NewMultiaddr(strma) + if err != nil { + return err + } + ainfo := lcli.APIInfo{Addr: apima} + addr, err := ainfo.DialArgs() + if err != nil { + return err + } + + api, closer, err := client.NewFullNodeRPC(cctx.Context, addr, nil) + if err != nil { + return err + } + defer closer() + + peerID, err := api.ID(ctx) + if err != nil { + return err + } + + version, err := api.Version(ctx) + if err != nil { + return err + } + + genesisTipset, err := api.ChainGetGenesis(ctx) + if err != nil { + return err + } + + headTipset, err := api.ChainHead(ctx) + if err != nil { + return err + } + + nodes = append(nodes, &consensusItem{ + genesisTipset: genesisTipset, + headTipset: headTipset, + multiaddr: apima, + api: api, + peerID: peerID, + version: version, + }) + + if errR != nil && errR != io.EOF { + return err + } + + if errR == io.EOF { + break + } + } + + if len(nodes) == 0 { + return fmt.Errorf("no nodes") + } + + genesisBuckets := make(map[types.TipSetKey][]*consensusItem) + for _, node := range nodes { + genesisBuckets[node.genesisTipset.Key()] = append(genesisBuckets[node.genesisTipset.Key()], node) + + } + + if len(genesisBuckets) != 1 { + for _, nodes := range genesisBuckets { + for _, node := range nodes { + log.Errorw( + "genesis do not match", + "genesis_tipset", node.genesisTipset.Key(), + "peer_id", node.peerID, + "version", node.version, + ) + } + } + + return fmt.Errorf("genesis does not match between all nodes") + } + + target := abi.ChainEpoch(0) + + switch cctx.String("height") { + case "@common": + minTipset := nodes[0].headTipset + for _, node := range nodes { + if node.headTipset.Height() < minTipset.Height() { + minTipset = node.headTipset + } + } + + target = minTipset.Height() + case "@expected": + tnow := uint64(time.Now().Unix()) + tgen := nodes[0].genesisTipset.MinTimestamp() + + target = abi.ChainEpoch((tnow - tgen) / build.BlockDelaySecs) + default: + h, err := strconv.Atoi(strings.TrimSpace(cctx.String("height"))) + if err != nil { + return fmt.Errorf("failed to parse string: %s", cctx.String("height")) + } + + target = abi.ChainEpoch(h) + } + + lookback := abi.ChainEpoch(cctx.Int("lookback")) + if lookback > target { + target = abi.ChainEpoch(0) + } else { + target = target - lookback + } + + for _, node := range nodes { + targetTipset, err := node.api.ChainGetTipSetByHeight(ctx, target, types.EmptyTSK) + if err != nil { + log.Errorw("error checking target", "err", err) + node.targetTipset = nil + } else { + node.targetTipset = targetTipset + } + + } + for _, node := range nodes { + log.Debugw( + "node info", + "peer_id", node.peerID, + "version", node.version, + "genesis_tipset", node.genesisTipset.Key(), + "head_tipset", node.headTipset.Key(), + "target_tipset", node.targetTipset.Key(), + ) + } + + targetBuckets := make(map[types.TipSetKey][]*consensusItem) + for _, node := range nodes { + if node.targetTipset == nil { + targetBuckets[types.EmptyTSK] = append(targetBuckets[types.EmptyTSK], node) + continue + } + + targetBuckets[node.targetTipset.Key()] = append(targetBuckets[node.targetTipset.Key()], node) + } + + if nodes, ok := targetBuckets[types.EmptyTSK]; ok { + for _, node := range nodes { + log.Errorw( + "targeted tipset not found", + "peer_id", node.peerID, + "version", node.version, + "genesis_tipset", node.genesisTipset.Key(), + "head_tipset", node.headTipset.Key(), + "target_tipset", node.targetTipset.Key(), + ) + } + + return fmt.Errorf("targeted tipset not found") + } + + if len(targetBuckets) != 1 { + for _, nodes := range targetBuckets { + for _, node := range nodes { + log.Errorw( + "targeted tipset not found", + "peer_id", node.peerID, + "version", node.version, + "genesis_tipset", node.genesisTipset.Key(), + "head_tipset", node.headTipset.Key(), + "target_tipset", node.targetTipset.Key(), + ) + } + } + return fmt.Errorf("nodes not in consensus at tipset height %d", target) + } + + return nil + }, +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index cff3059b6..1a56756d1 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -35,6 +35,7 @@ func main() { mathCmd, mpoolStatsCmd, exportChainCmd, + consensusCmd, } app := &cli.App{ @@ -49,6 +50,13 @@ func main() { Hidden: true, Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME }, + &cli.StringFlag{ + Name: "log-level", + Value: "info", + }, + }, + Before: func(cctx *cli.Context) error { + return logging.SetLogLevel("lotus-shed", cctx.String("log-level")) }, } From cfa041ca8ad7d54c1faf7e942ce70ef3e1313068 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 14 Sep 2020 12:41:00 +0100 Subject: [PATCH 45/54] refactor: chaos caller validation --- conformance/chaos/actor.go | 34 +++++--- conformance/chaos/cbor_gen.go | 159 ++++++++++++++++++++++++++++++++++ conformance/chaos/gen/gen.go | 1 + 3 files changed, 181 insertions(+), 13 deletions(-) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index 005a06f0d..bc68b086f 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -7,8 +7,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" - - typegen "github.com/whyrusleeping/cbor-gen" ) //go:generate go run ./gen @@ -31,10 +29,14 @@ type Actor struct{} type CallerValidationBranch int64 const ( + // CallerValidationBranchNone causes no caller validation to take place. CallerValidationBranchNone CallerValidationBranch = iota + // CallerValidationBranchTwice causes Runtime.ValidateImmediateCallerAcceptAny to be called twice. CallerValidationBranchTwice - CallerValidationBranchAddrNilSet - CallerValidationBranchTypeNilSet + // CallerValidationBranchIs causes caller validation against CallerValidationArgs.Addrs. + CallerValidationBranchIs + // CallerValidationBranchType causes caller validation against CallerValidationArgs.Types. + CallerValidationBranchType ) // MutateStateBranch is an enum used to select the type of state mutation to attempt. @@ -123,23 +125,29 @@ func (a Actor) Constructor(_ runtime.Runtime, _ *abi.EmptyValue) *abi.EmptyValue panic("constructor should not be called; the Chaos actor is a singleton actor") } +// CallerValidationArgs are the arguments to Actor.CallerValidation. +type CallerValidationArgs struct { + Branch CallerValidationBranch + Addrs []address.Address + Types []cid.Cid +} + // CallerValidation violates VM call validation constraints. // // CallerValidationBranchNone performs no validation. // CallerValidationBranchTwice validates twice. -// CallerValidationBranchAddrNilSet validates against an empty caller -// address set. -// CallerValidationBranchTypeNilSet validates against an empty caller type set. -func (a Actor) CallerValidation(rt runtime.Runtime, branch *typegen.CborInt) *abi.EmptyValue { - switch CallerValidationBranch(*branch) { +// CallerValidationBranchIs validates caller against CallerValidationArgs.Addrs. +// CallerValidationBranchType validates caller against CallerValidationArgs.Types. +func (a Actor) CallerValidation(rt runtime.Runtime, args *CallerValidationArgs) *abi.EmptyValue { + switch args.Branch { case CallerValidationBranchNone: case CallerValidationBranchTwice: rt.ValidateImmediateCallerAcceptAny() rt.ValidateImmediateCallerAcceptAny() - case CallerValidationBranchAddrNilSet: - rt.ValidateImmediateCallerIs() - case CallerValidationBranchTypeNilSet: - rt.ValidateImmediateCallerType() + case CallerValidationBranchIs: + rt.ValidateImmediateCallerIs(args.Addrs...) + case CallerValidationBranchType: + rt.ValidateImmediateCallerType(args.Types...) default: panic("invalid branch passed to CallerValidation") } diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 61e36e661..882af7026 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -6,8 +6,10 @@ import ( "fmt" "io" + address "github.com/filecoin-project/go-address" abi "github.com/filecoin-project/go-state-types/abi" exitcode "github.com/filecoin-project/go-state-types/exitcode" + cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -115,6 +117,163 @@ func (t *State) UnmarshalCBOR(r io.Reader) error { return nil } +var lengthBufCallerValidationArgs = []byte{131} + +func (t *CallerValidationArgs) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufCallerValidationArgs); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.Branch (chaos.CallerValidationBranch) (int64) + if t.Branch >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Branch)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Branch-1)); err != nil { + return err + } + } + + // t.Addrs ([]address.Address) (slice) + if len(t.Addrs) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Addrs was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Addrs))); err != nil { + return err + } + for _, v := range t.Addrs { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.Types ([]cid.Cid) (slice) + if len(t.Types) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Types was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Types))); err != nil { + return err + } + for _, v := range t.Types { + if err := cbg.WriteCidBuf(scratch, w, v); err != nil { + return xerrors.Errorf("failed writing cid field t.Types: %w", err) + } + } + return nil +} + +func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error { + *t = CallerValidationArgs{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.Branch (chaos.CallerValidationBranch) (int64) + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Branch = CallerValidationBranch(extraI) + } + // t.Addrs ([]address.Address) (slice) + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Addrs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Addrs = make([]address.Address, extra) + } + + for i := 0; i < int(extra); i++ { + + var v address.Address + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Addrs[i] = v + } + + // t.Types ([]cid.Cid) (slice) + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Types: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Types = make([]cid.Cid, extra) + } + + for i := 0; i < int(extra); i++ { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("reading cid field t.Types failed: %w", err) + } + t.Types[i] = c + } + + return nil +} + var lengthBufCreateActorArgs = []byte{132} func (t *CreateActorArgs) MarshalCBOR(w io.Writer) error { diff --git a/conformance/chaos/gen/gen.go b/conformance/chaos/gen/gen.go index ea04aee21..ac97da99e 100644 --- a/conformance/chaos/gen/gen.go +++ b/conformance/chaos/gen/gen.go @@ -9,6 +9,7 @@ import ( func main() { if err := gen.WriteTupleEncodersToFile("./cbor_gen.go", "chaos", chaos.State{}, + chaos.CallerValidationArgs{}, chaos.CreateActorArgs{}, chaos.ResolveAddressResponse{}, chaos.SendArgs{}, From ab53abf04eaee92973daade2d0f55a44b2ca972a Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 14:00:33 +0100 Subject: [PATCH 46/54] feat: add CallerValidationBranchIsReceiver and tests --- conformance/chaos/actor.go | 4 ++ conformance/chaos/actor_test.go | 119 ++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index bc68b086f..c56abcb8b 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -35,6 +35,8 @@ const ( CallerValidationBranchTwice // CallerValidationBranchIs causes caller validation against CallerValidationArgs.Addrs. CallerValidationBranchIs + // CallerValidationBranchIsReceiver causes validation that the caller was also the receiver. + CallerValidationBranchIsReceiver // CallerValidationBranchType causes caller validation against CallerValidationArgs.Types. CallerValidationBranchType ) @@ -146,6 +148,8 @@ func (a Actor) CallerValidation(rt runtime.Runtime, args *CallerValidationArgs) rt.ValidateImmediateCallerAcceptAny() case CallerValidationBranchIs: rt.ValidateImmediateCallerIs(args.Addrs...) + case CallerValidationBranchIsReceiver: + rt.ValidateImmediateCallerIs(rt.Message().Receiver()) case CallerValidationBranchType: rt.ValidateImmediateCallerType(args.Types...) default: diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index 2e899980d..c9d804309 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -2,6 +2,9 @@ package chaos import ( "context" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/ipfs/go-cid" "testing" "github.com/filecoin-project/go-state-types/abi" @@ -25,6 +28,108 @@ func TestSingleton(t *testing.T) { rt.Verify() } +func TestCallerValidationNone(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + var a Actor + + rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: CallerValidationBranchNone}) + rt.Verify() +} + +func TestCallerValidationIs(t *testing.T) { + caller := atesting.NewIDAddr(t, 100) + receiver := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + rt.SetCaller(caller, builtin.AccountActorCodeID) + var a Actor + + caddrs := []address.Address{atesting.NewIDAddr(t, 101)} + + rt.ExpectValidateCallerAddr(caddrs...) + // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 + rt.ExpectAbort(exitcode.ErrForbidden, func() { + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchIs, + Addrs: caddrs, + }) + }) + rt.Verify() + + rt.ExpectValidateCallerAddr(caller) + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchIs, + Addrs: []address.Address{caller}, + }) + rt.Verify() +} + +func TestCallerValidationIsReceiver(t *testing.T) { + caller := atesting.NewIDAddr(t, 100) + receiver := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + var a Actor + + rt.SetCaller(caller, builtin.AccountActorCodeID) + rt.ExpectValidateCallerAddr(receiver) + // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 + rt.ExpectAbort(exitcode.ErrForbidden, func() { + rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: CallerValidationBranchIsReceiver}) + }) + rt.Verify() + + rt.SetCaller(receiver, builtin.AccountActorCodeID) + rt.ExpectValidateCallerAddr(receiver) + rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: CallerValidationBranchIsReceiver}) + rt.Verify() +} + +func TestCallerValidationType(t *testing.T) { + caller := atesting.NewIDAddr(t, 100) + receiver := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + rt.SetCaller(caller, builtin.AccountActorCodeID) + var a Actor + + rt.ExpectValidateCallerType(builtin.CronActorCodeID) + // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 + rt.ExpectAbort(exitcode.ErrForbidden, func() { + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchType, + Types: []cid.Cid{builtin.CronActorCodeID}, + }) + }) + rt.Verify() + + rt.ExpectValidateCallerType(builtin.AccountActorCodeID) + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchType, + Types: []cid.Cid{builtin.AccountActorCodeID}, + }) + rt.Verify() +} + +func TestCallerValidationInvalidBranch(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + var a Actor + + rt.ExpectAssertionFailure("invalid branch passed to CallerValidation", func() { + rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: -1}) + }) + rt.Verify() +} + func TestDeleteActor(t *testing.T) { receiver := atesting.NewIDAddr(t, 100) beneficiary := atesting.NewIDAddr(t, 101) @@ -118,6 +223,20 @@ func TestMutateStateReadonly(t *testing.T) { rt.Verify() } +func TestMutateStateInvalidBranch(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + var a Actor + + rt.ExpectValidateCallerAny() + rt.ExpectAssertionFailure("unknown mutation type", func() { + rt.Call(a.MutateState, &MutateStateArgs{Branch: -1}) + }) + rt.Verify() +} + func TestAbortWith(t *testing.T) { receiver := atesting.NewIDAddr(t, 100) builder := mock.NewBuilder(context.Background(), receiver) From 49a94b7cbab9b2acef1fa1765035f4c8a5ce9002 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 14:10:20 +0100 Subject: [PATCH 47/54] chore: appease linter --- conformance/chaos/actor_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index c9d804309..675c617ab 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -2,16 +2,15 @@ package chaos import ( "context" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/ipfs/go-cid" "testing" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/support/mock" atesting "github.com/filecoin-project/specs-actors/support/testing" + "github.com/ipfs/go-cid" ) func TestSingleton(t *testing.T) { From d607ca6118a9d859b99da43e1ba74251faf0d498 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 14:56:58 +0100 Subject: [PATCH 48/54] refactor: remove IsReceiver branch --- conformance/chaos/actor.go | 4 ---- conformance/chaos/actor_test.go | 22 ---------------------- 2 files changed, 26 deletions(-) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index c56abcb8b..bc68b086f 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -35,8 +35,6 @@ const ( CallerValidationBranchTwice // CallerValidationBranchIs causes caller validation against CallerValidationArgs.Addrs. CallerValidationBranchIs - // CallerValidationBranchIsReceiver causes validation that the caller was also the receiver. - CallerValidationBranchIsReceiver // CallerValidationBranchType causes caller validation against CallerValidationArgs.Types. CallerValidationBranchType ) @@ -148,8 +146,6 @@ func (a Actor) CallerValidation(rt runtime.Runtime, args *CallerValidationArgs) rt.ValidateImmediateCallerAcceptAny() case CallerValidationBranchIs: rt.ValidateImmediateCallerIs(args.Addrs...) - case CallerValidationBranchIsReceiver: - rt.ValidateImmediateCallerIs(rt.Message().Receiver()) case CallerValidationBranchType: rt.ValidateImmediateCallerType(args.Types...) default: diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index 675c617ab..a6345da53 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -67,28 +67,6 @@ func TestCallerValidationIs(t *testing.T) { rt.Verify() } -func TestCallerValidationIsReceiver(t *testing.T) { - caller := atesting.NewIDAddr(t, 100) - receiver := atesting.NewIDAddr(t, 101) - builder := mock.NewBuilder(context.Background(), receiver) - - rt := builder.Build(t) - var a Actor - - rt.SetCaller(caller, builtin.AccountActorCodeID) - rt.ExpectValidateCallerAddr(receiver) - // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 - rt.ExpectAbort(exitcode.ErrForbidden, func() { - rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: CallerValidationBranchIsReceiver}) - }) - rt.Verify() - - rt.SetCaller(receiver, builtin.AccountActorCodeID) - rt.ExpectValidateCallerAddr(receiver) - rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: CallerValidationBranchIsReceiver}) - rt.Verify() -} - func TestCallerValidationType(t *testing.T) { caller := atesting.NewIDAddr(t, 100) receiver := atesting.NewIDAddr(t, 101) From dfaad08a8b57b08b3bbde9ef633dec97bb497ce7 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 15:03:57 +0100 Subject: [PATCH 49/54] fix: update cbor gen file --- conformance/chaos/cbor_gen.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 882af7026..0d34264ee 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -6,12 +6,12 @@ import ( "fmt" "io" - address "github.com/filecoin-project/go-address" - abi "github.com/filecoin-project/go-state-types/abi" - exitcode "github.com/filecoin-project/go-state-types/exitcode" - cid "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/exitcode" + "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" + "golang.org/x/xerrors" ) var _ = xerrors.Errorf From c1a9cf7edd9a8d089e97fccc70e97b73dffc93de Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Tue, 15 Sep 2020 15:47:11 +0100 Subject: [PATCH 50/54] refactor: renames --- conformance/chaos/actor.go | 16 ++++++++-------- conformance/chaos/actor_test.go | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index bc68b086f..477204374 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -33,10 +33,10 @@ const ( CallerValidationBranchNone CallerValidationBranch = iota // CallerValidationBranchTwice causes Runtime.ValidateImmediateCallerAcceptAny to be called twice. CallerValidationBranchTwice - // CallerValidationBranchIs causes caller validation against CallerValidationArgs.Addrs. - CallerValidationBranchIs - // CallerValidationBranchType causes caller validation against CallerValidationArgs.Types. - CallerValidationBranchType + // CallerValidationBranchIsAddress causes caller validation against CallerValidationArgs.Addrs. + CallerValidationBranchIsAddress + // CallerValidationBranchIsType causes caller validation against CallerValidationArgs.Types. + CallerValidationBranchIsType ) // MutateStateBranch is an enum used to select the type of state mutation to attempt. @@ -136,17 +136,17 @@ type CallerValidationArgs struct { // // CallerValidationBranchNone performs no validation. // CallerValidationBranchTwice validates twice. -// CallerValidationBranchIs validates caller against CallerValidationArgs.Addrs. -// CallerValidationBranchType validates caller against CallerValidationArgs.Types. +// CallerValidationBranchIsAddress validates caller against CallerValidationArgs.Addrs. +// CallerValidationBranchIsType validates caller against CallerValidationArgs.Types. func (a Actor) CallerValidation(rt runtime.Runtime, args *CallerValidationArgs) *abi.EmptyValue { switch args.Branch { case CallerValidationBranchNone: case CallerValidationBranchTwice: rt.ValidateImmediateCallerAcceptAny() rt.ValidateImmediateCallerAcceptAny() - case CallerValidationBranchIs: + case CallerValidationBranchIsAddress: rt.ValidateImmediateCallerIs(args.Addrs...) - case CallerValidationBranchType: + case CallerValidationBranchIsType: rt.ValidateImmediateCallerType(args.Types...) default: panic("invalid branch passed to CallerValidation") diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index a6345da53..2061efb82 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -53,7 +53,7 @@ func TestCallerValidationIs(t *testing.T) { // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 rt.ExpectAbort(exitcode.ErrForbidden, func() { rt.Call(a.CallerValidation, &CallerValidationArgs{ - Branch: CallerValidationBranchIs, + Branch: CallerValidationBranchIsAddress, Addrs: caddrs, }) }) @@ -61,7 +61,7 @@ func TestCallerValidationIs(t *testing.T) { rt.ExpectValidateCallerAddr(caller) rt.Call(a.CallerValidation, &CallerValidationArgs{ - Branch: CallerValidationBranchIs, + Branch: CallerValidationBranchIsAddress, Addrs: []address.Address{caller}, }) rt.Verify() @@ -80,7 +80,7 @@ func TestCallerValidationType(t *testing.T) { // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 rt.ExpectAbort(exitcode.ErrForbidden, func() { rt.Call(a.CallerValidation, &CallerValidationArgs{ - Branch: CallerValidationBranchType, + Branch: CallerValidationBranchIsType, Types: []cid.Cid{builtin.CronActorCodeID}, }) }) @@ -88,7 +88,7 @@ func TestCallerValidationType(t *testing.T) { rt.ExpectValidateCallerType(builtin.AccountActorCodeID) rt.Call(a.CallerValidation, &CallerValidationArgs{ - Branch: CallerValidationBranchType, + Branch: CallerValidationBranchIsType, Types: []cid.Cid{builtin.AccountActorCodeID}, }) rt.Verify() From d9951cdff402b400015b5213097c8db67049f8b2 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 21 Sep 2020 10:10:31 +0100 Subject: [PATCH 51/54] fix: regen --- conformance/chaos/cbor_gen.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 0d34264ee..882af7026 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -6,12 +6,12 @@ import ( "fmt" "io" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/exitcode" - "github.com/ipfs/go-cid" + address "github.com/filecoin-project/go-address" + abi "github.com/filecoin-project/go-state-types/abi" + exitcode "github.com/filecoin-project/go-state-types/exitcode" + cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" + xerrors "golang.org/x/xerrors" ) var _ = xerrors.Errorf From a79291070e216309e81aa929b0409bece4bf1820 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 21 Sep 2020 11:53:38 +0100 Subject: [PATCH 52/54] chore: update test-vectors --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 7d3becbeb..9db1dd8c9 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 7d3becbeb5b932baed419c43390595b5e5cece12 +Subproject commit 9db1dd8c9f85bd80eb624e77130a02a3aaf3a1f9 From d9f2b909ac5127589f7e385df5897d7bcd476da6 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 21 Sep 2020 11:58:45 +0100 Subject: [PATCH 53/54] chore: update test-vectors --- extern/test-vectors | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extern/test-vectors b/extern/test-vectors index 9db1dd8c9..6bea015ed 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 9db1dd8c9f85bd80eb624e77130a02a3aaf3a1f9 +Subproject commit 6bea015edddde116001a4251dce3c4a9966c25d9 From 1096f8acc6900dccc7db160cafe9284362f17db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 21 Sep 2020 18:59:05 +0200 Subject: [PATCH 54/54] miner: Invert show-removed option in sectors list --- cmd/lotus-storage-miner/sectors.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 0e6fe8435..f77c6b61c 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -2,7 +2,6 @@ package main import ( "fmt" - sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "os" "sort" "strconv" @@ -19,6 +18,7 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" ) var sectorsCmd = &cli.Command{ @@ -139,8 +139,8 @@ var sectorsListCmd = &cli.Command{ Usage: "List sectors", Flags: []cli.Flag{ &cli.BoolFlag{ - Name: "hide-removed", - Usage: "to hide Removed sectors", + Name: "show-removed", + Usage: "show removed sectors", }, }, Action: func(cctx *cli.Context) error { @@ -199,11 +199,11 @@ var sectorsListCmd = &cli.Command{ continue } - if !cctx.Bool("hide-removed") || st.State != api.SectorState(sealing.Removed) { + if cctx.Bool("show-removed") || st.State != api.SectorState(sealing.Removed) { _, inSSet := commitedIDs[s] _, inASet := activeIDs[s] - fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", + _, _ = fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", s, st.State, yesno(inSSet),