diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fdb3e38f..acd447f69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -254,6 +254,25 @@ jobs: path: /tmp/test-reports - store_artifacts: path: /tmp/test-artifacts/conformance-coverage.html + build-lotus-soup: + description: | + Compile `lotus-soup` Testground test plan using the current version of Lotus. + parameters: + <<: *test-params + executor: << parameters.executor >> + steps: + - install-deps + - prepare + - run: cd extern/oni && git submodule sync + - run: cd extern/oni && git submodule update --init + - run: cd extern/filecoin-ffi && make + - run: + name: "replace lotus, filecoin-ffi, blst and fil-blst deps" + command: cd extern/oni/lotus-soup && go mod edit -replace github.com/filecoin-project/lotus=../../../ && go mod edit -replace github.com/filecoin-project/filecoin-ffi=../../filecoin-ffi && go mod edit -replace github.com/supranational/blst=../../fil-blst/blst && go mod edit -replace github.com/filecoin-project/fil-blst=../../fil-blst + - run: + name: "build lotus-soup testplan" + command: pushd extern/oni/lotus-soup && go build -tags=testground . + build-macos: description: build darwin lotus binary @@ -428,6 +447,7 @@ workflows: test-suite-name: conformance-bleeding-edge packages: "./conformance" vectors-branch: master + - build-lotus-soup - build-debug - build-all: requires: diff --git a/.gitignore b/.gitignore index 940f37b3f..05f762d8d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ /lotus-stats /lotus-bench /lotus-gateway +/lotus-pcr /bench.json /lotuspond/front/node_modules /lotuspond/front/build diff --git a/.gitmodules b/.gitmodules index 4b450aaf3..35f5a3d3f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,3 +11,6 @@ [submodule "extern/fil-blst"] path = extern/fil-blst url = https://github.com/filecoin-project/fil-blst.git +[submodule "extern/oni"] + path = extern/oni + url = https://github.com/filecoin-project/oni diff --git a/CHANGELOG.md b/CHANGELOG.md index 8617554a9..06c0959b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Lotus changelog +# 0.7.1 / 2020-09-17 + +This optional release of Lotus introduces some critical fixes to the window PoSt process. It also upgrades some core dependencies, and introduces many improvements to the mining process, deal-making cycle, and overall User Experience. + +## Changes + +#### Some notable improvements: + +- Correctly construct params for `SubmitWindowedPoSt` messages (https://github.com/filecoin-project/lotus/pull/3909) +- Skip sectors correctly for Window PoSt (https://github.com/filecoin-project/lotus/pull/3839) +- Split window PoST submission into multiple messages (https://github.com/filecoin-project/lotus/pull/3689) +- Improve journal coverage (https://github.com/filecoin-project/lotus/pull/2455) +- Allow retrievals while sealing (https://github.com/filecoin-project/lotus/pull/3778) +- Don't prune locally published messages (https://github.com/filecoin-project/lotus/pull/3772) +- Add get-ask, set-ask retrieval commands (https://github.com/filecoin-project/lotus/pull/3886) +- Consistently name winning and window post in logs (https://github.com/filecoin-project/lotus/pull/3873)) +- Add auto flag to mpool replace (https://github.com/filecoin-project/lotus/pull/3752)) + +#### Dependencies + +- Upgrade markets to `v0.6.1` (https://github.com/filecoin-project/lotus/pull/3906) +- Upgrade specs-actors to `v0.9.10` (https://github.com/filecoin-project/lotus/pull/3846) +- Upgrade badger (https://github.com/filecoin-project/lotus/pull/3739) + # 0.7.0 / 2020-09-10 This consensus-breaking release of Lotus is designed to test a network upgrade on the space race testnet. The changes that break consensus are: diff --git a/README.md b/README.md index 6c1e23efa..766317e4f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more ## Building & Documentation -For instructions on how to build lotus from source, please visit [https://lotu.sh](https://lotu.sh) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation). +For instructions on how to build lotus from source, please visit [Lotus build and setup instruction](https://docs.filecoin.io/get-started/lotus/installation/#minimal-requirements) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation). ## Reporting a Vulnerability diff --git a/api/api_full.go b/api/api_full.go index cbf05b363..dace85ed3 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -354,6 +354,8 @@ type FullNode interface { StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*SectorLocation, error) // StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error) + // StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip + StateMsgGasCost(context.Context, cid.Cid, types.TipSetKey) (*MsgGasCost, error) // StateWaitMsg looks back in the chain for a message. If not found, it blocks until the // message arrives on chain, and gets to the indicated confidence depth. StateWaitMsg(ctx context.Context, cid cid.Cid, confidence uint64) (*MsgLookup, error) @@ -531,6 +533,17 @@ type MsgLookup struct { Height abi.ChainEpoch } +type MsgGasCost struct { + Message cid.Cid // Can be different than requested, in case it was replaced, but only gas values changed + GasUsed abi.TokenAmount + BaseFeeBurn abi.TokenAmount + OverEstimationBurn abi.TokenAmount + MinerPenalty abi.TokenAmount + MinerTip abi.TokenAmount + Refund abi.TokenAmount + TotalCost abi.TokenAmount +} + type BlockMessages struct { BlsMessages []*types.Message SecpkMessages []*types.SignedMessage diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 4006053a3..6a166a12a 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -183,6 +183,7 @@ type FullNodeStruct struct { StateReplay func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error) `perm:"read"` StateGetActor func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error) `perm:"read"` StateReadState func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error) `perm:"read"` + StateMsgGasCost func(context.Context, cid.Cid, types.TipSetKey) (*api.MsgGasCost, error) `perm:"read"` StateWaitMsg func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error) `perm:"read"` StateSearchMsg func(context.Context, cid.Cid) (*api.MsgLookup, error) `perm:"read"` StateListMiners func(context.Context, types.TipSetKey) ([]address.Address, error) `perm:"read"` @@ -827,6 +828,10 @@ func (c *FullNodeStruct) StateReadState(ctx context.Context, addr address.Addres return c.Internal.StateReadState(ctx, addr, tsk) } +func (c *FullNodeStruct) StateMsgGasCost(ctx context.Context, msgc cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { + return c.Internal.StateMsgGasCost(ctx, msgc, tsk) +} + func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) { return c.Internal.StateWaitMsg(ctx, msgc, confidence) } diff --git a/build/params_2k.go b/build/params_2k.go index 0ef1d9b34..9913374e1 100644 --- a/build/params_2k.go +++ b/build/params_2k.go @@ -29,7 +29,7 @@ func init() { BuildType |= Build2k } -const BlockDelaySecs = uint64(30) +const BlockDelaySecs = uint64(4) const PropagationDelaySecs = uint64(1) diff --git a/build/version.go b/build/version.go index 47e5e6a25..8ab789b2d 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.7.0" +const BuildVersion = "0.7.1" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit @@ -83,7 +83,7 @@ func VersionForType(nodeType NodeType) (Version, error) { // semver versions of the rpc api exposed var ( - FullAPIVersion = newVer(0, 14, 0) + FullAPIVersion = newVer(0, 15, 0) MinerAPIVersion = newVer(0, 15, 0) WorkerAPIVersion = newVer(0, 15, 0) ) diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index e6103e2b3..a7f94e6ee 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -233,8 +233,8 @@ func (sm *StateManager) ApplyBlocks(ctx context.Context, parentEpoch abi.ChainEp } receipts = append(receipts, &r.MessageReceipt) - gasReward = big.Add(gasReward, r.MinerTip) - penalty = big.Add(penalty, r.Penalty) + gasReward = big.Add(gasReward, r.GasCosts.MinerTip) + penalty = big.Add(penalty, r.GasCosts.MinerPenalty) if cb != nil { if err := cb(cm.Cid(), m, r); err != nil { diff --git a/chain/stmgr/utils.go b/chain/stmgr/utils.go index 27b71f358..c09f2b22a 100644 --- a/chain/stmgr/utils.go +++ b/chain/stmgr/utils.go @@ -554,7 +554,7 @@ func MinerGetBaseInfo(ctx context.Context, sm *StateManager, bcs beacon.Schedule sectors, err := GetSectorsForWinningPoSt(ctx, pv, sm, lbst, maddr, prand) if err != nil { - return nil, xerrors.Errorf("getting wpost proving set: %w", err) + return nil, xerrors.Errorf("getting winning post proving set: %w", err) } if len(sectors) == 0 { diff --git a/chain/store/store.go b/chain/store/store.go index 20a7e3031..ad30e0324 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -18,7 +18,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/util/adt" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/vm" "github.com/filecoin-project/lotus/journal" bstore "github.com/filecoin-project/lotus/lib/blockstore" @@ -767,32 +766,16 @@ type BlockMessages struct { func (cs *ChainStore) BlockMsgsForTipset(ts *types.TipSet) ([]BlockMessages, error) { applied := make(map[address.Address]uint64) - cst := cbor.NewCborStore(cs.bs) - st, err := state.LoadStateTree(cst, ts.Blocks()[0].ParentStateRoot) - if err != nil { - return nil, xerrors.Errorf("failed to load state tree") - } - - preloadAddr := func(a address.Address) error { - if _, ok := applied[a]; !ok { - act, err := st.GetActor(a) - if err != nil { - return err - } - - applied[a] = act.Nonce - } - return nil - } - selectMsg := func(m *types.Message) (bool, error) { - if err := preloadAddr(m.From); err != nil { - return false, err + // The first match for a sender is guaranteed to have correct nonce -- the block isn't valid otherwise + if _, ok := applied[m.From]; !ok { + applied[m.From] = m.Nonce } if applied[m.From] != m.Nonce { return false, nil } + applied[m.From]++ return true, nil diff --git a/chain/sync.go b/chain/sync.go index f7530f556..b5716a343 100644 --- a/chain/sync.go +++ b/chain/sync.go @@ -991,7 +991,7 @@ func (syncer *Syncer) VerifyWinningPoStProof(ctx context.Context, h *types.Block rand, err := store.DrawRandomness(rbase.Data, crypto.DomainSeparationTag_WinningPoStChallengeSeed, h.Height, buf.Bytes()) if err != nil { - return xerrors.Errorf("failed to get randomness for verifying winningPost proof: %w", err) + return xerrors.Errorf("failed to get randomness for verifying winning post proof: %w", err) } mid, err := address.IDFromAddress(h.Miner) diff --git a/chain/sync_test.go b/chain/sync_test.go index 0b0d1ed00..f4c83341f 100644 --- a/chain/sync_test.go +++ b/chain/sync_test.go @@ -662,6 +662,49 @@ func TestDuplicateNonce(t *testing.T) { require.Equal(t, includedMsg, mft[0].VMMessage().Cid(), "messages for tipset didn't contain expected message") } +// This test asserts that a block that includes a message with bad nonce can't be synced. A nonce is "bad" if it can't +// be applied on the parent state. +func TestBadNonce(t *testing.T) { + H := 10 + tu := prepSyncTest(t, H) + + base := tu.g.CurTipset + + // Produce a message from the banker with a bad nonce + makeBadMsg := func() *types.SignedMessage { + + ba, err := tu.nds[0].StateGetActor(context.TODO(), tu.g.Banker(), base.TipSet().Key()) + require.NoError(t, err) + msg := types.Message{ + To: tu.g.Banker(), + From: tu.g.Banker(), + + Nonce: ba.Nonce + 5, + + Value: types.NewInt(1), + + Method: 0, + + GasLimit: 100_000_000, + GasFeeCap: types.NewInt(0), + GasPremium: types.NewInt(0), + } + + sig, err := tu.g.Wallet().Sign(context.TODO(), tu.g.Banker(), msg.Cid().Bytes()) + require.NoError(t, err) + + return &types.SignedMessage{ + Message: msg, + Signature: *sig, + } + } + + msgs := make([][]*types.SignedMessage, 1) + msgs[0] = []*types.SignedMessage{makeBadMsg()} + + tu.mineOnBlock(base, 0, []int{0}, true, true, msgs) +} + func BenchmarkSyncBasic(b *testing.B) { for i := 0; i < b.N; i++ { runSyncBenchLength(b, 100) diff --git a/chain/vm/burn.go b/chain/vm/burn.go index eb0611349..9f9b95755 100644 --- a/chain/vm/burn.go +++ b/chain/vm/burn.go @@ -22,6 +22,17 @@ type GasOutputs struct { GasBurned int64 } +// ZeroGasOutputs returns a logically zeroed GasOutputs. +func ZeroGasOutputs() GasOutputs { + return GasOutputs{ + BaseFeeBurn: big.Zero(), + OverEstimationBurn: big.Zero(), + MinerPenalty: big.Zero(), + MinerTip: big.Zero(), + Refund: big.Zero(), + } +} + // ComputeGasOverestimationBurn computes amount of gas to be refunded and amount of gas to be burned // Result is (refund, burn) func ComputeGasOverestimationBurn(gasUsed, gasLimit int64) (int64, int64) { @@ -58,13 +69,7 @@ func ComputeGasOverestimationBurn(gasUsed, gasLimit int64) (int64, int64) { func ComputeGasOutputs(gasUsed, gasLimit int64, baseFee, feeCap, gasPremium abi.TokenAmount) GasOutputs { gasUsedBig := big.NewInt(gasUsed) - out := GasOutputs{ - BaseFeeBurn: big.Zero(), - OverEstimationBurn: big.Zero(), - MinerPenalty: big.Zero(), - MinerTip: big.Zero(), - Refund: big.Zero(), - } + out := ZeroGasOutputs() baseFeeToPay := baseFee if baseFee.Cmp(feeCap.Int) > 0 { diff --git a/chain/vm/vm.go b/chain/vm/vm.go index c9acc8658..a90856a1b 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -201,10 +201,9 @@ type Rand interface { type ApplyRet struct { types.MessageReceipt ActorErr aerrors.ActorError - Penalty types.BigInt - MinerTip types.BigInt ExecutionTrace types.ExecutionTrace Duration time.Duration + GasCosts GasOutputs } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, @@ -328,8 +327,7 @@ func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*Ap }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - Penalty: types.NewInt(0), - MinerTip: types.NewInt(0), + GasCosts: GasOutputs{}, Duration: time.Since(start), }, actorErr } @@ -357,14 +355,15 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, msgGasCost := msgGas.Total() // this should never happen, but is currently still exercised by some tests if msgGasCost > msg.GasLimit { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = types.BigMul(vm.baseFee, abi.NewTokenAmount(msgGasCost)) return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrOutOfGas, GasUsed: 0, }, - Penalty: types.BigMul(vm.baseFee, abi.NewTokenAmount(msgGasCost)), + GasCosts: gasOutputs, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } @@ -375,15 +374,16 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, // this should never happen, but is currently still exercised by some tests if err != nil { if xerrors.Is(err, types.ErrActorNotFound) { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderInvalid, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "actor not found: %s", msg.From), - Penalty: minerPenaltyAmount, + GasCosts: gasOutputs, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } return nil, xerrors.Errorf("failed to look up from actor: %w", err) @@ -391,19 +391,22 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, // this should never happen, but is currently still exercised by some tests if !fromActor.Code.Equals(builtin.AccountActorCodeID) { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderInvalid, GasUsed: 0, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderInvalid, "send from not account actor: %s", fromActor.Code), - Penalty: minerPenaltyAmount, + GasCosts: gasOutputs, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } if msg.Nonce != fromActor.Nonce { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderStateInvalid, @@ -411,14 +414,16 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor nonce invalid: msg:%d != state:%d", msg.Nonce, fromActor.Nonce), - Penalty: minerPenaltyAmount, + + GasCosts: gasOutputs, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } gascost := types.BigMul(types.NewInt(uint64(msg.GasLimit)), msg.GasFeeCap) if fromActor.Balance.LessThan(gascost) { + gasOutputs := ZeroGasOutputs() + gasOutputs.MinerPenalty = minerPenaltyAmount return &ApplyRet{ MessageReceipt: types.MessageReceipt{ ExitCode: exitcode.SysErrSenderStateInvalid, @@ -426,9 +431,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: aerrors.Newf(exitcode.SysErrSenderStateInvalid, "actor balance less than needed: %s < %s", types.FIL(fromActor.Balance), types.FIL(gascost)), - Penalty: minerPenaltyAmount, + GasCosts: gasOutputs, Duration: time.Since(start), - MinerTip: big.Zero(), }, nil } @@ -521,8 +525,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, }, ActorErr: actorErr, ExecutionTrace: rt.executionTrace, - Penalty: gasOutputs.MinerPenalty, - MinerTip: gasOutputs.MinerTip, + GasCosts: gasOutputs, Duration: time.Since(start), }, nil } diff --git a/cli/client.go b/cli/client.go index e68f98791..d9ce9251e 100644 --- a/cli/client.go +++ b/cli/client.go @@ -12,6 +12,8 @@ import ( "text/tabwriter" "time" + "github.com/filecoin-project/specs-actors/actors/builtin" + tm "github.com/buger/goterm" "github.com/docker/go-units" "github.com/fatih/color" @@ -527,6 +529,11 @@ func interactiveDeal(cctx *cli.Context) error { continue } + if days < int(build.MinDealDuration/builtin.EpochsInDay) { + printErr(xerrors.Errorf("minimum duration is %d days", int(build.MinDealDuration/builtin.EpochsInDay))) + continue + } + state = "miner" case "miner": fmt.Print("Miner Address (t0..): ") diff --git a/cli/state.go b/cli/state.go index bfcbdf94a..2476d7c59 100644 --- a/cli/state.go +++ b/cli/state.go @@ -66,6 +66,7 @@ var stateCmd = &cli.Command{ stateGetDealSetCmd, stateWaitMsgCmd, stateSearchMsgCmd, + stateMsgCostCmd, stateMinerInfo, stateMarketCmd, }, @@ -1312,6 +1313,60 @@ var stateSearchMsgCmd = &cli.Command{ }, } +var stateMsgCostCmd = &cli.Command{ + Name: "msg-cost", + Usage: "Get the detailed gas costs of a message", + ArgsUsage: "[messageCid]", + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return fmt.Errorf("must specify message cid to get gas costs for") + } + + api, closer, err := GetFullNodeAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := ReqContext(cctx) + + msg, err := cid.Decode(cctx.Args().First()) + if err != nil { + return err + } + + tsk := types.EmptyTSK + + ts, err := LoadTipSet(ctx, cctx, api) + if err != nil { + return err + } + + if ts != nil { + tsk = ts.Key() + } + + mgc, err := api.StateMsgGasCost(ctx, msg, tsk) + if err != nil { + return err + } + + if mgc != nil { + fmt.Printf("Message CID: %s", mgc.Message) + fmt.Printf("\nGas Used: %d", mgc.GasUsed) + fmt.Printf("\nBase Fee Burn: %d", mgc.BaseFeeBurn) + fmt.Printf("\nOverestimation Burn: %d", mgc.OverEstimationBurn) + fmt.Printf("\nMiner Tip: %d", mgc.MinerTip) + fmt.Printf("\nRefund: %d", mgc.Refund) + fmt.Printf("\nTotal Cost: %d", mgc.TotalCost) + fmt.Printf("\nMiner Penalty: %d", mgc.MinerPenalty) + } else { + fmt.Print("message was not found on chain") + } + return nil + }, +} + var stateCallCmd = &cli.Command{ Name: "call", Usage: "Invoke a method on an actor locally", diff --git a/cli/sync.go b/cli/sync.go index bff34960e..3b4e2e9fb 100644 --- a/cli/sync.go +++ b/cli/sync.go @@ -249,14 +249,24 @@ func SyncWait(ctx context.Context, napi api.FullNode) error { ss := state.ActiveSyncs[working] + var baseHeight abi.ChainEpoch var target []cid.Cid var theight abi.ChainEpoch + var heightDiff int64 + + if ss.Base != nil { + baseHeight = ss.Base.Height() + heightDiff = int64(ss.Base.Height()) + } if ss.Target != nil { target = ss.Target.Cids() theight = ss.Target.Height() + heightDiff = int64(ss.Target.Height()) - heightDiff + } else { + heightDiff = 0 } - fmt.Printf("\r\x1b[2KWorker %d: Target Height: %d\tTarget: %s\tState: %s\tHeight: %d", working, theight, target, ss.Stage, ss.Height) + fmt.Printf("\r\x1b[2KWorker %d: Base Height: %d\tTarget Height: %d\t Height diff: %d\tTarget: %s\tState: %s\tHeight: %d", working, baseHeight, theight, heightDiff, target, ss.Stage, ss.Height) if time.Now().Unix()-int64(head.MinTimestamp()) < int64(build.BlockDelaySecs) { fmt.Println("\nDone!") diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 7fe63eae1..f2845ba20 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -56,6 +56,10 @@ var importBenchCmd = &cli.Command{ Usage: "set the parallelism factor for batch seal verification", Value: runtime.NumCPU(), }, + &cli.StringFlag{ + Name: "repodir", + Usage: "set the repo directory for the lotus bench run (defaults to /tmp)", + }, }, Action: func(cctx *cli.Context) error { vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads") @@ -70,9 +74,15 @@ var importBenchCmd = &cli.Command{ } defer cfi.Close() //nolint:errcheck // read only file - tdir, err := ioutil.TempDir("", "lotus-import-bench") - if err != nil { - return err + var tdir string + if rdir := cctx.String("repodir"); rdir != "" { + tdir = rdir + } else { + tmp, err := ioutil.TempDir("", "lotus-import-bench") + if err != nil { + return err + } + tdir = tmp } bds, err := badger.NewDatastore(tdir, nil) diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 9410cc760..2155f464a 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -387,7 +387,7 @@ var sealBenchCmd = &cli.Command{ return err } if !ok { - log.Error("post verification failed") + log.Error("window post verification failed") } verifyWindowpost1 := time.Now() @@ -403,7 +403,7 @@ var sealBenchCmd = &cli.Command{ return err } if !ok { - log.Error("post verification failed") + log.Error("window post verification failed") } verifyWindowpost2 := time.Now() diff --git a/cmd/lotus-shed/bitfield.go b/cmd/lotus-shed/bitfield.go index 79ce214ee..442cbef48 100644 --- a/cmd/lotus-shed/bitfield.go +++ b/cmd/lotus-shed/bitfield.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "io" "io/ioutil" "os" @@ -28,6 +29,9 @@ var bitFieldCmd = &cli.Command{ bitFieldRunsCmd, bitFieldStatCmd, bitFieldDecodeCmd, + bitFieldIntersectCmd, + bitFieldEncodeCmd, + bitFieldSubCmd, }, } @@ -200,38 +204,9 @@ var bitFieldDecodeCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - var val string - if cctx.Args().Present() { - val = cctx.Args().Get(0) - } else { - b, err := ioutil.ReadAll(os.Stdin) - if err != nil { - return err - } - val = string(b) - } - - var dec []byte - switch cctx.String("enc") { - case "base64": - d, err := base64.StdEncoding.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding base64 value: %w", err) - } - dec = d - case "hex": - d, err := hex.DecodeString(val) - if err != nil { - return fmt.Errorf("decoding hex value: %w", err) - } - dec = d - default: - return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) - } - - rle, err := bitfield.NewFromBytes(dec) + rle, err := decode(cctx, 0) if err != nil { - return xerrors.Errorf("failed to parse bitfield: %w", err) + return err } vals, err := rle.All(100000000000) @@ -243,3 +218,170 @@ var bitFieldDecodeCmd = &cli.Command{ return nil }, } + +var bitFieldIntersectCmd = &cli.Command{ + Name: "intersect", + Description: "intersect 2 bitfields and print the resulting bitfield as base64", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + b, err := decode(cctx, 1) + if err != nil { + return err + } + + a, err := decode(cctx, 0) + if err != nil { + return err + } + + o, err := bitfield.IntersectBitField(a, b) + if err != nil { + return xerrors.Errorf("intersect: %w", err) + } + + s, err := o.RunIterator() + if err != nil { + return err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + + return nil + }, +} + +var bitFieldSubCmd = &cli.Command{ + Name: "sub", + Description: "subtract 2 bitfields and print the resulting bitfield as base64", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + b, err := decode(cctx, 1) + if err != nil { + return err + } + + a, err := decode(cctx, 0) + if err != nil { + return err + } + + o, err := bitfield.SubtractBitField(a, b) + if err != nil { + return xerrors.Errorf("intersect: %w", err) + } + + s, err := o.RunIterator() + if err != nil { + return err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + + return nil + }, +} + +var bitFieldEncodeCmd = &cli.Command{ + Name: "encode", + Description: "encode a series of decimal numbers into a bitfield", + ArgsUsage: "[infile]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "enc", + Value: "base64", + Usage: "specify input encoding to parse", + }, + }, + Action: func(cctx *cli.Context) error { + f, err := os.Open(cctx.Args().First()) + if err != nil { + return err + } + defer f.Close() // nolint + + out := bitfield.New() + for { + var i uint64 + _, err := fmt.Fscan(f, &i) + if err == io.EOF { + break + } + out.Set(i) + } + + s, err := out.RunIterator() + if err != nil { + return err + } + + bytes, err := rlepluslazy.EncodeRuns(s, []byte{}) + if err != nil { + return err + } + + fmt.Println(base64.StdEncoding.EncodeToString(bytes)) + + return nil + }, +} + +func decode(cctx *cli.Context, a int) (bitfield.BitField, error) { + var val string + if cctx.Args().Present() { + if a >= cctx.NArg() { + return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a) + } + val = cctx.Args().Get(a) + } else { + if a > 0 { + return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a) + } + b, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return bitfield.BitField{}, err + } + val = string(b) + } + + var dec []byte + switch cctx.String("enc") { + case "base64": + d, err := base64.StdEncoding.DecodeString(val) + if err != nil { + return bitfield.BitField{}, fmt.Errorf("decoding base64 value: %w", err) + } + dec = d + case "hex": + d, err := hex.DecodeString(val) + if err != nil { + return bitfield.BitField{}, fmt.Errorf("decoding hex value: %w", err) + } + dec = d + default: + return bitfield.BitField{}, fmt.Errorf("unrecognized encoding: %s", cctx.String("enc")) + } + + return bitfield.NewFromBytes(dec) +} diff --git a/cmd/lotus-shed/consensus.go b/cmd/lotus-shed/consensus.go new file mode 100644 index 000000000..59d9555df --- /dev/null +++ b/cmd/lotus-shed/consensus.go @@ -0,0 +1,286 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "os" + "strconv" + "strings" + "time" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/multiformats/go-multiaddr" + "github.com/urfave/cli/v2" +) + +var consensusCmd = &cli.Command{ + Name: "consensus", + Usage: "tools for gathering information about consensus between nodes", + Flags: []cli.Flag{}, + Subcommands: []*cli.Command{ + consensusCheckCmd, + }, +} + +type consensusItem struct { + multiaddr multiaddr.Multiaddr + genesisTipset *types.TipSet + targetTipset *types.TipSet + headTipset *types.TipSet + peerID peer.ID + version api.Version + api api.FullNode +} + +var consensusCheckCmd = &cli.Command{ + Name: "check", + Usage: "verify if all nodes agree upon a common tipset for a given tipset height", + Description: `Consensus check verifies that all nodes share a common tipset for a given + height. + + The height flag specifies a chain height to start a comparison from. There are two special + arguments for this flag. All other expected values should be chain tipset heights. + + @common - Use the maximum common chain height between all nodes + @expected - Use the current time and the genesis timestamp to determine a height + + Examples + + Find the highest common tipset and look back 10 tipsets + lotus-shed consensus check --height @common --lookback 10 + + Calculate the expected tipset height and look back 10 tipsets + lotus-shed consensus check --height @expected --lookback 10 + + Check if nodes all share a common genesis + lotus-shed consensus check --height 0 + + Check that all nodes agree upon the tipset for 1day post genesis + lotus-shed consensus check --height 2880 --lookback 0 + `, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "height", + Value: "@common", + Usage: "height of tipset to start check from", + }, + &cli.IntFlag{ + Name: "lookback", + Value: int(build.MessageConfidence * 2), + Usage: "number of tipsets behind to look back when comparing nodes", + }, + }, + Action: func(cctx *cli.Context) error { + filePath := cctx.Args().First() + + var input *bufio.Reader + if cctx.Args().Len() == 0 { + input = bufio.NewReader(os.Stdin) + } else { + var err error + inputFile, err := os.Open(filePath) + if err != nil { + return err + } + defer inputFile.Close() //nolint:errcheck + input = bufio.NewReader(inputFile) + } + + var nodes []*consensusItem + ctx := lcli.ReqContext(cctx) + + for { + strma, errR := input.ReadString('\n') + strma = strings.TrimSpace(strma) + + if len(strma) == 0 { + if errR == io.EOF { + break + } + continue + } + + apima, err := multiaddr.NewMultiaddr(strma) + if err != nil { + return err + } + ainfo := lcli.APIInfo{Addr: apima} + addr, err := ainfo.DialArgs() + if err != nil { + return err + } + + api, closer, err := client.NewFullNodeRPC(cctx.Context, addr, nil) + if err != nil { + return err + } + defer closer() + + peerID, err := api.ID(ctx) + if err != nil { + return err + } + + version, err := api.Version(ctx) + if err != nil { + return err + } + + genesisTipset, err := api.ChainGetGenesis(ctx) + if err != nil { + return err + } + + headTipset, err := api.ChainHead(ctx) + if err != nil { + return err + } + + nodes = append(nodes, &consensusItem{ + genesisTipset: genesisTipset, + headTipset: headTipset, + multiaddr: apima, + api: api, + peerID: peerID, + version: version, + }) + + if errR != nil && errR != io.EOF { + return err + } + + if errR == io.EOF { + break + } + } + + if len(nodes) == 0 { + return fmt.Errorf("no nodes") + } + + genesisBuckets := make(map[types.TipSetKey][]*consensusItem) + for _, node := range nodes { + genesisBuckets[node.genesisTipset.Key()] = append(genesisBuckets[node.genesisTipset.Key()], node) + + } + + if len(genesisBuckets) != 1 { + for _, nodes := range genesisBuckets { + for _, node := range nodes { + log.Errorw( + "genesis do not match", + "genesis_tipset", node.genesisTipset.Key(), + "peer_id", node.peerID, + "version", node.version, + ) + } + } + + return fmt.Errorf("genesis does not match between all nodes") + } + + target := abi.ChainEpoch(0) + + switch cctx.String("height") { + case "@common": + minTipset := nodes[0].headTipset + for _, node := range nodes { + if node.headTipset.Height() < minTipset.Height() { + minTipset = node.headTipset + } + } + + target = minTipset.Height() + case "@expected": + tnow := uint64(time.Now().Unix()) + tgen := nodes[0].genesisTipset.MinTimestamp() + + target = abi.ChainEpoch((tnow - tgen) / build.BlockDelaySecs) + default: + h, err := strconv.Atoi(strings.TrimSpace(cctx.String("height"))) + if err != nil { + return fmt.Errorf("failed to parse string: %s", cctx.String("height")) + } + + target = abi.ChainEpoch(h) + } + + lookback := abi.ChainEpoch(cctx.Int("lookback")) + if lookback > target { + target = abi.ChainEpoch(0) + } else { + target = target - lookback + } + + for _, node := range nodes { + targetTipset, err := node.api.ChainGetTipSetByHeight(ctx, target, types.EmptyTSK) + if err != nil { + log.Errorw("error checking target", "err", err) + node.targetTipset = nil + } else { + node.targetTipset = targetTipset + } + + } + for _, node := range nodes { + log.Debugw( + "node info", + "peer_id", node.peerID, + "version", node.version, + "genesis_tipset", node.genesisTipset.Key(), + "head_tipset", node.headTipset.Key(), + "target_tipset", node.targetTipset.Key(), + ) + } + + targetBuckets := make(map[types.TipSetKey][]*consensusItem) + for _, node := range nodes { + if node.targetTipset == nil { + targetBuckets[types.EmptyTSK] = append(targetBuckets[types.EmptyTSK], node) + continue + } + + targetBuckets[node.targetTipset.Key()] = append(targetBuckets[node.targetTipset.Key()], node) + } + + if nodes, ok := targetBuckets[types.EmptyTSK]; ok { + for _, node := range nodes { + log.Errorw( + "targeted tipset not found", + "peer_id", node.peerID, + "version", node.version, + "genesis_tipset", node.genesisTipset.Key(), + "head_tipset", node.headTipset.Key(), + "target_tipset", node.targetTipset.Key(), + ) + } + + return fmt.Errorf("targeted tipset not found") + } + + if len(targetBuckets) != 1 { + for _, nodes := range targetBuckets { + for _, node := range nodes { + log.Errorw( + "targeted tipset not found", + "peer_id", node.peerID, + "version", node.version, + "genesis_tipset", node.genesisTipset.Key(), + "head_tipset", node.headTipset.Key(), + "target_tipset", node.targetTipset.Key(), + ) + } + } + return fmt.Errorf("nodes not in consensus at tipset height %d", target) + } + + return nil + }, +} diff --git a/cmd/lotus-shed/jwt.go b/cmd/lotus-shed/jwt.go index d37359f71..7fa1a18dd 100644 --- a/cmd/lotus-shed/jwt.go +++ b/cmd/lotus-shed/jwt.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "crypto/rand" "encoding/hex" "encoding/json" @@ -8,10 +9,12 @@ import ( "io" "io/ioutil" "os" + "strings" "github.com/gbrlsnchs/jwt/v3" "github.com/urfave/cli/v2" + "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/lotus/api/apistruct" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/node/modules" @@ -24,6 +27,102 @@ var jwtCmd = &cli.Command{ having to run the lotus daemon.`, Subcommands: []*cli.Command{ jwtNewCmd, + jwtTokenCmd, + }, +} + +var jwtTokenCmd = &cli.Command{ + Name: "token", + Usage: "create a token for a given jwt secret", + ArgsUsage: "", + Description: `The jwt tokens have four different levels of permissions that provide some ability + to control access to what methods can be invoked by the holder of the token. + + This command only works on jwt secrets that are base16 encoded files, such as those produced by the + sibling 'new' command. + `, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "output", + Value: "token", + Usage: "specify a name", + }, + &cli.BoolFlag{ + Name: "read", + Value: false, + Usage: "add read permissions to the token", + }, + &cli.BoolFlag{ + Name: "write", + Value: false, + Usage: "add write permissions to the token", + }, + &cli.BoolFlag{ + Name: "sign", + Value: false, + Usage: "add sign permissions to the token", + }, + &cli.BoolFlag{ + Name: "admin", + Value: false, + Usage: "add admin permissions to the token", + }, + }, + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + return fmt.Errorf("please specify a name") + } + + inputFile, err := os.Open(cctx.Args().First()) + if err != nil { + return err + } + defer inputFile.Close() //nolint:errcheck + input := bufio.NewReader(inputFile) + + encoded, err := ioutil.ReadAll(input) + if err != nil { + return err + } + + decoded, err := hex.DecodeString(strings.TrimSpace(string(encoded))) + if err != nil { + return err + } + + var keyInfo types.KeyInfo + if err := json.Unmarshal(decoded, &keyInfo); err != nil { + return err + } + + perms := []auth.Permission{} + + if cctx.Bool("read") { + perms = append(perms, apistruct.PermRead) + } + + if cctx.Bool("write") { + perms = append(perms, apistruct.PermWrite) + } + + if cctx.Bool("sign") { + perms = append(perms, apistruct.PermSign) + } + + if cctx.Bool("admin") { + perms = append(perms, apistruct.PermAdmin) + } + + p := modules.JwtPayload{ + Allow: perms, + } + + token, err := jwt.Sign(&p, jwt.NewHS256(keyInfo.PrivateKey)) + if err != nil { + return err + } + + return ioutil.WriteFile(cctx.String("output"), token, 0600) }, } diff --git a/cmd/lotus-shed/keyinfo.go b/cmd/lotus-shed/keyinfo.go index f397bd4c7..6090b7a0f 100644 --- a/cmd/lotus-shed/keyinfo.go +++ b/cmd/lotus-shed/keyinfo.go @@ -9,16 +9,22 @@ import ( "io" "io/ioutil" "os" + "path" "strings" "text/template" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + + "github.com/multiformats/go-base32" + "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/peer" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" + "github.com/filecoin-project/lotus/node/modules" "github.com/filecoin-project/lotus/node/modules/lp2p" "github.com/filecoin-project/lotus/node/repo" @@ -43,6 +49,90 @@ var keyinfoCmd = &cli.Command{ keyinfoNewCmd, keyinfoInfoCmd, keyinfoImportCmd, + keyinfoVerifyCmd, + }, +} + +var keyinfoVerifyCmd = &cli.Command{ + Name: "verify", + Usage: "verify the filename of a keystore object on disk with it's contents", + Description: `Keystore objects are base32 enocded strings, with wallets being dynamically named via + the wallet address. This command can ensure that the naming of these keystore objects are correct`, + Action: func(cctx *cli.Context) error { + filePath := cctx.Args().First() + fileName := path.Base(filePath) + + inputFile, err := os.Open(filePath) + if err != nil { + return err + } + defer inputFile.Close() //nolint:errcheck + input := bufio.NewReader(inputFile) + + keyContent, err := ioutil.ReadAll(input) + if err != nil { + return err + } + + var keyInfo types.KeyInfo + if err := json.Unmarshal(keyContent, &keyInfo); err != nil { + return err + } + + switch keyInfo.Type { + case lp2p.KTLibp2pHost: + name, err := base32.RawStdEncoding.DecodeString(fileName) + if err != nil { + return xerrors.Errorf("decoding key: '%s': %w", fileName, err) + } + + if string(name) != keyInfo.Type { + return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type) + } + case modules.KTJwtHmacSecret: + name, err := base32.RawStdEncoding.DecodeString(fileName) + if err != nil { + return xerrors.Errorf("decoding key: '%s': %w", fileName, err) + } + + if string(name) != modules.JWTSecretName { + return fmt.Errorf("%s of type %s is incorrect", fileName, keyInfo.Type) + } + case wallet.KTSecp256k1, wallet.KTBLS: + keystore := wallet.NewMemKeyStore() + w, err := wallet.NewWallet(keystore) + if err != nil { + return err + } + + if _, err := w.Import(&keyInfo); err != nil { + return err + } + + list, err := keystore.List() + if err != nil { + return err + } + + if len(list) != 1 { + return fmt.Errorf("Unexpected number of keys, expected 1, found %d", len(list)) + } + + name, err := base32.RawStdEncoding.DecodeString(fileName) + if err != nil { + return xerrors.Errorf("decoding key: '%s': %w", fileName, err) + } + + if string(name) != list[0] { + return fmt.Errorf("%s of type %s; file is named for %s, but key is actually %s", fileName, keyInfo.Type, string(name), list[0]) + } + + break + default: + return fmt.Errorf("Unknown keytype %s", keyInfo.Type) + } + + return nil }, } diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index cff3059b6..1a56756d1 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -35,6 +35,7 @@ func main() { mathCmd, mpoolStatsCmd, exportChainCmd, + consensusCmd, } app := &cli.App{ @@ -49,6 +50,13 @@ func main() { Hidden: true, Value: "~/.lotus", // TODO: Consider XDG_DATA_HOME }, + &cli.StringFlag{ + Name: "log-level", + Value: "info", + }, + }, + Before: func(cctx *cli.Context) error { + return logging.SetLogLevel("lotus-shed", cctx.String("log-level")) }, } diff --git a/cmd/lotus-storage-miner/retrieval-deals.go b/cmd/lotus-storage-miner/retrieval-deals.go index df194978d..03d397852 100644 --- a/cmd/lotus-storage-miner/retrieval-deals.go +++ b/cmd/lotus-storage-miner/retrieval-deals.go @@ -5,9 +5,12 @@ import ( "os" "text/tabwriter" + "github.com/docker/go-units" "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-state-types/abi" "github.com/urfave/cli/v2" + "github.com/filecoin-project/lotus/chain/types" lcli "github.com/filecoin-project/lotus/cli" ) @@ -17,6 +20,8 @@ var retrievalDealsCmd = &cli.Command{ Subcommands: []*cli.Command{ retrievalDealSelectionCmd, retrievalDealsListCmd, + retrievalSetAskCmd, + retrievalGetAskCmd, }, } @@ -154,3 +159,112 @@ var retrievalDealsListCmd = &cli.Command{ return w.Flush() }, } + +var retrievalSetAskCmd = &cli.Command{ + Name: "set-ask", + Usage: "Configure the provider's retrieval ask", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "price", + Usage: "Set the price of the ask for retrievals (FIL/GiB)", + }, + &cli.StringFlag{ + Name: "unseal-price", + Usage: "Set the price to unseal", + }, + &cli.StringFlag{ + Name: "payment-interval", + Usage: "Set the payment interval (in bytes) for retrieval", + DefaultText: "1MiB", + }, + &cli.StringFlag{ + Name: "payment-interval-increase", + Usage: "Set the payment interval increase (in bytes) for retrieval", + DefaultText: "1MiB", + }, + }, + Action: func(cctx *cli.Context) error { + ctx := lcli.DaemonContext(cctx) + + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + ask, err := api.MarketGetRetrievalAsk(ctx) + if err != nil { + return err + } + + if cctx.IsSet("price") { + v, err := types.ParseFIL(cctx.String("price")) + if err != nil { + return err + } + ask.PricePerByte = types.BigDiv(types.BigInt(v), types.NewInt(1<<30)) + } + + if cctx.IsSet("unseal-price") { + v, err := types.ParseFIL(cctx.String("unseal-price")) + if err != nil { + return err + } + ask.UnsealPrice = abi.TokenAmount(v) + } + + if cctx.IsSet("payment-interval") { + v, err := units.RAMInBytes(cctx.String("payment-interval")) + if err != nil { + return err + } + ask.PaymentInterval = uint64(v) + } + + if cctx.IsSet("payment-interval-increase") { + v, err := units.RAMInBytes(cctx.String("payment-interval-increase")) + if err != nil { + return err + } + ask.PaymentIntervalIncrease = uint64(v) + } + + return api.MarketSetRetrievalAsk(ctx, ask) + }, +} + +var retrievalGetAskCmd = &cli.Command{ + Name: "get-ask", + Usage: "Get the provider's current retrieval ask", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + ctx := lcli.DaemonContext(cctx) + + api, closer, err := lcli.GetStorageMinerAPI(cctx) + if err != nil { + return err + } + defer closer() + + ask, err := api.MarketGetRetrievalAsk(ctx) + if err != nil { + return err + } + + w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) + fmt.Fprintf(w, "Price per Byte\tUnseal Price\tPayment Interval\tPayment Interval Increase\n") + if ask == nil { + fmt.Fprintf(w, "\n") + return w.Flush() + } + + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", + types.FIL(ask.PricePerByte), + types.FIL(ask.UnsealPrice), + units.BytesSize(float64(ask.PaymentInterval)), + units.BytesSize(float64(ask.PaymentIntervalIncrease)), + ) + return w.Flush() + + }, +} diff --git a/cmd/lotus-storage-miner/run.go b/cmd/lotus-storage-miner/run.go index 83d78172e..a5d996f78 100644 --- a/cmd/lotus-storage-miner/run.go +++ b/cmd/lotus-storage-miner/run.go @@ -163,8 +163,10 @@ var runCmd = &cli.Command{ sigChan := make(chan os.Signal, 2) go func() { select { - case <-sigChan: + case sig := <-sigChan: + log.Warnw("received shutdown", "signal", sig) case <-shutdownChan: + log.Warn("received shutdown") } log.Warn("Shutting down...") diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 06f09fe20..438aaad0d 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -17,7 +17,9 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + lcli "github.com/filecoin-project/lotus/cli" + sealing "github.com/filecoin-project/lotus/extern/storage-sealing" ) var sectorsCmd = &cli.Command{ @@ -136,6 +138,12 @@ var sectorsStatusCmd = &cli.Command{ var sectorsListCmd = &cli.Command{ Name: "list", Usage: "List sectors", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "show-removed", + Usage: "show removed sectors", + }, + }, Action: func(cctx *cli.Context) error { nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { @@ -192,19 +200,21 @@ var sectorsListCmd = &cli.Command{ continue } - _, inSSet := commitedIDs[s] - _, inASet := activeIDs[s] + if cctx.Bool("show-removed") || st.State != api.SectorState(sealing.Removed) { + _, inSSet := commitedIDs[s] + _, inASet := activeIDs[s] - fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", - s, - st.State, - yesno(inSSet), - yesno(inASet), - st.Ticket.Epoch, - st.Seed.Epoch, - st.Deals, - st.ToUpgrade, - ) + _, _ = fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", + s, + st.State, + yesno(inSSet), + yesno(inASet), + st.Ticket.Epoch, + st.Seed.Epoch, + st.Deals, + st.ToUpgrade, + ) + } } return w.Flush() @@ -420,6 +430,10 @@ var sectorsUpdateCmd = &cli.Command{ return xerrors.Errorf("could not parse sector number: %w", err) } + if _, ok := sealing.ExistSectorStateList[sealing.SectorState(cctx.Args().Get(1))]; !ok { + return xerrors.Errorf("Not existing sector state") + } + return nodeApi.SectorsUpdate(ctx, abi.SectorNumber(id), api.SectorState(cctx.Args().Get(1))) }, } diff --git a/cmd/lotus-storage-miner/storage.go b/cmd/lotus-storage-miner/storage.go index 3d6f48e00..ebc5d2fbe 100644 --- a/cmd/lotus-storage-miner/storage.go +++ b/cmd/lotus-storage-miner/storage.go @@ -371,7 +371,7 @@ var storageFindCmd = &cli.Command{ } fmt.Printf("In %s (%s)\n", info.id, types[:len(types)-2]) - fmt.Printf("\tSealing: %t; Storage: %t\n", info.store.CanSeal, info.store.CanSeal) + fmt.Printf("\tSealing: %t; Storage: %t\n", info.store.CanSeal, info.store.CanStore) if localPath, ok := local[info.id]; ok { fmt.Printf("\tLocal (%s)\n", localPath) } else { diff --git a/cmd/lotus/rpc.go b/cmd/lotus/rpc.go index fbe05e938..9718deb3a 100644 --- a/cmd/lotus/rpc.go +++ b/cmd/lotus/rpc.go @@ -66,8 +66,10 @@ func serveRPC(a api.FullNode, stop node.StopFunc, addr multiaddr.Multiaddr, shut shutdownDone := make(chan struct{}) go func() { select { - case <-sigCh: + case sig := <-sigCh: + log.Warnw("received shutdown", "signal", sig) case <-shutdownCh: + log.Warn("received shutdown") } log.Warn("Shutting down...") diff --git a/conformance/chaos/actor.go b/conformance/chaos/actor.go index daff26694..477204374 100644 --- a/conformance/chaos/actor.go +++ b/conformance/chaos/actor.go @@ -7,8 +7,6 @@ import ( "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/runtime" "github.com/ipfs/go-cid" - - typegen "github.com/whyrusleeping/cbor-gen" ) //go:generate go run ./gen @@ -31,10 +29,14 @@ type Actor struct{} type CallerValidationBranch int64 const ( + // CallerValidationBranchNone causes no caller validation to take place. CallerValidationBranchNone CallerValidationBranch = iota + // CallerValidationBranchTwice causes Runtime.ValidateImmediateCallerAcceptAny to be called twice. CallerValidationBranchTwice - CallerValidationBranchAddrNilSet - CallerValidationBranchTypeNilSet + // CallerValidationBranchIsAddress causes caller validation against CallerValidationArgs.Addrs. + CallerValidationBranchIsAddress + // CallerValidationBranchIsType causes caller validation against CallerValidationArgs.Types. + CallerValidationBranchIsType ) // MutateStateBranch is an enum used to select the type of state mutation to attempt. @@ -64,6 +66,9 @@ const ( // MethodAbortWith is the identifier for the method that panics optionally with // a passed exit code. MethodAbortWith + // MethodInspectRuntime is the identifier for the method that returns the + // current runtime values. + MethodInspectRuntime ) // Exports defines the methods this actor exposes publicly. @@ -77,6 +82,7 @@ func (a Actor) Exports() []interface{} { MethodSend: a.Send, MethodMutateState: a.MutateState, MethodAbortWith: a.AbortWith, + MethodInspectRuntime: a.InspectRuntime, } } @@ -119,23 +125,29 @@ func (a Actor) Constructor(_ runtime.Runtime, _ *abi.EmptyValue) *abi.EmptyValue panic("constructor should not be called; the Chaos actor is a singleton actor") } +// CallerValidationArgs are the arguments to Actor.CallerValidation. +type CallerValidationArgs struct { + Branch CallerValidationBranch + Addrs []address.Address + Types []cid.Cid +} + // CallerValidation violates VM call validation constraints. // // CallerValidationBranchNone performs no validation. // CallerValidationBranchTwice validates twice. -// CallerValidationBranchAddrNilSet validates against an empty caller -// address set. -// CallerValidationBranchTypeNilSet validates against an empty caller type set. -func (a Actor) CallerValidation(rt runtime.Runtime, branch *typegen.CborInt) *abi.EmptyValue { - switch CallerValidationBranch(*branch) { +// CallerValidationBranchIsAddress validates caller against CallerValidationArgs.Addrs. +// CallerValidationBranchIsType validates caller against CallerValidationArgs.Types. +func (a Actor) CallerValidation(rt runtime.Runtime, args *CallerValidationArgs) *abi.EmptyValue { + switch args.Branch { case CallerValidationBranchNone: case CallerValidationBranchTwice: rt.ValidateImmediateCallerAcceptAny() rt.ValidateImmediateCallerAcceptAny() - case CallerValidationBranchAddrNilSet: - rt.ValidateImmediateCallerIs() - case CallerValidationBranchTypeNilSet: - rt.ValidateImmediateCallerType() + case CallerValidationBranchIsAddress: + rt.ValidateImmediateCallerIs(args.Addrs...) + case CallerValidationBranchIsType: + rt.ValidateImmediateCallerType(args.Types...) default: panic("invalid branch passed to CallerValidation") } @@ -247,3 +259,28 @@ func (a Actor) AbortWith(rt runtime.Runtime, args *AbortWithArgs) *abi.EmptyValu } return nil } + +// InspectRuntimeReturn is the return value for the Actor.InspectRuntime method. +type InspectRuntimeReturn struct { + Caller address.Address + Receiver address.Address + ValueReceived abi.TokenAmount + CurrEpoch abi.ChainEpoch + CurrentBalance abi.TokenAmount + State State +} + +// InspectRuntime returns a copy of the serializable values available in the Runtime. +func (a Actor) InspectRuntime(rt runtime.Runtime, _ *abi.EmptyValue) *InspectRuntimeReturn { + rt.ValidateImmediateCallerAcceptAny() + var st State + rt.StateReadonly(&st) + return &InspectRuntimeReturn{ + Caller: rt.Caller(), + Receiver: rt.Receiver(), + ValueReceived: rt.ValueReceived(), + CurrEpoch: rt.CurrEpoch(), + CurrentBalance: rt.CurrentBalance(), + State: st, + } +} diff --git a/conformance/chaos/actor_test.go b/conformance/chaos/actor_test.go index 67ced2899..2061efb82 100644 --- a/conformance/chaos/actor_test.go +++ b/conformance/chaos/actor_test.go @@ -4,10 +4,13 @@ import ( "context" "testing" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/support/mock" atesting "github.com/filecoin-project/specs-actors/support/testing" + "github.com/ipfs/go-cid" ) func TestSingleton(t *testing.T) { @@ -24,6 +27,86 @@ func TestSingleton(t *testing.T) { rt.Verify() } +func TestCallerValidationNone(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + var a Actor + + rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: CallerValidationBranchNone}) + rt.Verify() +} + +func TestCallerValidationIs(t *testing.T) { + caller := atesting.NewIDAddr(t, 100) + receiver := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + rt.SetCaller(caller, builtin.AccountActorCodeID) + var a Actor + + caddrs := []address.Address{atesting.NewIDAddr(t, 101)} + + rt.ExpectValidateCallerAddr(caddrs...) + // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 + rt.ExpectAbort(exitcode.ErrForbidden, func() { + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchIsAddress, + Addrs: caddrs, + }) + }) + rt.Verify() + + rt.ExpectValidateCallerAddr(caller) + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchIsAddress, + Addrs: []address.Address{caller}, + }) + rt.Verify() +} + +func TestCallerValidationType(t *testing.T) { + caller := atesting.NewIDAddr(t, 100) + receiver := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + rt.SetCaller(caller, builtin.AccountActorCodeID) + var a Actor + + rt.ExpectValidateCallerType(builtin.CronActorCodeID) + // FIXME: https://github.com/filecoin-project/specs-actors/pull/1155 + rt.ExpectAbort(exitcode.ErrForbidden, func() { + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchIsType, + Types: []cid.Cid{builtin.CronActorCodeID}, + }) + }) + rt.Verify() + + rt.ExpectValidateCallerType(builtin.AccountActorCodeID) + rt.Call(a.CallerValidation, &CallerValidationArgs{ + Branch: CallerValidationBranchIsType, + Types: []cid.Cid{builtin.AccountActorCodeID}, + }) + rt.Verify() +} + +func TestCallerValidationInvalidBranch(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + var a Actor + + rt.ExpectAssertionFailure("invalid branch passed to CallerValidation", func() { + rt.Call(a.CallerValidation, &CallerValidationArgs{Branch: -1}) + }) + rt.Verify() +} + func TestDeleteActor(t *testing.T) { receiver := atesting.NewIDAddr(t, 100) beneficiary := atesting.NewIDAddr(t, 101) @@ -117,6 +200,20 @@ func TestMutateStateReadonly(t *testing.T) { rt.Verify() } +func TestMutateStateInvalidBranch(t *testing.T) { + receiver := atesting.NewIDAddr(t, 100) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + var a Actor + + rt.ExpectValidateCallerAny() + rt.ExpectAssertionFailure("unknown mutation type", func() { + rt.Call(a.MutateState, &MutateStateArgs{Branch: -1}) + }) + rt.Verify() +} + func TestAbortWith(t *testing.T) { receiver := atesting.NewIDAddr(t, 100) builder := mock.NewBuilder(context.Background(), receiver) @@ -151,3 +248,28 @@ func TestAbortWithUncontrolled(t *testing.T) { }) rt.Verify() } + +func TestInspectRuntime(t *testing.T) { + caller := atesting.NewIDAddr(t, 100) + receiver := atesting.NewIDAddr(t, 101) + builder := mock.NewBuilder(context.Background(), receiver) + + rt := builder.Build(t) + rt.SetCaller(caller, builtin.AccountActorCodeID) + rt.StateCreate(&State{}) + var a Actor + + rt.ExpectValidateCallerAny() + ret := rt.Call(a.InspectRuntime, abi.Empty) + rtr, ok := ret.(*InspectRuntimeReturn) + if !ok { + t.Fatal("invalid return value") + } + if rtr.Caller != caller { + t.Fatal("unexpected runtime caller") + } + if rtr.Receiver != receiver { + t.Fatal("unexpected runtime receiver") + } + rt.Verify() +} diff --git a/conformance/chaos/cbor_gen.go b/conformance/chaos/cbor_gen.go index 710b84b93..882af7026 100644 --- a/conformance/chaos/cbor_gen.go +++ b/conformance/chaos/cbor_gen.go @@ -6,8 +6,10 @@ import ( "fmt" "io" + address "github.com/filecoin-project/go-address" abi "github.com/filecoin-project/go-state-types/abi" exitcode "github.com/filecoin-project/go-state-types/exitcode" + cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -115,6 +117,163 @@ func (t *State) UnmarshalCBOR(r io.Reader) error { return nil } +var lengthBufCallerValidationArgs = []byte{131} + +func (t *CallerValidationArgs) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufCallerValidationArgs); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.Branch (chaos.CallerValidationBranch) (int64) + if t.Branch >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.Branch)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.Branch-1)); err != nil { + return err + } + } + + // t.Addrs ([]address.Address) (slice) + if len(t.Addrs) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Addrs was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Addrs))); err != nil { + return err + } + for _, v := range t.Addrs { + if err := v.MarshalCBOR(w); err != nil { + return err + } + } + + // t.Types ([]cid.Cid) (slice) + if len(t.Types) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Types was too long") + } + + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Types))); err != nil { + return err + } + for _, v := range t.Types { + if err := cbg.WriteCidBuf(scratch, w, v); err != nil { + return xerrors.Errorf("failed writing cid field t.Types: %w", err) + } + } + return nil +} + +func (t *CallerValidationArgs) UnmarshalCBOR(r io.Reader) error { + *t = CallerValidationArgs{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 3 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.Branch (chaos.CallerValidationBranch) (int64) + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Branch = CallerValidationBranch(extraI) + } + // t.Addrs ([]address.Address) (slice) + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Addrs: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Addrs = make([]address.Address, extra) + } + + for i := 0; i < int(extra); i++ { + + var v address.Address + if err := v.UnmarshalCBOR(br); err != nil { + return err + } + + t.Addrs[i] = v + } + + // t.Types ([]cid.Cid) (slice) + + maj, extra, err = cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Types: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Types = make([]cid.Cid, extra) + } + + for i := 0; i < int(extra); i++ { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("reading cid field t.Types failed: %w", err) + } + t.Types[i] = c + } + + return nil +} + var lengthBufCreateActorArgs = []byte{132} func (t *CreateActorArgs) MarshalCBOR(w io.Writer) error { @@ -730,3 +889,145 @@ func (t *AbortWithArgs) UnmarshalCBOR(r io.Reader) error { } return nil } + +var lengthBufInspectRuntimeReturn = []byte{134} + +func (t *InspectRuntimeReturn) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write(lengthBufInspectRuntimeReturn); err != nil { + return err + } + + scratch := make([]byte, 9) + + // t.Caller (address.Address) (struct) + if err := t.Caller.MarshalCBOR(w); err != nil { + return err + } + + // t.Receiver (address.Address) (struct) + if err := t.Receiver.MarshalCBOR(w); err != nil { + return err + } + + // t.ValueReceived (big.Int) (struct) + if err := t.ValueReceived.MarshalCBOR(w); err != nil { + return err + } + + // t.CurrEpoch (abi.ChainEpoch) (int64) + if t.CurrEpoch >= 0 { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajUnsignedInt, uint64(t.CurrEpoch)); err != nil { + return err + } + } else { + if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajNegativeInt, uint64(-t.CurrEpoch-1)); err != nil { + return err + } + } + + // t.CurrentBalance (big.Int) (struct) + if err := t.CurrentBalance.MarshalCBOR(w); err != nil { + return err + } + + // t.State (chaos.State) (struct) + if err := t.State.MarshalCBOR(w); err != nil { + return err + } + return nil +} + +func (t *InspectRuntimeReturn) UnmarshalCBOR(r io.Reader) error { + *t = InspectRuntimeReturn{} + + br := cbg.GetPeeker(r) + scratch := make([]byte, 8) + + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 6 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.Caller (address.Address) (struct) + + { + + if err := t.Caller.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.Caller: %w", err) + } + + } + // t.Receiver (address.Address) (struct) + + { + + if err := t.Receiver.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.Receiver: %w", err) + } + + } + // t.ValueReceived (big.Int) (struct) + + { + + if err := t.ValueReceived.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.ValueReceived: %w", err) + } + + } + // t.CurrEpoch (abi.ChainEpoch) (int64) + { + maj, extra, err := cbg.CborReadHeaderBuf(br, scratch) + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.CurrEpoch = abi.ChainEpoch(extraI) + } + // t.CurrentBalance (big.Int) (struct) + + { + + if err := t.CurrentBalance.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.CurrentBalance: %w", err) + } + + } + // t.State (chaos.State) (struct) + + { + + if err := t.State.UnmarshalCBOR(br); err != nil { + return xerrors.Errorf("unmarshaling t.State: %w", err) + } + + } + return nil +} diff --git a/conformance/chaos/gen/gen.go b/conformance/chaos/gen/gen.go index 97ea98dc8..ac97da99e 100644 --- a/conformance/chaos/gen/gen.go +++ b/conformance/chaos/gen/gen.go @@ -7,14 +7,16 @@ import ( ) func main() { - if err := gen.WriteTupleEncodersToFile("../cbor_gen.go", "chaos", + if err := gen.WriteTupleEncodersToFile("./cbor_gen.go", "chaos", chaos.State{}, + chaos.CallerValidationArgs{}, chaos.CreateActorArgs{}, chaos.ResolveAddressResponse{}, chaos.SendArgs{}, chaos.SendReturn{}, chaos.MutateStateArgs{}, chaos.AbortWithArgs{}, + chaos.InspectRuntimeReturn{}, ); err != nil { panic(err) } diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index d72beca98..84d72ab67 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -148,6 +148,7 @@ * [StateMinerRecoveries](#StateMinerRecoveries) * [StateMinerSectorCount](#StateMinerSectorCount) * [StateMinerSectors](#StateMinerSectors) + * [StateMsgGasCost](#StateMsgGasCost) * [StateNetworkName](#StateNetworkName) * [StateReadState](#StateReadState) * [StateReplay](#StateReplay) @@ -211,7 +212,7 @@ Response: ```json { "Version": "string value", - "APIVersion": 3584, + "APIVersion": 3840, "BlockDelay": 42 } ``` @@ -3737,6 +3738,45 @@ Inputs: Response: `null` +### StateMsgGasCost +StateMsgGasCost searches for a message in the chain, and returns details of the messages gas costs, including the penalty and miner tip + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: +```json +{ + "Message": { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + "GasUsed": "0", + "BaseFeeBurn": "0", + "OverEstimationBurn": "0", + "MinerPenalty": "0", + "MinerTip": "0", + "Refund": "0", + "TotalCost": "0" +} +``` + ### StateNetworkName StateNetworkName returns the name of the network the node is synced to diff --git a/extern/oni b/extern/oni new file mode 160000 index 000000000..8b7e7d438 --- /dev/null +++ b/extern/oni @@ -0,0 +1 @@ +Subproject commit 8b7e7d438c4cc38a0d2d671876d4590ad20655b3 diff --git a/extern/storage-sealing/sector_state.go b/extern/storage-sealing/sector_state.go index 4e674603d..10b96e504 100644 --- a/extern/storage-sealing/sector_state.go +++ b/extern/storage-sealing/sector_state.go @@ -2,6 +2,38 @@ package sealing type SectorState string +var ExistSectorStateList = map[SectorState]struct{}{ + Empty: {}, + WaitDeals: {}, + Packing: {}, + PreCommit1: {}, + PreCommit2: {}, + PreCommitting: {}, + PreCommitWait: {}, + WaitSeed: {}, + Committing: {}, + SubmitCommit: {}, + CommitWait: {}, + FinalizeSector: {}, + Proving: {}, + FailedUnrecoverable: {}, + SealPreCommit1Failed: {}, + SealPreCommit2Failed: {}, + PreCommitFailed: {}, + ComputeProofFailed: {}, + CommitFailed: {}, + PackingFailed: {}, + FinalizeFailed: {}, + DealsExpired: {}, + RecoverDealIDs: {}, + Faulty: {}, + FaultReported: {}, + FaultedFinal: {}, + Removing: {}, + RemoveFailed: {}, + Removed: {}, +} + const ( UndefinedSectorState SectorState = "" diff --git a/extern/storage-sealing/upgrade_queue.go b/extern/storage-sealing/upgrade_queue.go index 650fdc83d..af2d1827d 100644 --- a/extern/storage-sealing/upgrade_queue.go +++ b/extern/storage-sealing/upgrade_queue.go @@ -67,6 +67,8 @@ func (m *Sealing) tryUpgradeSector(ctx context.Context, params *miner.SectorPreC params.ReplaceSectorDeadline = loc.Deadline params.ReplaceSectorPartition = loc.Partition + log.Infof("replacing sector %d with %d", *replace, params.SectorNumber) + ri, err := m.api.StateSectorGetInfo(ctx, m.maddr, *replace, nil) if err != nil { log.Errorf("error calling StateSectorGetInfo for replaced sector: %+v", err) diff --git a/extern/test-vectors b/extern/test-vectors index 7d3becbeb..6bea015ed 160000 --- a/extern/test-vectors +++ b/extern/test-vectors @@ -1 +1 @@ -Subproject commit 7d3becbeb5b932baed419c43390595b5e5cece12 +Subproject commit 6bea015edddde116001a4251dce3c4a9966c25d9 diff --git a/go.mod b/go.mod index 65c2addb8..41bc727b3 100644 --- a/go.mod +++ b/go.mod @@ -25,9 +25,9 @@ require ( github.com/filecoin-project/go-bitfield v0.2.0 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 - github.com/filecoin-project/go-data-transfer v0.6.3 + github.com/filecoin-project/go-data-transfer v0.6.4 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f - github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c + github.com/filecoin-project/go-fil-markets v0.6.1 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 github.com/filecoin-project/go-multistore v0.0.3 github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20 diff --git a/go.sum b/go.sum index 77b556fb2..11d607034 100644 --- a/go.sum +++ b/go.sum @@ -222,12 +222,12 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-data-transfer v0.6.3 h1:7TLwm8nuodHYD/uiwJjKc/PGRR+LwqM8jmlZqgWuUfY= -github.com/filecoin-project/go-data-transfer v0.6.3/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM= +github.com/filecoin-project/go-data-transfer v0.6.4 h1:Q08ABa+cOTOLoAyHeA94fPLcwu53p6eeAaxMxQb0m0A= +github.com/filecoin-project/go-data-transfer v0.6.4/go.mod h1:PmBKVXkhh67/tnEdJXQwDHl5mT+7Tbcwe1NPninqhnM= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c h1:YGoyYmELQ0LHwDj/WcOvY3oYt+3iM0wdrAhqJQUAIy4= -github.com/filecoin-project/go-fil-markets v0.6.1-0.20200911011457-2959ccca6a3c/go.mod h1:PLr9svZxsnHkae1Ky7+66g7fP9AlneVxIVu+oSMq56A= +github.com/filecoin-project/go-fil-markets v0.6.1 h1:qCFLcVkUCbxwEfH/6EcqTuQvibXt/TXZr+vh8tWv/BQ= +github.com/filecoin-project/go-fil-markets v0.6.1/go.mod h1:dBJl59dAyl8+cGVb/ONPlEQW4+YzhjI3d6bxLfHVpX0= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0= diff --git a/miner/miner.go b/miner/miner.go index bcfdb46c1..2d8591992 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -139,6 +139,7 @@ func (m *Miner) niceSleep(d time.Duration) bool { case <-build.Clock.After(d): return true case <-m.stop: + log.Infow("received interrupt while trying to sleep in mining cycle") return false } } @@ -169,7 +170,9 @@ func (m *Miner) mine(ctx context.Context) { prebase, err := m.GetBestMiningCandidate(ctx) if err != nil { log.Errorf("failed to get best mining candidate: %s", err) - m.niceSleep(time.Second * 5) + if !m.niceSleep(time.Second * 5) { + break + } continue } @@ -199,7 +202,9 @@ func (m *Miner) mine(ctx context.Context) { _, err = m.api.BeaconGetEntry(ctx, prebase.TipSet.Height()+prebase.NullRounds+1) if err != nil { log.Errorf("failed getting beacon entry: %s", err) - m.niceSleep(time.Second) + if !m.niceSleep(time.Second) { + break + } continue } @@ -208,7 +213,9 @@ func (m *Miner) mine(ctx context.Context) { if base.TipSet.Equals(lastBase.TipSet) && lastBase.NullRounds == base.NullRounds { log.Warnf("BestMiningCandidate from the previous round: %s (nulls:%d)", lastBase.TipSet.Cids(), lastBase.NullRounds) - m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) + if !m.niceSleep(time.Duration(build.BlockDelaySecs) * time.Second) { + break + } continue } @@ -217,7 +224,9 @@ func (m *Miner) mine(ctx context.Context) { b, err := m.mineOne(ctx, base) if err != nil { log.Errorf("mining block failed: %+v", err) - m.niceSleep(time.Second) + if !m.niceSleep(time.Second) { + break + } onDone(false, 0, err) continue } diff --git a/node/builder.go b/node/builder.go index 284f13a4a..d850846dc 100644 --- a/node/builder.go +++ b/node/builder.go @@ -384,7 +384,7 @@ func StorageMiner(out *api.StorageMiner) Option { func(s *Settings) error { resAPI := &impl.StorageMinerAPI{} - s.invokes[ExtractApiKey] = fx.Extract(resAPI) + s.invokes[ExtractApiKey] = fx.Populate(resAPI) *out = resAPI return nil }, @@ -511,7 +511,7 @@ func FullAPI(out *api.FullNode) Option { }, func(s *Settings) error { resAPI := &impl.FullNodeAPI{} - s.invokes[ExtractApiKey] = fx.Extract(resAPI) + s.invokes[ExtractApiKey] = fx.Populate(resAPI) *out = resAPI return nil }, diff --git a/node/impl/full/state.go b/node/impl/full/state.go index f9fcdab8a..0bd776a60 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1225,3 +1225,52 @@ func (a *StateAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetK return a.StateManager.GetCirculatingSupplyDetailed(ctx, ts.Height(), sTree) } + +func (a *StateAPI) StateMsgGasCost(ctx context.Context, inputMsg cid.Cid, tsk types.TipSetKey) (*api.MsgGasCost, error) { + var msg cid.Cid + var ts *types.TipSet + var err error + if tsk != types.EmptyTSK { + msg = inputMsg + ts, err = a.Chain.LoadTipSet(tsk) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) + } + } else { + mlkp, err := a.StateSearchMsg(ctx, inputMsg) + if err != nil { + return nil, xerrors.Errorf("searching for msg %s: %w", inputMsg, err) + } + if mlkp == nil { + return nil, xerrors.Errorf("didn't find msg %s", inputMsg) + } + + executionTs, err := a.Chain.GetTipSetFromKey(mlkp.TipSet) + if err != nil { + return nil, xerrors.Errorf("loading tipset %s: %w", mlkp.TipSet, err) + } + + ts, err = a.Chain.LoadTipSet(executionTs.Parents()) + if err != nil { + return nil, xerrors.Errorf("loading parent tipset %s: %w", mlkp.TipSet, err) + } + + msg = mlkp.Message + } + + m, r, err := a.StateManager.Replay(ctx, ts, msg) + if err != nil { + return nil, err + } + + return &api.MsgGasCost{ + Message: msg, + GasUsed: big.NewInt(r.GasUsed), + BaseFeeBurn: r.GasCosts.BaseFeeBurn, + OverEstimationBurn: r.GasCosts.OverEstimationBurn, + MinerPenalty: r.GasCosts.MinerPenalty, + MinerTip: r.GasCosts.MinerTip, + Refund: r.GasCosts.Refund, + TotalCost: big.Sub(m.RequiredFunds(), r.GasCosts.Refund), + }, nil +} diff --git a/scripts/lotus-miner.service b/scripts/lotus-miner.service index c079f44a9..54c48d411 100644 --- a/scripts/lotus-miner.service +++ b/scripts/lotus-miner.service @@ -2,7 +2,7 @@ Description=Lotus Miner After=network.target After=lotus-daemon.service -Requires=lotus-daemon.service +Wants=lotus-daemon.service [Service] ExecStart=/usr/local/bin/lotus-miner run diff --git a/storage/addresses.go b/storage/addresses.go index bef845367..2388b9fca 100644 --- a/storage/addresses.go +++ b/storage/addresses.go @@ -60,7 +60,7 @@ func AddressFor(ctx context.Context, a addrSelectApi, mi api.MinerInfo, use Addr return addr, nil } - log.Warnw("control address didn't have enough funds for PoSt message", "address", addr, "required", types.FIL(minFunds), "balance", types.FIL(b)) + log.Warnw("control address didn't have enough funds for window post message", "address", addr, "required", types.FIL(minFunds), "balance", types.FIL(b)) } // Try to use the owner account if we can, fallback to worker if we can't diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 7aa90bbba..463c7a3c5 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -78,7 +78,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, posts, err := s.runPost(ctx, *deadline, ts) if err != nil { - log.Errorf("runPost failed: %+v", err) + log.Errorf("run window post failed: %+v", err) s.failPost(err, deadline) return } @@ -92,7 +92,7 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, post := &posts[i] sm, err := s.submitPost(ctx, post) if err != nil { - log.Errorf("submitPost failed: %+v", err) + log.Errorf("submit window post failed: %+v", err) s.failPost(err, deadline) } else { recordProofsEvent(post.Partitions, sm.Cid()) @@ -397,7 +397,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty rand, err := s.api.ChainGetRandomnessFromBeacon(ctx, ts.Key(), crypto.DomainSeparationTag_WindowedPoStChallengeSeed, di.Challenge, buf.Bytes()) if err != nil { - return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), di, err) + return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err) } // Get the partitions for the given deadline @@ -431,7 +431,9 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty postSkipped := bitfield.New() var postOut []proof.PoStProof somethingToProve := true + for retries := 0; retries < 5; retries++ { + var partitions []miner.PoStPartition var sinfos []proof.SectorInfo for partIdx, partition := range batch { // TODO: Can do this in parallel @@ -477,7 +479,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } sinfos = append(sinfos, ssi...) - params.Partitions = append(params.Partitions, miner.PoStPartition{ + partitions = append(partitions, miner.PoStPartition{ Index: uint64(batchPartitionStartIdx + partIdx), Skipped: skipped, }) @@ -490,7 +492,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty } // Generate proof - log.Infow("running windowPost", + log.Infow("running window post", "chain-random", rand, "deadline", di, "height", ts.Height(), @@ -507,20 +509,22 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty postOut, ps, err = s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand)) elapsed := time.Since(tsStart) - log.Infow("computing window PoSt", "batch", batchIdx, "elapsed", elapsed) + log.Infow("computing window post", "batch", batchIdx, "elapsed", elapsed) if err == nil { // Proof generation successful, stop retrying + params.Partitions = append(params.Partitions, partitions...) + break } // Proof generation failed, so retry if len(ps) == 0 { - return nil, xerrors.Errorf("running post failed: %w", err) + return nil, xerrors.Errorf("running window post failed: %w", err) } - log.Warnw("generate window PoSt skipped sectors", "sectors", ps, "error", err, "try", retries) + log.Warnw("generate window post skipped sectors", "sectors", ps, "error", err, "try", retries) skipCount += uint64(len(ps)) for _, sector := range ps { @@ -547,7 +551,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty commEpoch := di.Open commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil) if err != nil { - return nil, xerrors.Errorf("failed to get chain randomness for windowPost (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) + return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err) } for i := range posts { @@ -644,7 +648,7 @@ func (s *WindowPoStScheduler) submitPost(ctx context.Context, proof *miner.Submi enc, aerr := actors.SerializeParams(proof) if aerr != nil { - return nil, xerrors.Errorf("could not serialize submit post parameters: %w", aerr) + return nil, xerrors.Errorf("could not serialize submit window post parameters: %w", aerr) } msg := &types.Message{ @@ -705,7 +709,7 @@ func (s *WindowPoStScheduler) setSender(ctx context.Context, msg *types.Message, pa, err := AddressFor(ctx, s.api, mi, PoStAddr, minFunds) if err != nil { - log.Errorw("error selecting address for post", "error", err) + log.Errorw("error selecting address for window post", "error", err) msg.From = s.worker return } diff --git a/storage/wdpost_sched.go b/storage/wdpost_sched.go index 037f4e481..d130d801c 100644 --- a/storage/wdpost_sched.go +++ b/storage/wdpost_sched.go @@ -110,7 +110,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { select { case changes, ok := <-notifs: if !ok { - log.Warn("WindowPoStScheduler notifs channel closed") + log.Warn("window post scheduler notifs channel closed") notifs = nil continue } @@ -151,10 +151,10 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) { } if err := s.revert(ctx, lowest); err != nil { - log.Error("handling head reverts in windowPost sched: %+v", err) + log.Error("handling head reverts in window post sched: %+v", err) } if err := s.update(ctx, highest); err != nil { - log.Error("handling head updates in windowPost sched: %+v", err) + log.Error("handling head updates in window post sched: %+v", err) } span.End() @@ -184,7 +184,7 @@ func (s *WindowPoStScheduler) revert(ctx context.Context, newLowest *types.TipSe func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) error { if new == nil { - return xerrors.Errorf("no new tipset in WindowPoStScheduler.update") + return xerrors.Errorf("no new tipset in window post sched update") } di, err := s.api.StateMinerProvingDeadline(ctx, s.actor, new.Key()) @@ -206,7 +206,7 @@ func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) err // (Need to get correct deadline above, which is tricky) if di.Open+StartConfidence >= new.Height() { - log.Info("not starting windowPost yet, waiting for startconfidence", di.Open, di.Open+StartConfidence, new.Height()) + log.Info("not starting window post yet, waiting for startconfidence", di.Open, di.Open+StartConfidence, new.Height()) return nil } @@ -216,7 +216,7 @@ func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) err s.activeEPS = 0 } s.failLk.Unlock()*/ - log.Infof("at %d, doPost for P %d, dd %d", new.Height(), di.PeriodStart, di.Index) + log.Infof("at %d, do window post for P %d, dd %d", new.Height(), di.PeriodStart, di.Index) s.doPost(ctx, di, new) @@ -238,7 +238,7 @@ func (s *WindowPoStScheduler) abortActivePoSt() { } }) - log.Warnf("Aborting Window PoSt (Deadline: %+v)", s.activeDeadline) + log.Warnf("Aborting window post (Deadline: %+v)", s.activeDeadline) } s.activeDeadline = nil diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index ae79d9273..e50ac953f 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -189,7 +189,7 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti pl.AddPoint(p) } { - blks := len(cids) + blks := int64(len(cids)) p = NewPoint("chain.gas_fill_ratio", float64(totalGasLimit)/float64(blks*build.BlockGasTarget)) pl.AddPoint(p) p = NewPoint("chain.gas_capacity_ratio", float64(totalUniqGasLimit)/float64(blks*build.BlockGasTarget))