Merge remote-tracking branch 'origin/master' into feat/post-worker

This commit is contained in:
Łukasz Magiera
2022-03-18 10:54:44 +01:00
169 changed files with 2319 additions and 871 deletions
+1
View File
@@ -1,3 +1,4 @@
//stm: #unit
package main
import (
+2
View File
@@ -1,3 +1,4 @@
//stm: #unit
package main
import (
@@ -8,6 +9,7 @@ import (
)
func TestRateLimit(t *testing.T) {
//stm: @CMD_LIMITER_GET_IP_LIMITER_001, @CMD_LIMITER_GET_WALLET_LIMITER_001
limiter := NewLimiter(LimiterConfig{
TotalRate: time.Second,
TotalBurst: 20,
+3
View File
@@ -1,3 +1,4 @@
//stm: #unit
package main
import (
@@ -9,6 +10,7 @@ import (
)
func TestAppendCIDsToWindow(t *testing.T) {
//stm: @CMD_HEALTH_APPEND_CIDS_001
assert := assert.New(t)
var window CidWindow
threshold := 3
@@ -27,6 +29,7 @@ func TestAppendCIDsToWindow(t *testing.T) {
}
func TestCheckWindow(t *testing.T) {
//stm: @CMD_HEALTH_APPEND_CIDS_001, @CMD_HEALTH_CHECK_WINDOW_001
assert := assert.New(t)
threshold := 3
+2
View File
@@ -1,3 +1,4 @@
//stm: #unit
package main
import (
@@ -23,6 +24,7 @@ import (
)
func TestWorkerKeyChange(t *testing.T) {
//stm: @OTHER_WORKER_KEY_CHANGE_001
if testing.Short() {
t.Skip("skipping test in short mode")
}
+2
View File
@@ -1,3 +1,4 @@
//stm: #integration
package main
import (
@@ -49,6 +50,7 @@ func TestMinerAllInfo(t *testing.T) {
t.Run("pre-info-all", run)
//stm: @CLIENT_DATA_IMPORT_001, @CLIENT_STORAGE_DEALS_GET_001
dh := kit.NewDealHarness(t, client, miner, miner)
deal, res, inPath := dh.MakeOnlineDeal(context.Background(), kit.MakeFullDealParams{Rseed: 6})
outPath := dh.PerformRetrieval(context.Background(), deal, res.Root, false)
+1
View File
@@ -466,6 +466,7 @@ var stateOrder = map[sealing.SectorState]stateMeta{}
var stateList = []stateMeta{
{col: 39, state: "Total"},
{col: color.FgGreen, state: sealing.Proving},
{col: color.FgGreen, state: sealing.Available},
{col: color.FgGreen, state: sealing.UpdateActivating},
{col: color.FgBlue, state: sealing.Empty},
+2 -57
View File
@@ -11,9 +11,6 @@ import (
"strings"
"time"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/docker/go-units"
"github.com/fatih/color"
cbor "github.com/ipfs/go-ipld-cbor"
@@ -56,7 +53,6 @@ var sectorsCmd = &cli.Command{
sectorsRemoveCmd,
sectorsSnapUpCmd,
sectorsSnapAbortCmd,
sectorsMarkForUpgradeCmd,
sectorsStartSealCmd,
sectorsSealDelayCmd,
sectorsCapacityCollateralCmd,
@@ -351,7 +347,7 @@ var sectorsListCmd = &cli.Command{
if cctx.Bool("unproven") {
for state := range sealing.ExistSectorStateList {
if state == sealing.Proving {
if state == sealing.Proving || state == sealing.Available {
continue
}
states = append(states, api.SectorState(state))
@@ -437,7 +433,7 @@ var sectorsListCmd = &cli.Command{
const verifiedPowerGainMul = 9
dw, vp := .0, .0
estimate := st.Expiration-st.Activation <= 0
estimate := (st.Expiration-st.Activation <= 0) || sealing.IsUpgradeState(sealing.SectorState(st.State))
if !estimate {
rdw := big.Add(st.DealWeight, st.VerifiedDealWeight)
dw = float64(big.Div(rdw, big.NewInt(int64(st.Expiration-st.Activation))).Uint64())
@@ -1568,57 +1564,6 @@ var sectorsSnapAbortCmd = &cli.Command{
},
}
var sectorsMarkForUpgradeCmd = &cli.Command{
Name: "mark-for-upgrade",
Usage: "Mark a committed capacity sector for replacement by a sector with deals",
ArgsUsage: "<sectorNum>",
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 1 {
return lcli.ShowHelp(cctx, xerrors.Errorf("must pass sector number"))
}
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
api, nCloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer nCloser()
ctx := lcli.ReqContext(cctx)
nv, err := api.StateNetworkVersion(ctx, types.EmptyTSK)
if err != nil {
return xerrors.Errorf("failed to get network version: %w", err)
}
if nv >= network.Version15 {
return xerrors.Errorf("classic cc upgrades disabled v15 and beyond, use `snap-up`")
}
// disable mark for upgrade two days before the ntwk v15 upgrade
// TODO: remove the following block in v1.15.1
head, err := api.ChainHead(ctx)
if err != nil {
return xerrors.Errorf("failed to get chain head: %w", err)
}
twoDays := abi.ChainEpoch(2 * builtin.EpochsInDay)
if head.Height() > (build.UpgradeOhSnapHeight - twoDays) {
return xerrors.Errorf("OhSnap is coming soon, " +
"please use `snap-up` to upgrade your cc sectors after the network v15 upgrade!")
}
id, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
if err != nil {
return xerrors.Errorf("could not parse sector number: %w", err)
}
return nodeApi.SectorMarkForUpgrade(ctx, abi.SectorNumber(id), false)
},
}
var sectorsStartSealCmd = &cli.Command{
Name: "seal",
Usage: "Manually start sealing a sector (filling any unused space with junk)",
+7 -4
View File
@@ -598,7 +598,7 @@ var storageListSectorsCmd = &cli.Command{
ft storiface.SectorFileType
urls string
primary, seal, store bool
primary, copy, main, seal, store bool
state api.SectorState
}
@@ -626,8 +626,11 @@ var storageListSectorsCmd = &cli.Command{
urls: strings.Join(info.URLs, ";"),
primary: info.Primary,
seal: info.CanSeal,
store: info.CanStore,
copy: !info.Primary && len(si) > 1,
main: !info.Primary && len(si) == 1, // only copy, but not primary
seal: info.CanSeal,
store: info.CanStore,
state: st.State,
})
@@ -680,7 +683,7 @@ var storageListSectorsCmd = &cli.Command{
"Sector": e.id,
"Type": e.ft.String(),
"State": color.New(stateOrder[sealing.SectorState(e.state)].col).Sprint(e.state),
"Primary": maybeStr(e.seal, color.FgGreen, "primary"),
"Primary": maybeStr(e.primary, color.FgGreen, "primary") + maybeStr(e.copy, color.FgBlue, "copy") + maybeStr(e.main, color.FgRed, "main"),
"Path use": maybeStr(e.seal, color.FgMagenta, "seal ") + maybeStr(e.store, color.FgCyan, "store"),
"URLs": e.urls,
}
+118
View File
@@ -0,0 +1,118 @@
package main
import (
"context"
"fmt"
"io"
"unicode/utf8"
"github.com/filecoin-project/lotus/chain/actors/builtin/market"
"github.com/filecoin-project/lotus/chain/consensus/filcns"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/chain/state"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/specs-actors/v4/actors/util/adt"
"github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/urfave/cli/v2"
)
var dealLabelCmd = &cli.Command{
Name: "deal-label",
Usage: "Scrape state to report on how many deals have non UTF-8 labels",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "repo",
Value: "~/.lotus",
},
},
Action: func(cctx *cli.Context) error {
ctx := context.TODO()
if !cctx.Args().Present() {
return fmt.Errorf("must pass state root")
}
sroot, err := cid.Decode(cctx.Args().First())
if err != nil {
return fmt.Errorf("failed to parse input: %w", err)
}
fsrepo, err := repo.NewFS(cctx.String("repo"))
if err != nil {
return err
}
lkrepo, err := fsrepo.Lock(repo.FullNode)
if err != nil {
return err
}
defer lkrepo.Close() //nolint:errcheck
bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
if err != nil {
return fmt.Errorf("failed to open blockstore: %w", err)
}
defer func() {
if c, ok := bs.(io.Closer); ok {
if err := c.Close(); err != nil {
log.Warnf("failed to close blockstore: %s", err)
}
}
}()
mds, err := lkrepo.Datastore(context.Background(), "/metadata")
if err != nil {
return err
}
cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
defer cs.Close() //nolint:errcheck
cst := cbor.NewCborStore(bs)
store := adt.WrapStore(ctx, cst)
tree, err := state.LoadStateTree(cst, sroot)
if err != nil {
return err
}
ma, err := tree.GetActor(market.Address)
if err != nil {
return err
}
ms, err := market.Load(store, ma)
if err != nil {
return err
}
ps, err := ms.Proposals()
if err != nil {
return err
}
var deals []abi.DealID
if err = ps.ForEach(func(id abi.DealID, dp market.DealProposal) error {
if !utf8.Valid([]byte(dp.Label)) {
deals = append(deals, id)
}
return nil
}); err != nil {
return err
}
fmt.Println("there are ", len(deals), " bad labels")
for _, d := range deals {
fmt.Print(d, " ")
}
return nil
},
}
+1
View File
@@ -22,6 +22,7 @@ func main() {
bitFieldCmd,
cronWcCmd,
frozenMinersCmd,
dealLabelCmd,
keyinfoCmd,
jwtCmd,
noncefix,
+300
View File
@@ -8,6 +8,8 @@ import (
"github.com/filecoin-project/go-state-types/abi"
miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
msig5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/multisig"
"github.com/filecoin-project/go-address"
@@ -29,6 +31,9 @@ var minerMultisigsCmd = &cli.Command{
mmApproveWithdrawBalance,
mmProposeChangeOwner,
mmApproveChangeOwner,
mmProposeChangeWorker,
mmConfirmChangeWorker,
mmProposeControlSet,
},
Flags: []cli.Flag{
&cli.StringFlag{
@@ -368,6 +373,301 @@ var mmApproveChangeOwner = &cli.Command{
},
}
var mmProposeChangeWorker = &cli.Command{
Name: "propose-change-worker",
Usage: "Propose an worker address change",
ArgsUsage: "[newWorker]",
Action: func(cctx *cli.Context) error {
if !cctx.Args().Present() {
return fmt.Errorf("must pass new worker address")
}
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
multisigAddr, sender, minerAddr, err := getInputs(cctx)
if err != nil {
return err
}
na, err := address.NewFromString(cctx.Args().First())
if err != nil {
return err
}
newAddr, err := api.StateLookupID(ctx, na, types.EmptyTSK)
if err != nil {
return err
}
mi, err := api.StateMinerInfo(ctx, minerAddr, types.EmptyTSK)
if err != nil {
return err
}
if mi.NewWorker.Empty() {
if mi.Worker == newAddr {
return fmt.Errorf("worker address already set to %s", na)
}
} else {
if mi.NewWorker == newAddr {
fmt.Fprintf(cctx.App.Writer, "Worker key change to %s successfully proposed.\n", na)
fmt.Fprintf(cctx.App.Writer, "Call 'confirm-change-worker' at or after height %d to complete.\n", mi.WorkerChangeEpoch)
return fmt.Errorf("change to worker address %s already pending", na)
}
}
cwp := &miner2.ChangeWorkerAddressParams{
NewWorker: newAddr,
NewControlAddrs: mi.ControlAddresses,
}
fmt.Fprintf(cctx.App.Writer, "newAddr: %s\n", newAddr)
fmt.Fprintf(cctx.App.Writer, "NewControlAddrs: %s\n", mi.ControlAddresses)
sp, err := actors.SerializeParams(cwp)
if err != nil {
return xerrors.Errorf("serializing params: %w", err)
}
pcid, err := api.MsigPropose(ctx, multisigAddr, minerAddr, big.Zero(), sender, uint64(miner.Methods.ChangeWorkerAddress), sp)
if err != nil {
return xerrors.Errorf("proposing message: %w", err)
}
fmt.Fprintln(cctx.App.Writer, "Propose Message CID:", pcid)
// wait for it to get mined into a block
wait, err := api.StateWaitMsg(ctx, pcid, build.MessageConfidence)
if err != nil {
return err
}
// check it executed successfully
if wait.Receipt.ExitCode != 0 {
fmt.Fprintln(cctx.App.Writer, "Propose worker change tx failed!")
return err
}
var retval msig5.ProposeReturn
if err := retval.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil {
return fmt.Errorf("failed to unmarshal propose return value: %w", err)
}
fmt.Printf("Transaction ID: %d\n", retval.TxnID)
if retval.Applied {
fmt.Printf("Transaction was executed during propose\n")
fmt.Printf("Exit Code: %d\n", retval.Code)
fmt.Printf("Return Value: %x\n", retval.Ret)
}
return nil
},
}
var mmConfirmChangeWorker = &cli.Command{
Name: "confirm-change-worker",
Usage: "Confirm an worker address change",
ArgsUsage: "[newWorker]",
Action: func(cctx *cli.Context) error {
if !cctx.Args().Present() {
return fmt.Errorf("must pass new worker address")
}
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
multisigAddr, sender, minerAddr, err := getInputs(cctx)
if err != nil {
return err
}
na, err := address.NewFromString(cctx.Args().First())
if err != nil {
return err
}
newAddr, err := api.StateLookupID(ctx, na, types.EmptyTSK)
if err != nil {
return err
}
mi, err := api.StateMinerInfo(ctx, minerAddr, types.EmptyTSK)
if err != nil {
return err
}
if mi.NewWorker.Empty() {
return xerrors.Errorf("no worker key change proposed")
} else if mi.NewWorker != newAddr {
return xerrors.Errorf("worker key %s does not match current worker key proposal %s", newAddr, mi.NewWorker)
}
if head, err := api.ChainHead(ctx); err != nil {
return xerrors.Errorf("failed to get the chain head: %w", err)
} else if head.Height() < mi.WorkerChangeEpoch {
return xerrors.Errorf("worker key change cannot be confirmed until %d, current height is %d", mi.WorkerChangeEpoch, head.Height())
}
pcid, err := api.MsigPropose(ctx, multisigAddr, minerAddr, big.Zero(), sender, uint64(miner.Methods.ConfirmUpdateWorkerKey), nil)
if err != nil {
return xerrors.Errorf("proposing message: %w", err)
}
fmt.Fprintln(cctx.App.Writer, "Propose Message CID:", pcid)
// wait for it to get mined into a block
wait, err := api.StateWaitMsg(ctx, pcid, build.MessageConfidence)
if err != nil {
return err
}
// check it executed successfully
if wait.Receipt.ExitCode != 0 {
fmt.Fprintln(cctx.App.Writer, "Propose worker change tx failed!")
return err
}
var retval msig5.ProposeReturn
if err := retval.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil {
return fmt.Errorf("failed to unmarshal propose return value: %w", err)
}
fmt.Printf("Transaction ID: %d\n", retval.TxnID)
if retval.Applied {
fmt.Printf("Transaction was executed during propose\n")
fmt.Printf("Exit Code: %d\n", retval.Code)
fmt.Printf("Return Value: %x\n", retval.Ret)
}
return nil
},
}
var mmProposeControlSet = &cli.Command{
Name: "propose-control-set",
Usage: "Set control address(-es)",
ArgsUsage: "[...address]",
Action: func(cctx *cli.Context) error {
if !cctx.Args().Present() {
return fmt.Errorf("must pass new owner address")
}
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
multisigAddr, sender, minerAddr, err := getInputs(cctx)
if err != nil {
return err
}
mi, err := api.StateMinerInfo(ctx, minerAddr, types.EmptyTSK)
if err != nil {
return err
}
del := map[address.Address]struct{}{}
existing := map[address.Address]struct{}{}
for _, controlAddress := range mi.ControlAddresses {
ka, err := api.StateAccountKey(ctx, controlAddress, types.EmptyTSK)
if err != nil {
return err
}
del[ka] = struct{}{}
existing[ka] = struct{}{}
}
var toSet []address.Address
for i, as := range cctx.Args().Slice() {
a, err := address.NewFromString(as)
if err != nil {
return xerrors.Errorf("parsing address %d: %w", i, err)
}
ka, err := api.StateAccountKey(ctx, a, types.EmptyTSK)
if err != nil {
return err
}
// make sure the address exists on chain
_, err = api.StateLookupID(ctx, ka, types.EmptyTSK)
if err != nil {
return xerrors.Errorf("looking up %s: %w", ka, err)
}
delete(del, ka)
toSet = append(toSet, ka)
}
for a := range del {
fmt.Println("Remove", a)
}
for _, a := range toSet {
if _, exists := existing[a]; !exists {
fmt.Println("Add", a)
}
}
cwp := &miner2.ChangeWorkerAddressParams{
NewWorker: mi.Worker,
NewControlAddrs: toSet,
}
sp, err := actors.SerializeParams(cwp)
if err != nil {
return xerrors.Errorf("serializing params: %w", err)
}
pcid, err := api.MsigPropose(ctx, multisigAddr, minerAddr, big.Zero(), sender, uint64(miner.Methods.ChangeWorkerAddress), sp)
if err != nil {
return xerrors.Errorf("proposing message: %w", err)
}
fmt.Fprintln(cctx.App.Writer, "Propose Message CID:", pcid)
// wait for it to get mined into a block
wait, err := api.StateWaitMsg(ctx, pcid, build.MessageConfidence)
if err != nil {
return err
}
// check it executed successfully
if wait.Receipt.ExitCode != 0 {
fmt.Fprintln(cctx.App.Writer, "Propose worker change tx failed!")
return err
}
var retval msig5.ProposeReturn
if err := retval.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil {
return fmt.Errorf("failed to unmarshal propose return value: %w", err)
}
fmt.Printf("Transaction ID: %d\n", retval.TxnID)
if retval.Applied {
fmt.Printf("Transaction was executed during propose\n")
fmt.Printf("Exit Code: %d\n", retval.Code)
fmt.Printf("Return Value: %x\n", retval.Ret)
}
return nil
},
}
func getInputs(cctx *cli.Context) (address.Address, address.Address, address.Address, error) {
multisigAddr, err := address.NewFromString(cctx.String("multisig"))
if err != nil {
@@ -44,7 +44,7 @@ type BlockBuilder struct {
parentTs *types.TipSet
parentSt *state.StateTree
vm *vm.VM
vm *vm.LegacyVM
sm *stmgr.StateManager
gasTotal int64
@@ -73,9 +73,9 @@ func NewBlockBuilder(ctx context.Context, logger *zap.SugaredLogger, sm *stmgr.S
parentSt: parentSt,
}
// Then we construct a VM to execute messages for gas estimation.
// Then we construct a LegacyVM to execute messages for gas estimation.
//
// Most parts of this VM are "real" except:
// Most parts of this LegacyVM are "real" except:
// 1. We don't charge a fee.
// 2. The runtime has "fake" proof logic.
// 3. We don't actually save any of the results.
@@ -92,7 +92,7 @@ func NewBlockBuilder(ctx context.Context, logger *zap.SugaredLogger, sm *stmgr.S
BaseFee: abi.NewTokenAmount(0),
LookbackState: stmgr.LookbackStateGetterForTipset(sm, parentTs),
}
bb.vm, err = vm.NewVM(bb.ctx, vmopt)
bb.vm, err = vm.NewLegacyVM(bb.ctx, vmopt)
if err != nil {
return nil, err
}
@@ -190,12 +190,12 @@ func (bb *BlockBuilder) PushMessage(msg *types.Message) (*types.MessageReceipt,
return &ret.MessageReceipt, nil
}
// ActorStore returns the VM's current (pending) blockstore.
// ActorStore returns the LegacyVM's current (pending) blockstore.
func (bb *BlockBuilder) ActorStore() adt.Store {
return bb.vm.ActorStore(bb.ctx)
}
// StateTree returns the VM's current (pending) state-tree. This includes any changes made by
// StateTree returns the LegacyVM's current (pending) state-tree. This includes any changes made by
// successfully pushed messages.
//
// You probably want ParentStateTree
@@ -1,3 +1,4 @@
//stm: #unit
package stages
import (
@@ -13,6 +14,7 @@ import (
)
func TestCommitQueue(t *testing.T) {
//stm: @CMD_COMMIT_Q_ENQUEUE_COMMIT_001
var q commitQueue
addr1, err := address.NewIDAddress(1000)
require.NoError(t, err)
@@ -46,6 +48,7 @@ func TestCommitQueue(t *testing.T) {
SectorNumber: 6,
}))
//stm: @CMD_COMMIT_Q_ADVANCE_EPOCH_001, @CMD_COMMIT_Q_NEXT_MINER_001
epoch := abi.ChainEpoch(0)
q.advanceEpoch(epoch)
_, _, ok := q.nextMiner()
+2
View File
@@ -1,3 +1,4 @@
//stm: #unit
package main
import (
@@ -10,6 +11,7 @@ import (
)
func TestProtocolCodenames(t *testing.T) {
//stm: @OTHER_IMPLEMENTATION_EPOCH_CODENAMES_001
if height := abi.ChainEpoch(100); GetProtocolCodename(height) != "genesis" {
t.Fatal("expected genesis codename")
}