Merge pull request #4640 from filecoin-project/feat/compact-sectors-numbers-cmd

add a command for compacting sector numbers bitfield

chain/actors/builtin/miner/actor.go.template

@@ -105,6 +105,7 @@ type State interface {
         // UnallocatedSectorNumbers returns up to count unallocated sector numbers (or less than
         // count if there aren't enough).
 	UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
+	GetAllocatedSectors() (*bitfield.BitField, error)
 
 	// Note that ProvingPeriodStart is deprecated and will be renamed / removed in a future version of actors
 	GetProvingPeriodStart() (abi.ChainEpoch, error)

chain/actors/builtin/miner/miner.go

@@ -164,6 +164,7 @@ type State interface {
 	// UnallocatedSectorNumbers returns up to count unallocated sector numbers (or less than
 	// count if there aren't enough).
 	UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
+	GetAllocatedSectors() (*bitfield.BitField, error)
 
 	// Note that ProvingPeriodStart is deprecated and will be renamed / removed in a future version of actors
 	GetProvingPeriodStart() (abi.ChainEpoch, error)

chain/actors/builtin/miner/state.go.template

@@ -318,6 +318,15 @@ func (s *state{{.v}}) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, e
 	return sectors, nil
 }
 
+func (s *state{{.v}}) GetAllocatedSectors() (*bitfield.BitField, error) {
+	var allocatedSectors bitfield.BitField
+	if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
+		return nil, err
+	}
+
+	return &allocatedSectors, nil
+}
+
 func (s *state{{.v}}) LoadDeadline(idx uint64) (Deadline, error) {
 	dls, err := s.State.LoadDeadlines(s.store)
 	if err != nil {

chain/actors/builtin/miner/v0.go

@@ -311,6 +311,15 @@ func (s *state0) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
 	return sectors, nil
 }
 
+func (s *state0) GetAllocatedSectors() (*bitfield.BitField, error) {
+	var allocatedSectors bitfield.BitField
+	if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
+		return nil, err
+	}
+
+	return &allocatedSectors, nil
+}
+
 func (s *state0) LoadDeadline(idx uint64) (Deadline, error) {
 	dls, err := s.State.LoadDeadlines(s.store)
 	if err != nil {

chain/actors/builtin/miner/v2.go

@@ -309,6 +309,15 @@ func (s *state2) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
 	return sectors, nil
 }
 
+func (s *state2) GetAllocatedSectors() (*bitfield.BitField, error) {
+	var allocatedSectors bitfield.BitField
+	if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
+		return nil, err
+	}
+
+	return &allocatedSectors, nil
+}
+
 func (s *state2) LoadDeadline(idx uint64) (Deadline, error) {
 	dls, err := s.State.LoadDeadlines(s.store)
 	if err != nil {

chain/actors/builtin/miner/v3.go

@@ -311,6 +311,15 @@ func (s *state3) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
 	return sectors, nil
 }
 
+func (s *state3) GetAllocatedSectors() (*bitfield.BitField, error) {
+	var allocatedSectors bitfield.BitField
+	if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
+		return nil, err
+	}
+
+	return &allocatedSectors, nil
+}
+
 func (s *state3) LoadDeadline(idx uint64) (Deadline, error) {
 	dls, err := s.State.LoadDeadlines(s.store)
 	if err != nil {

chain/actors/builtin/miner/v4.go

@@ -311,6 +311,15 @@ func (s *state4) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
 	return sectors, nil
 }
 
+func (s *state4) GetAllocatedSectors() (*bitfield.BitField, error) {
+	var allocatedSectors bitfield.BitField
+	if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
+		return nil, err
+	}
+
+	return &allocatedSectors, nil
+}
+
 func (s *state4) LoadDeadline(idx uint64) (Deadline, error) {
 	dls, err := s.State.LoadDeadlines(s.store)
 	if err != nil {

chain/actors/builtin/miner/v5.go

@@ -311,6 +311,15 @@ func (s *state5) UnallocatedSectorNumbers(count int) ([]abi.SectorNumber, error)
 	return sectors, nil
 }
 
+func (s *state5) GetAllocatedSectors() (*bitfield.BitField, error) {
+	var allocatedSectors bitfield.BitField
+	if err := s.store.Get(s.store.Context(), s.State.AllocatedSectors, &allocatedSectors); err != nil {
+		return nil, err
+	}
+
+	return &allocatedSectors, nil
+}
+
 func (s *state5) LoadDeadline(idx uint64) (Deadline, error) {
 	dls, err := s.State.LoadDeadlines(s.store)
 	if err != nil {

cmd/lotus-storage-miner/actor.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"strings"
 
+	rlepluslazy "github.com/filecoin-project/go-bitfield/rle"
 	cbor "github.com/ipfs/go-ipld-cbor"
 
 	"github.com/fatih/color"
@@ -14,6 +15,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 
@@ -41,6 +43,7 @@ var actorCmd = &cli.Command{
 		actorControl,
 		actorProposeChangeWorker,
 		actorConfirmChangeWorker,
+		actorCompactAllocatedCmd,
 	},
 }
 
@@ -987,3 +990,154 @@ var actorConfirmChangeWorker = &cli.Command{
 		return nil
 	},
 }
+
+var actorCompactAllocatedCmd = &cli.Command{
+	Name:  "compact-allocated",
+	Usage: "compact allocated sectors bitfield",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:  "mask-last-offset",
+			Usage: "Mask sector IDs from 0 to 'higest_allocated - offset'",
+		},
+		&cli.Uint64Flag{
+			Name:  "mask-upto-n",
+			Usage: "Mask sector IDs from 0 to 'n'",
+		},
+		&cli.BoolFlag{
+			Name:  "really-do-it",
+			Usage: "Actually send transaction performing the action",
+			Value: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Bool("really-do-it") {
+			fmt.Println("Pass --really-do-it to actually execute this action")
+			return nil
+		}
+
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass address of new owner address")
+		}
+
+		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		api, acloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer acloser()
+
+		ctx := lcli.ReqContext(cctx)
+
+		maddr, err := nodeApi.ActorAddress(ctx)
+		if err != nil {
+			return err
+		}
+
+		mact, err := api.StateGetActor(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(api)))
+
+		mst, err := miner.Load(store, mact)
+		if err != nil {
+			return err
+		}
+
+		allocs, err := mst.GetAllocatedSectors()
+		if err != nil {
+			return err
+		}
+
+		var maskBf bitfield.BitField
+
+		{
+			exclusiveFlags := []string{"mask-last-offset", "mask-upto-n"}
+			hasFlag := false
+			for _, f := range exclusiveFlags {
+				if hasFlag && cctx.IsSet(f) {
+					return xerrors.Errorf("more than one 'mask` flag set")
+				}
+				hasFlag = hasFlag || cctx.IsSet(f)
+			}
+		}
+		switch {
+		case cctx.IsSet("mask-last-offset"):
+			last, err := allocs.Last()
+			if err != nil {
+				return err
+			}
+
+			m := cctx.Uint64("mask-last-offset")
+			if last <= m+1 {
+				return xerrors.Errorf("highest allocated sector lower than mask offset %d: %d", m+1, last)
+			}
+			// securty to not brick a miner
+			if last > 1<<60 {
+				return xerrors.Errorf("very high last sector number, refusing to mask: %d", last)
+			}
+
+			maskBf, err = bitfield.NewFromIter(&rlepluslazy.RunSliceIterator{
+				Runs: []rlepluslazy.Run{{Val: true, Len: last - m}}})
+			if err != nil {
+				return xerrors.Errorf("forming bitfield: %w", err)
+			}
+		case cctx.IsSet("mask-upto-n"):
+			n := cctx.Uint64("mask-upto-n")
+			maskBf, err = bitfield.NewFromIter(&rlepluslazy.RunSliceIterator{
+				Runs: []rlepluslazy.Run{{Val: true, Len: n}}})
+			if err != nil {
+				return xerrors.Errorf("forming bitfield: %w", err)
+			}
+		default:
+			return xerrors.Errorf("no 'mask' flags set")
+		}
+
+		mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		params := &miner2.CompactSectorNumbersParams{
+			MaskSectorNumbers: maskBf,
+		}
+
+		sp, err := actors.SerializeParams(params)
+		if err != nil {
+			return xerrors.Errorf("serializing params: %w", err)
+		}
+
+		smsg, err := api.MpoolPushMessage(ctx, &types.Message{
+			From:   mi.Worker,
+			To:     maddr,
+			Method: miner.Methods.CompactSectorNumbers,
+			Value:  big.Zero(),
+			Params: sp,
+		}, nil)
+		if err != nil {
+			return xerrors.Errorf("mpool push: %w", err)
+		}
+
+		fmt.Println("CompactSectorNumbers Message CID:", smsg.Cid())
+
+		// wait for it to get mined into a block
+		wait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence)
+		if err != nil {
+			return err
+		}
+
+		// check it executed successfully
+		if wait.Receipt.ExitCode != 0 {
+			fmt.Println("Propose owner change failed!")
+			return err
+		}
+
+		return nil
+	},
+}

documentation/en/cli-lotus-miner.md

@@ -207,6 +207,7 @@ COMMANDS:
    control                Manage control addresses
    propose-change-worker  Propose a worker address change
    confirm-change-worker  Confirm a worker address change
+   compact-allocated      compact allocated sectors bitfield
    help, h                Shows a list of commands or help for one command
 
 OPTIONS:
@@ -361,6 +362,22 @@ OPTIONS:
    
 ```
 
+### lotus-miner actor compact-allocated
+```
+NAME:
+   lotus-miner actor compact-allocated - compact allocated sectors bitfield
+
+USAGE:
+   lotus-miner actor compact-allocated [command options] [arguments...]
+
+OPTIONS:
+   --mask-last-offset value  Mask sector IDs from 0 to 'higest_allocated - offset' (default: 0)
+   --mask-upto-n value       Mask sector IDs from 0 to 'n' (default: 0)
+   --really-do-it            Actually send transaction performing the action (default: false)
+   --help, -h                show help (default: false)
+   
+```
+
 ## lotus-miner info
 ```
 NAME: