cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

numassigner: SectorNumReserveCount

Browse Source
This commit is contained in:
Łukasz Magiera 2022-08-24 11:25:37 -04:00
parent b290dac3cb
commit 335f5df3ec
13 changed files with 272 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
api/api_storage.go
View File

@ -125,10 +125,19 @@ type StorageMiner interface {
SectorAbortUpgrade(context.Context, abi.SectorNumber) error //perm:admin
// SectorNumAssignerMeta returns sector number assigner metadata - reserved/allocated
SectorNumAssignerMeta(ctx context.Context) (NumAssignerMeta, error) //perm:read
SectorNumReservations(ctx context.Context) (map[string]bitfield.BitField, error) //perm:read
SectorNumReserve(ctx context.Context, name string, field bitfield.BitField, force bool) error //perm:admin
SectorNumFree(ctx context.Context, name string) error //perm:admin
SectorNumAssignerMeta(ctx context.Context) (NumAssignerMeta, error) //perm:read
// SectorNumReservations returns a list of sector number reservations
SectorNumReservations(ctx context.Context) (map[string]bitfield.BitField, error) //perm:read
// SectorNumReserve creates a new sector number reservation. Will fail if any other reservation has colliding
// numbers or name. Set force to true to override safety checks.
// Valid characters for name: a-z, A-Z, 0-9, _, -
SectorNumReserve(ctx context.Context, name string, sectors bitfield.BitField, force bool) error //perm:admin
// SectorNumReserveCount creates a new sector number reservation for `count` sector numbers.
// by default lotus will allocate lowest-available sector numbers to the reservation.
// For restrictions on `name` see SectorNumReserve
SectorNumReserveCount(ctx context.Context, name string, count uint64) (bitfield.BitField, error) //perm:admin
// SectorNumFree drops a sector reservation
SectorNumFree(ctx context.Context, name string) error //perm:admin
// WorkerConnect tells the node to connect to workers RPC
WorkerConnect(context.Context, string) error //perm:admin retry:true
@ -480,5 +489,8 @@ type NumAssignerMeta struct {
Reserved bitfield.BitField
Allocated bitfield.BitField
// ChainAllocated+Reserved+Allocated
InUse bitfield.BitField
Next abi.SectorNumber
}

13
api/proxy_gen.go
View File

@ -836,6 +836,8 @@ type StorageMinerStruct struct {
SectorNumReserve func(p0 context.Context, p1 string, p2 bitfield.BitField, p3 bool) error `perm:"admin"`
SectorNumReserveCount func(p0 context.Context, p1 string, p2 uint64) (bitfield.BitField, error) `perm:"admin"`
SectorPreCommitFlush func(p0 context.Context) ([]sealiface.PreCommitBatchRes, error) `perm:"admin"`
SectorPreCommitPending func(p0 context.Context) ([]abi.SectorID, error) `perm:"admin"`
@ -4978,6 +4980,17 @@ func (s *StorageMinerStub) SectorNumReserve(p0 context.Context, p1 string, p2 bi
return ErrNotSupported
}
func (s *StorageMinerStruct) SectorNumReserveCount(p0 context.Context, p1 string, p2 uint64) (bitfield.BitField, error) {
if s.Internal.SectorNumReserveCount == nil {
return *new(bitfield.BitField), ErrNotSupported
}
return s.Internal.SectorNumReserveCount(p0, p1, p2)
}
func (s *StorageMinerStub) SectorNumReserveCount(p0 context.Context, p1 string, p2 uint64) (bitfield.BitField, error) {
return *new(bitfield.BitField), ErrNotSupported
}
func (s *StorageMinerStruct) SectorPreCommitFlush(p0 context.Context) ([]sealiface.PreCommitBatchRes, error) {
if s.Internal.SectorPreCommitFlush == nil {
return *new([]sealiface.PreCommitBatchRes), ErrNotSupported

BIN
build/openrpc/full.json.gz
View File

Binary file not shown.

BIN
build/openrpc/gateway.json.gz
View File

Binary file not shown.

BIN
build/openrpc/miner.json.gz
View File

Binary file not shown.

BIN
build/openrpc/worker.json.gz
View File

Binary file not shown.

97
cmd/lotus-miner/sectors.go
View File

@ -18,7 +18,6 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/go-bitfield"
rlepluslazy "github.com/filecoin-project/go-bitfield/rle"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/builtin"
@ -33,6 +32,7 @@ import (
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/lib/strle"
"github.com/filecoin-project/lotus/lib/tablewriter"
sealing "github.com/filecoin-project/lotus/storage/pipeline"
)
@ -2231,12 +2231,12 @@ var sectorsNumbersInfoCmd = &cli.Command{
return err
}
alloc, err := bitfieldToHumanRanges(am.Allocated)
alloc, err := strle.BitfieldToHumanRanges(am.Allocated)
if err != nil {
return err
}
reserved, err := bitfieldToHumanRanges(am.Reserved)
reserved, err := strle.BitfieldToHumanRanges(am.Reserved)
if err != nil {
return err
}
@ -2268,7 +2268,7 @@ var sectorsNumbersReservationsCmd = &cli.Command{
var out []string
for name, field := range rs {
hr, err := bitfieldToHumanRanges(field)
hr, err := strle.BitfieldToHumanRanges(field)
if err != nil {
return err
}
@ -2314,7 +2314,7 @@ var sectorsNumbersReserveCmd = &cli.Command{
return xerrors.Errorf("expected 2 arguments: [reservation name] [reserved ranges]")
}
bf, err := humanRangesToBitField(cctx.Args().Get(1))
bf, err := strle.HumanRangesToBitField(cctx.Args().Get(1))
if err != nil {
return xerrors.Errorf("parsing ranges: %w", err)
}
@ -2342,90 +2342,3 @@ var sectorsNumbersFreeCmd = &cli.Command{
return api.SectorNumFree(ctx, cctx.Args().First())
},
}
func humanRangesToBitField(h string) (bitfield.BitField, error) {
var runs []rlepluslazy.Run
var last uint64
strRanges := strings.Split(h, ",")
for i, strRange := range strRanges {
lr := strings.Split(strRange, "-")
var start, end uint64
var err error
switch len(lr) {
case 1: // one number
start, err = strconv.ParseUint(lr[0], 10, 64)
if err != nil {
return bitfield.BitField{}, xerrors.Errorf("parsing left side of run %d: %w", i, err)
}
end = start
case 2: // x-y
start, err = strconv.ParseUint(lr[0], 10, 64)
if err != nil {
return bitfield.BitField{}, xerrors.Errorf("parsing left side of run %d: %w", i, err)
}
end, err = strconv.ParseUint(lr[1], 10, 64)
if err != nil {
return bitfield.BitField{}, xerrors.Errorf("parsing right side of run %d: %w", i, err)
}
}
if start < last {
return bitfield.BitField{}, xerrors.Errorf("run %d start(%d) was less than last run end(%d)", i, start, last)
}
if start == last && last > 0 {
return bitfield.BitField{}, xerrors.Errorf("run %d start(%d) was equal to last run end(%d)", i, start, last)
}
if start > end {
return bitfield.BitField{}, xerrors.Errorf("run start(%d) can't be greater than run end(%d) (run %d)", start, end, i)
}
if start > last {
runs = append(runs, rlepluslazy.Run{Val: false, Len: start - last})
}
runs = append(runs, rlepluslazy.Run{Val: true, Len: end - start + 1})
last = end + 1
}
return bitfield.NewFromIter(&rlepluslazy.RunSliceIterator{Runs: runs})
}
func bitfieldToHumanRanges(bf bitfield.BitField) (string, error) {
bj, err := bf.MarshalJSON()
if err != nil {
return "", err
}
var bints []int64
if err := json.Unmarshal(bj, &bints); err != nil {
return "", err
}
var at int64
var out string
for i, bi := range bints {
at += bi
if i%2 == 0 {
if i > 0 {
out += ","
}
out += fmt.Sprint(at)
continue
}
if bi > 1 {
out += "-"
out += fmt.Sprint(at - 1)
}
}
return out, err
}

34
documentation/en/api-v0-methods-miner.md
View File

@ -143,6 +143,7 @@
* [SectorNumFree](#SectorNumFree)
* [SectorNumReservations](#SectorNumReservations)
* [SectorNumReserve](#SectorNumReserve)
* [SectorNumReserveCount](#SectorNumReserveCount)
* [SectorPreCommitFlush](#SectorPreCommitFlush)
* [SectorPreCommitPending](#SectorPreCommitPending)
* [SectorRemove](#SectorRemove)
@ -2961,11 +2962,16 @@ Response:
5,
1
],
"InUse": [
5,
1
],
"Next": 9
}
```
### SectorNumFree
SectorNumFree drops a sector reservation
Perms: admin
@ -2980,6 +2986,7 @@ Inputs:
Response: `{}`
### SectorNumReservations
SectorNumReservations returns a list of sector number reservations
Perms: read
@ -2999,6 +3006,9 @@ Response:
```
### SectorNumReserve
SectorNumReserve creates a new sector number reservation. Will fail if any other reservation has colliding
numbers or name. Set force to true to override safety checks.
Valid characters for name: a-z, A-Z, 0-9, _, -
Perms: admin
@ -3017,6 +3027,30 @@ Inputs:
Response: `{}`
### SectorNumReserveCount
SectorNumReserveCount creates a new sector number reservation for `count` sector numbers.
by default lotus will allocate lowest-available sector numbers to the reservation.
For restrictions on `name` see SectorNumReserve
Perms: admin
Inputs:
```json
[
"string value",
42
]
```
Response:
```json
[
5,
1
]
```
### SectorPreCommitFlush
SectorPreCommitFlush immediately sends a PreCommit message with sectors batched for PreCommit.
Returns null if message wasn't sent

47
itests/sector_numassign_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/itests/kit"
"github.com/filecoin-project/lotus/lib/strle"
)
func TestAssignBasic(t *testing.T) {
@ -128,3 +129,49 @@ func TestAssignReservation(t *testing.T) {
require.Equal(t, abi.SectorNumber(11), sl[1])
require.Equal(t, abi.SectorNumber(12), sl[2])
}
func TestReserveCount(t *testing.T) {
kit.QuietMiningLogs()
ctx := context.Background()
_, miner, _ := kit.EnsembleMinimal(t, kit.ThroughRPC(), kit.MockProofs())
// with no reservations higher
r1, err := miner.SectorNumReserveCount(ctx, "r1", 2)
require.NoError(t, err)
requireBitField(t, "3-4", r1)
// reserve some higher numbers
err = miner.SectorNumReserve(ctx, "test-reservation", rangeBitField(10, 15), false)
require.NoError(t, err)
// reserve a few below an existing reservation
r2, err := miner.SectorNumReserveCount(ctx, "r2", 2)
require.NoError(t, err)
requireBitField(t, "5-6", r2)
// reserve a few through an existing reservation
r3, err := miner.SectorNumReserveCount(ctx, "r3", 6)
require.NoError(t, err)
requireBitField(t, "7-9,16-18", r3)
// do one more
r4, err := miner.SectorNumReserveCount(ctx, "r4", 4)
require.NoError(t, err)
requireBitField(t, "19-22", r4)
resvs, err := miner.SectorNumReservations(ctx)
require.NoError(t, err)
requireBitField(t, "3-4", resvs["r1"])
requireBitField(t, "5-6", resvs["r2"])
requireBitField(t, "7-9,16-18", resvs["r3"])
requireBitField(t, "19-22", resvs["r4"])
}
func requireBitField(t *testing.T, expect string, bf bitfield.BitField) {
s, err := strle.BitfieldToHumanRanges(bf)
require.NoError(t, err)
require.Equal(t, expect, s)
}

100
lib/strle/human.go Normal file
View File

@ -0,0 +1,100 @@
package strle
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-bitfield"
rlepluslazy "github.com/filecoin-project/go-bitfield/rle"
)
func HumanRangesToBitField(h string) (bitfield.BitField, error) {
var runs []rlepluslazy.Run
var last uint64
strRanges := strings.Split(h, ",")
for i, strRange := range strRanges {
lr := strings.Split(strRange, "-")
var start, end uint64
var err error
switch len(lr) {
case 1: // one number
start, err = strconv.ParseUint(lr[0], 10, 64)
if err != nil {
return bitfield.BitField{}, xerrors.Errorf("parsing left side of run %d: %w", i, err)
}
end = start
case 2: // x-y
start, err = strconv.ParseUint(lr[0], 10, 64)
if err != nil {
return bitfield.BitField{}, xerrors.Errorf("parsing left side of run %d: %w", i, err)
}
end, err = strconv.ParseUint(lr[1], 10, 64)
if err != nil {
return bitfield.BitField{}, xerrors.Errorf("parsing right side of run %d: %w", i, err)
}
}
if start < last {
return bitfield.BitField{}, xerrors.Errorf("run %d start(%d) was less than last run end(%d)", i, start, last)
}
if start == last && last > 0 {
return bitfield.BitField{}, xerrors.Errorf("run %d start(%d) was equal to last run end(%d)", i, start, last)
}
if start > end {
return bitfield.BitField{}, xerrors.Errorf("run start(%d) can't be greater than run end(%d) (run %d)", start, end, i)
}
if start > last {
runs = append(runs, rlepluslazy.Run{Val: false, Len: start - last})
}
runs = append(runs, rlepluslazy.Run{Val: true, Len: end - start + 1})
last = end + 1
}
return bitfield.NewFromIter(&rlepluslazy.RunSliceIterator{Runs: runs})
}
func BitfieldToHumanRanges(bf bitfield.BitField) (string, error) {
bj, err := bf.MarshalJSON()
if err != nil {
return "", err
}
var bints []int64
if err := json.Unmarshal(bj, &bints); err != nil {
return "", err
}
var at int64
var out string
for i, bi := range bints {
at += bi
if i%2 == 0 {
if i > 0 {
out += ","
}
out += fmt.Sprint(at)
continue
}
if bi > 1 {
out += "-"
out += fmt.Sprint(at - 1)
}
}
return out, err
}

8
cmd/lotus-miner/sectors_test.go → lib/strle/human_test.go
View File

@ -1,4 +1,4 @@
package main
package strle
import (
"testing"
@ -11,7 +11,7 @@ import (
func TestHumanBitfield(t *testing.T) {
check := func(ints []uint64, out string) {
bf := bitfield.NewFromSet(ints)
h, err := bitfieldToHumanRanges(bf)
h, err := BitfieldToHumanRanges(bf)
require.NoError(t, err)
require.Equal(t, out, h)
}
@ -26,10 +26,10 @@ func TestHumanBitfield(t *testing.T) {
func TestHumanBitfieldRoundtrip(t *testing.T) {
check := func(ints []uint64, out string) {
parsed, err := humanRangesToBitField(out)
parsed, err := HumanRangesToBitField(out)
require.NoError(t, err)
h, err := bitfieldToHumanRanges(parsed)
h, err := BitfieldToHumanRanges(parsed)
require.NoError(t, err)
require.Equal(t, out, h)

4
node/impl/storminer.go
View File

@ -440,6 +440,10 @@ func (sm *StorageMinerAPI) SectorNumReserve(ctx context.Context, name string, fi
return sm.Miner.NumReserve(ctx, name, field, force)
}
func (sm *StorageMinerAPI) SectorNumReserveCount(ctx context.Context, name string, count uint64) (bitfield.BitField, error) {
return sm.Miner.NumReserveCount(ctx, name, count)
}
func (sm *StorageMinerAPI) SectorNumFree(ctx context.Context, name string) error {
return sm.Miner.NumFree(ctx, name)
}

49
storage/pipeline/numassign.go
View File

@ -168,6 +168,7 @@ func (m *Sealing) numAssignerMetaLocked(ctx context.Context) (api.NumAssignerMet
return api.NumAssignerMeta{
Reserved: *reserved,
Allocated: *allocated,
InUse: inuse,
Next: firstFree,
}, nil
}
@ -208,6 +209,11 @@ func (m *Sealing) NumReserve(ctx context.Context, name string, reserving bitfiel
m.sclk.Lock()
defer m.sclk.Unlock()
return m.numReserveLocked(ctx, name, reserving, force)
}
// NumReserve creates a new sector reservation
func (m *Sealing) numReserveLocked(ctx context.Context, name string, reserving bitfield.BitField, force bool) error {
rk, err := reservationKey(name)
if err != nil {
return err
@ -294,6 +300,49 @@ func (m *Sealing) NumReserve(ctx context.Context, name string, reserving bitfiel
return nil
}
func (m *Sealing) NumReserveCount(ctx context.Context, name string, count uint64) (bitfield.BitField, error) {
m.sclk.Lock()
defer m.sclk.Unlock()
nm, err := m.numAssignerMetaLocked(ctx)
if err != nil {
return bitfield.BitField{}, err
}
// figure out `count` unused sectors at lowest possible numbers
usedCount, err := nm.InUse.Count()
if err != nil {
return bitfield.BitField{}, err
}
// get a bitfield mask which has at least `count` bits more set than the nm.InUse field
mask, err := bitfield.NewFromIter(&rlepluslazy.RunSliceIterator{Runs: []rlepluslazy.Run{
{
Val: true,
Len: count + usedCount,
},
}})
free, err := bitfield.SubtractBitField(mask, nm.InUse)
if err != nil {
return bitfield.BitField{}, err
}
// free now has at least 'count' bits set - it's possible that InUse had some bits set outside the count+usedCount range
free, err = free.Slice(0, count)
if err != nil {
return bitfield.BitField{}, err
}
if err := m.numReserveLocked(ctx, name, free, false); err != nil {
return bitfield.BitField{}, err
}
return free, nil
}
// NumFree removes a named sector reservation
func (m *Sealing) NumFree(ctx context.Context, name string) error {
rk, err := reservationKey(name)