sealing: collateral buffer / falback config

.circleci/config.yml

@@ -815,7 +815,7 @@ workflows:
           name: test-itest-nonce
           suite: itest-nonce
           target: "./itests/nonce_test.go"
-
+      
       - test:
           name: test-itest-paych_api
           suite: itest-paych_api
@@ -835,12 +835,12 @@ workflows:
           name: test-itest-sector_finalize_early
           suite: itest-sector_finalize_early
           target: "./itests/sector_finalize_early_test.go"
-
+      
       - test:
           name: test-itest-sector_miner_collateral
           suite: itest-sector_miner_collateral
           target: "./itests/sector_miner_collateral_test.go"
-
+      
       - test:
           name: test-itest-sector_pledge
           suite: itest-sector_pledge

extern/storage-sealing/commit_batch.go

@@ -7,10 +7,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/filecoin-project/go-state-types/network"
-
-	"github.com/filecoin-project/lotus/chain/actors"
-
 	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 
@@ -18,13 +14,16 @@ import (
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/network"
 	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
 	proof5 "github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
+	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 	"github.com/filecoin-project/lotus/node/config"
@@ -46,6 +45,7 @@ type CommitBatcherApi interface {
 	StateSectorPreCommitInfo(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok TipSetToken) (*miner.SectorPreCommitOnChainInfo, error)
 	StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, TipSetToken) (big.Int, error)
 	StateNetworkVersion(ctx context.Context, tok TipSetToken) (network.Version, error)
+	StateMinerAvailableBalance(context.Context, address.Address, TipSetToken) (big.Int, error)
 }
 
 type AggregateInput struct {
@@ -341,9 +341,9 @@ func (b *CommitBatcher) processBatch(cfg sealiface.Config) ([]sealiface.CommitBa
 	aggFee := big.Div(big.Mul(policy.AggregateNetworkFee(nv, len(infos), bf), aggFeeNum), aggFeeDen)
 
 	needFunds := big.Add(collateral, aggFee)
-
-	if cfg.CollateralFromMinerBalance {
-		needFunds = big.Zero()
+	needFunds, err = collateralSendAmount(b.mctx, b.api, b.maddr, cfg, needFunds)
+	if err != nil {
+		return []sealiface.CommitBatchRes{res}, err
 	}
 
 	goodFunds := big.Add(maxFee, needFunds)
@@ -371,6 +371,20 @@ func (b *CommitBatcher) processIndividually(cfg sealiface.Config) ([]sealiface.C
 		return nil, xerrors.Errorf("couldn't get miner info: %w", err)
 	}
 
+	avail := types.TotalFilecoinInt
+
+	if cfg.CollateralFromMinerBalance && !cfg.DisableCollateralFallback {
+		avail, err = b.api.StateMinerAvailableBalance(b.mctx, b.maddr, nil)
+		if err != nil {
+			return nil, xerrors.Errorf("getting available miner balance: %w", err)
+		}
+
+		avail = big.Sub(avail, cfg.AvailableBalanceBuffer)
+		if avail.LessThan(big.Zero()) {
+			avail = big.Zero()
+		}
+	}
+
 	tok, _, err := b.api.ChainHead(b.mctx)
 	if err != nil {
 		return nil, err
@@ -384,7 +398,7 @@ func (b *CommitBatcher) processIndividually(cfg sealiface.Config) ([]sealiface.C
 			FailedSectors: map[abi.SectorNumber]string{},
 		}
 
-		mcid, err := b.processSingle(cfg, mi, sn, info, tok)
+		mcid, err := b.processSingle(cfg, mi, &avail, sn, info, tok)
 		if err != nil {
 			log.Errorf("process single error: %+v", err) // todo: return to user
 			r.FailedSectors[sn] = err.Error()
@@ -398,7 +412,7 @@ func (b *CommitBatcher) processIndividually(cfg sealiface.Config) ([]sealiface.C
 	return res, nil
 }
 
-func (b *CommitBatcher) processSingle(cfg sealiface.Config, mi miner.MinerInfo, sn abi.SectorNumber, info AggregateInput, tok TipSetToken) (cid.Cid, error) {
+func (b *CommitBatcher) processSingle(cfg sealiface.Config, mi miner.MinerInfo, avail *abi.TokenAmount, sn abi.SectorNumber, info AggregateInput, tok TipSetToken) (cid.Cid, error) {
 	enc := new(bytes.Buffer)
 	params := &miner.ProveCommitSectorParams{
 		SectorNumber: sn,
@@ -415,7 +429,16 @@ func (b *CommitBatcher) processSingle(cfg sealiface.Config, mi miner.MinerInfo,
 	}
 
 	if cfg.CollateralFromMinerBalance {
-		collateral = big.Zero()
+		c := big.Sub(collateral, *avail)
+		*avail = big.Sub(*avail, collateral)
+		collateral = c
+
+		if collateral.LessThan(big.Zero()) {
+			collateral = big.Zero()
+		}
+		if (*avail).LessThan(big.Zero()) {
+			*avail = big.Zero()
+		}
 	}
 
 	goodFunds := big.Add(collateral, big.Int(b.feeCfg.MaxCommitGasFee))

extern/storage-sealing/mocks/mock_commit_batcher.go

@@ -88,6 +88,21 @@ func (mr *MockCommitBatcherApiMockRecorder) SendMsg(arg0, arg1, arg2, arg3, arg4
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockCommitBatcherApi)(nil).SendMsg), arg0, arg1, arg2, arg3, arg4, arg5, arg6)
 }
 
+// StateMinerAvailableBalance mocks base method.
+func (m *MockCommitBatcherApi) StateMinerAvailableBalance(arg0 context.Context, arg1 address.Address, arg2 sealing.TipSetToken) (big.Int, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "StateMinerAvailableBalance", arg0, arg1, arg2)
+	ret0, _ := ret[0].(big.Int)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// StateMinerAvailableBalance indicates an expected call of StateMinerAvailableBalance.
+func (mr *MockCommitBatcherApiMockRecorder) StateMinerAvailableBalance(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateMinerAvailableBalance", reflect.TypeOf((*MockCommitBatcherApi)(nil).StateMinerAvailableBalance), arg0, arg1, arg2)
+}
+
 // StateMinerInfo mocks base method.
 func (m *MockCommitBatcherApi) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 sealing.TipSetToken) (miner.MinerInfo, error) {
 	m.ctrl.T.Helper()

extern/storage-sealing/mocks/mock_precommit_batcher.go

@@ -71,6 +71,21 @@ func (mr *MockPreCommitBatcherApiMockRecorder) SendMsg(arg0, arg1, arg2, arg3, a
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockPreCommitBatcherApi)(nil).SendMsg), arg0, arg1, arg2, arg3, arg4, arg5, arg6)
 }
 
+// StateMinerAvailableBalance mocks base method.
+func (m *MockPreCommitBatcherApi) StateMinerAvailableBalance(arg0 context.Context, arg1 address.Address, arg2 sealing.TipSetToken) (big.Int, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "StateMinerAvailableBalance", arg0, arg1, arg2)
+	ret0, _ := ret[0].(big.Int)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// StateMinerAvailableBalance indicates an expected call of StateMinerAvailableBalance.
+func (mr *MockPreCommitBatcherApiMockRecorder) StateMinerAvailableBalance(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateMinerAvailableBalance", reflect.TypeOf((*MockPreCommitBatcherApi)(nil).StateMinerAvailableBalance), arg0, arg1, arg2)
+}
+
 // StateMinerInfo mocks base method.
 func (m *MockPreCommitBatcherApi) StateMinerInfo(arg0 context.Context, arg1 address.Address, arg2 sealing.TipSetToken) (miner.MinerInfo, error) {
 	m.ctrl.T.Helper()

extern/storage-sealing/precommit_batch.go

@@ -7,9 +7,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
-
 	"github.com/ipfs/go-cid"
 	"golang.org/x/xerrors"
 
@@ -20,7 +17,9 @@ import (
 	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
 	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 	"github.com/filecoin-project/lotus/node/config"
 )
@@ -30,6 +29,7 @@ import (
 type PreCommitBatcherApi interface {
 	SendMsg(ctx context.Context, from, to address.Address, method abi.MethodNum, value, maxFee abi.TokenAmount, params []byte) (cid.Cid, error)
 	StateMinerInfo(context.Context, address.Address, TipSetToken) (miner.MinerInfo, error)
+	StateMinerAvailableBalance(context.Context, address.Address, TipSetToken) (big.Int, error)
 	ChainHead(ctx context.Context) (TipSetToken, abi.ChainEpoch, error)
 }
 
@@ -225,8 +225,10 @@ func (b *PreCommitBatcher) processBatch(cfg sealiface.Config) ([]sealiface.PreCo
 		params.Sectors = append(params.Sectors, *p.pci)
 		deposit = big.Add(deposit, p.deposit)
 	}
-	if cfg.CollateralFromMinerBalance {
-		deposit = big.Zero()
+
+	deposit, err := collateralSendAmount(b.mctx, b.api, b.maddr, cfg, deposit)
+	if err != nil {
+		return []sealiface.PreCommitBatchRes{res}, err
 	}
 
 	enc := new(bytes.Buffer)

extern/storage-sealing/sealiface/config.go

@@ -25,6 +25,8 @@ type Config struct {
 	FinalizeEarly bool
 
 	CollateralFromMinerBalance bool
+	AvailableBalanceBuffer     abi.TokenAmount
+	DisableCollateralFallback  bool
 
 	BatchPreCommits     bool
 	MaxPreCommitBatch   int

extern/storage-sealing/sealing.go

@@ -59,6 +59,7 @@ type SealingAPI interface {
 	StateMinerPreCommitDepositForPower(context.Context, address.Address, miner.SectorPreCommitInfo, TipSetToken) (big.Int, error)
 	StateMinerInitialPledgeCollateral(context.Context, address.Address, miner.SectorPreCommitInfo, TipSetToken) (big.Int, error)
 	StateMinerInfo(context.Context, address.Address, TipSetToken) (miner.MinerInfo, error)
+	StateMinerAvailableBalance(context.Context, address.Address, TipSetToken) (big.Int, error)
 	StateMinerSectorAllocated(context.Context, address.Address, abi.SectorNumber, TipSetToken) (bool, error)
 	StateMarketStorageDeal(context.Context, abi.DealID, TipSetToken) (*api.MarketDeal, error)
 	StateMarketStorageDealProposal(context.Context, abi.DealID, TipSetToken) (market.DealProposal, error)

extern/storage-sealing/states_sealing.go

@@ -362,9 +362,9 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
 		return err
 	}
 
-	deposit := pcd
-	if cfg.CollateralFromMinerBalance {
-		deposit = big.Zero() // pay using available miner balance
+	deposit, err := collateralSendAmount(ctx.Context(), m.api, m.maddr, cfg, pcd)
+	if err != nil {
+		return err
 	}
 
 	enc := new(bytes.Buffer)
@@ -633,8 +633,9 @@ func (m *Sealing) handleSubmitCommit(ctx statemachine.Context, sector SectorInfo
 		collateral = big.Zero()
 	}
 
-	if cfg.CollateralFromMinerBalance {
-		collateral = big.Zero() // pay using available miner balance
+	collateral, err = collateralSendAmount(ctx.Context(), m.api, m.maddr, cfg, collateral)
+	if err != nil {
+		return err
 	}
 
 	goodFunds := big.Add(collateral, big.Int(m.feeCfg.MaxCommitGasFee))

extern/storage-sealing/utils.go

@@ -1,9 +1,17 @@
 package sealing
 
 import (
+	"context"
 	"math/bits"
 
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/extern/storage-sealing/sealiface"
 )
 
 func fillersFromRem(in abi.UnpaddedPieceSize) ([]abi.UnpaddedPieceSize, error) {
@@ -55,3 +63,31 @@ func (m *Sealing) GetSectorInfo(sid abi.SectorNumber) (SectorInfo, error) {
 	err := m.sectors.Get(uint64(sid)).Get(&out)
 	return out, err
 }
+
+func collateralSendAmount(ctx context.Context, api interface {
+	StateMinerAvailableBalance(context.Context, address.Address, TipSetToken) (big.Int, error)
+}, maddr address.Address, cfg sealiface.Config, collateral abi.TokenAmount) (abi.TokenAmount, error) {
+	if cfg.CollateralFromMinerBalance {
+		avail := types.TotalFilecoinInt
+
+		if !cfg.DisableCollateralFallback {
+			var err error
+			avail, err = api.StateMinerAvailableBalance(ctx, maddr, nil)
+			if err != nil {
+				return big.Zero(), xerrors.Errorf("getting available miner balance: %w", err)
+			}
+
+			avail = big.Sub(avail, cfg.AvailableBalanceBuffer)
+			if avail.LessThan(big.Zero()) {
+				avail = big.Zero()
+			}
+		}
+
+		collateral = big.Sub(collateral, avail)
+		if collateral.LessThan(big.Zero()) {
+			collateral = big.Zero()
+		}
+	}
+
+	return collateral, nil
+}

itests/deals_pricing_test.go

@@ -14,7 +14,7 @@ import (
 func TestQuotePriceForUnsealedRetrieval(t *testing.T) {
 	var (
 		ctx       = context.Background()
-		blocktime = time.Second
+		blocktime = 50 * time.Millisecond
 	)
 
 	kit.QuietMiningLogs()

itests/sector_miner_collateral_test.go

@@ -51,6 +51,9 @@ func TestMinerBalanceCollateral(t *testing.T) {
 						MaxCommitBatch:    nSectors,
 
 						CollateralFromMinerBalance: enabled,
+						AvailableBalanceBuffer:     big.Zero(),
+						DisableCollateralFallback:  false,
+						AggregateAboveBaseFee:      big.Zero(),
 					}, nil
 				}, nil
 			})),

node/config/def.go

@@ -127,6 +127,10 @@ type SealingConfig struct {
 
 	// Whether to use available miner balance for sector collateral instead of sending it with each message
 	CollateralFromMinerBalance bool
+	// Minimum available balance to keep in the miner actor before sending it with messages
+	AvailableBalanceBuffer types.FIL
+	// Don't send collateral with messages even if there is no available balance in the miner actor
+	DisableCollateralFallback bool
 
 	// enable / disable precommit batching (takes effect after nv13)
 	BatchPreCommits bool
@@ -320,13 +324,16 @@ func DefaultStorageMiner() *StorageMiner {
 		Common: defCommon(),
 
 		Sealing: SealingConfig{
-			MaxWaitDealsSectors:        2, // 64G with 32G sectors
-			MaxSealingSectors:          0,
-			MaxSealingSectorsForDeals:  0,
-			WaitDealsDelay:             Duration(time.Hour * 6),
-			AlwaysKeepUnsealedCopy:     true,
-			FinalizeEarly:              false,
+			MaxWaitDealsSectors:       2, // 64G with 32G sectors
+			MaxSealingSectors:         0,
+			MaxSealingSectorsForDeals: 0,
+			WaitDealsDelay:            Duration(time.Hour * 6),
+			AlwaysKeepUnsealedCopy:    true,
+			FinalizeEarly:             false,
+
 			CollateralFromMinerBalance: false,
+			AvailableBalanceBuffer:     types.FIL(big.Zero()),
+			DisableCollateralFallback:  false,
 
 			BatchPreCommits:     true,
 			MaxPreCommitBatch:   miner5.PreCommitSectorBatchMaxSize, // up to 256 sectors

node/modules/storageminer.go

@@ -863,13 +863,16 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error
 	return func(cfg sealiface.Config) (err error) {
 		err = mutateCfg(r, func(c *config.StorageMiner) {
 			c.Sealing = config.SealingConfig{
-				MaxWaitDealsSectors:        cfg.MaxWaitDealsSectors,
-				MaxSealingSectors:          cfg.MaxSealingSectors,
-				MaxSealingSectorsForDeals:  cfg.MaxSealingSectorsForDeals,
-				WaitDealsDelay:             config.Duration(cfg.WaitDealsDelay),
-				AlwaysKeepUnsealedCopy:     cfg.AlwaysKeepUnsealedCopy,
-				FinalizeEarly:              cfg.FinalizeEarly,
+				MaxWaitDealsSectors:       cfg.MaxWaitDealsSectors,
+				MaxSealingSectors:         cfg.MaxSealingSectors,
+				MaxSealingSectorsForDeals: cfg.MaxSealingSectorsForDeals,
+				WaitDealsDelay:            config.Duration(cfg.WaitDealsDelay),
+				AlwaysKeepUnsealedCopy:    cfg.AlwaysKeepUnsealedCopy,
+				FinalizeEarly:             cfg.FinalizeEarly,
+
 				CollateralFromMinerBalance: cfg.CollateralFromMinerBalance,
+				AvailableBalanceBuffer:     types.FIL(cfg.AvailableBalanceBuffer),
+				DisableCollateralFallback:  cfg.DisableCollateralFallback,
 
 				BatchPreCommits:     cfg.BatchPreCommits,
 				MaxPreCommitBatch:   cfg.MaxPreCommitBatch,
@@ -894,13 +897,16 @@ func NewSetSealConfigFunc(r repo.LockedRepo) (dtypes.SetSealingConfigFunc, error
 
 func ToSealingConfig(cfg *config.StorageMiner) sealiface.Config {
 	return sealiface.Config{
-		MaxWaitDealsSectors:        cfg.Sealing.MaxWaitDealsSectors,
-		MaxSealingSectors:          cfg.Sealing.MaxSealingSectors,
-		MaxSealingSectorsForDeals:  cfg.Sealing.MaxSealingSectorsForDeals,
-		WaitDealsDelay:             time.Duration(cfg.Sealing.WaitDealsDelay),
-		AlwaysKeepUnsealedCopy:     cfg.Sealing.AlwaysKeepUnsealedCopy,
-		FinalizeEarly:              cfg.Sealing.FinalizeEarly,
+		MaxWaitDealsSectors:       cfg.Sealing.MaxWaitDealsSectors,
+		MaxSealingSectors:         cfg.Sealing.MaxSealingSectors,
+		MaxSealingSectorsForDeals: cfg.Sealing.MaxSealingSectorsForDeals,
+		WaitDealsDelay:            time.Duration(cfg.Sealing.WaitDealsDelay),
+		AlwaysKeepUnsealedCopy:    cfg.Sealing.AlwaysKeepUnsealedCopy,
+		FinalizeEarly:             cfg.Sealing.FinalizeEarly,
+
 		CollateralFromMinerBalance: cfg.Sealing.CollateralFromMinerBalance,
+		AvailableBalanceBuffer:     types.BigInt(cfg.Sealing.AvailableBalanceBuffer),
+		DisableCollateralFallback:  cfg.Sealing.DisableCollateralFallback,
 
 		BatchPreCommits:     cfg.Sealing.BatchPreCommits,
 		MaxPreCommitBatch:   cfg.Sealing.MaxPreCommitBatch,

storage/adapter_storage_miner.go

@@ -76,6 +76,15 @@ func (s SealingAPIAdapter) StateMinerInfo(ctx context.Context, maddr address.Add
 	return s.delegate.StateMinerInfo(ctx, maddr, tsk)
 }
 
+func (s SealingAPIAdapter) StateMinerAvailableBalance(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) (big.Int, error) {
+	tsk, err := types.TipSetKeyFromBytes(tok)
+	if err != nil {
+		return big.Zero(), xerrors.Errorf("failed to unmarshal TipSetToken to TipSetKey: %w", err)
+	}
+
+	return s.delegate.StateMinerAvailableBalance(ctx, maddr, tsk)
+}
+
 func (s SealingAPIAdapter) StateMinerWorkerAddress(ctx context.Context, maddr address.Address, tok sealing.TipSetToken) (address.Address, error) {
 	// TODO: update storage-fsm to just StateMinerInfo
 	mi, err := s.StateMinerInfo(ctx, maddr, tok)

storage/miner.go

@@ -89,6 +89,7 @@ type fullNodeFilteredAPI interface {
 	StateSectorGetInfo(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*miner.SectorOnChainInfo, error)
 	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*miner.SectorLocation, error)
 	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error)
+	StateMinerAvailableBalance(ctx context.Context, maddr address.Address, tok types.TipSetKey) (types.BigInt, error)
 	StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error)
 	StateMinerPartitions(context.Context, address.Address, uint64, types.TipSetKey) ([]api.Partition, error)
 	StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error)