Merge branch 'master' into fix/new-rand

CHANGELOG.md

@@ -1,5 +1,38 @@
 # Lotus changelog
 
+# 1.10.1 / 2021-07-05
+
+This is an optional but **highly recommended** release of Lotus for lotus miners that has many bug fixes and improvements based on the feedback we got from the community since HyperDrive.
+
+## New Features
+- commit batch: AggregateAboveBaseFee config #6650
+  - `AggregateAboveBaseFee` is added to miner sealing configuration for setting the network base fee to start aggregating proofs. When the network base fee is lower than this value, the prove commits will be submitted individually via `ProveCommitSector`. According to the [Batch Incentive Alignment](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0013.md#batch-incentive-alignment) introduced in FIP-0013, we recommend miners to set this value to 0.15 nanoFIL(which is the default value) to avoid unexpected aggregation fee in burn and enjoy the most benefits of aggregation!
+    
+## Bug Fixes
+- storage: Fix FinalizeSector with sectors in storage paths #6652
+- Fix tiny error in check-client-datacap #6664  
+- Fix: precommit_batch method used the wrong cfg.PreCommitBatchWait #6658
+- to optimize the batchwait #6636
+- fix getTicket: sector precommitted but expired case #6635
+- handleSubmitCommitAggregate() exception handling #6595
+- remove precommit check in handleCommitFailed #6634
+- ensure agg fee is adequate
+- fix: miner balance is not enough, so that ProveCommitAggregate msg exec failed #6623
+- commit batch: Initialize the FailedSectors map #6647
+
+Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| @magik6k| 7 | +151/-56 | 21 |
+| @llifezou | 4 | +59/-20 | 4 |
+| @johnli-helloworld | 2 | +45/-14 | 4 |
+| @wangchao | 1 | +1/-27 | 1 |
+| Jerry | 2 | +9/-4 | 2 |
+| @zhoutian527 | 1 | +2/-2 | 1 |
+| @ribasushi| 1 | +1/-1 | 1 |
+
+
 # 1.10.0 / 2021-06-23
 
 This is a mandatory release of Lotus that introduces Filecoin network v13, codenamed the HyperDrive upgrade. The

api/api_gateway.go

@@ -45,6 +45,7 @@ type Gateway interface {
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
+	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*ActorState, error) //perm:read
 	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
 	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MarketBalance, error)

api/proxy_gen.go

@@ -525,6 +525,8 @@ type GatewayStruct struct {
 
 		StateNetworkVersion func(p0 context.Context, p1 types.TipSetKey) (apitypes.NetworkVersion, error) ``
 
+		StateReadState func(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*ActorState, error) `perm:"read"`
+
 		StateSearchMsg func(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) ``
 
 		StateSectorGetInfo func(p0 context.Context, p1 address.Address, p2 abi.SectorNumber, p3 types.TipSetKey) (*miner.SectorOnChainInfo, error) ``
@@ -2663,6 +2665,14 @@ func (s *GatewayStub) StateNetworkVersion(p0 context.Context, p1 types.TipSetKey
 	return *new(apitypes.NetworkVersion), xerrors.New("method not supported")
 }
 
+func (s *GatewayStruct) StateReadState(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*ActorState, error) {
+	return s.Internal.StateReadState(p0, p1, p2)
+}
+
+func (s *GatewayStub) StateReadState(p0 context.Context, p1 address.Address, p2 types.TipSetKey) (*ActorState, error) {
+	return nil, xerrors.New("method not supported")
+}
+
 func (s *GatewayStruct) StateSearchMsg(p0 context.Context, p1 types.TipSetKey, p2 cid.Cid, p3 abi.ChainEpoch, p4 bool) (*MsgLookup, error) {
 	return s.Internal.StateSearchMsg(p0, p1, p2, p3, p4)
 }

cli/state.go

@@ -446,6 +446,9 @@ var StateExecTraceCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
+		if lookup == nil {
+			return fmt.Errorf("failed to find message: %s", mcid)
+		}
 
 		ts, err := capi.ChainGetTipSet(ctx, lookup.TipSet)
 		if err != nil {
@@ -1491,6 +1494,10 @@ var StateSearchMsgCmd = &cli.Command{
 			return err
 		}
 
+		if mw == nil {
+			return fmt.Errorf("failed to find message: %s", msg)
+		}
+
 		m, err := api.ChainGetMessage(ctx, msg)
 		if err != nil {
 			return err

cmd/tvx/extract_message.go

@@ -337,6 +337,9 @@ func resolveFromChain(ctx context.Context, api v0api.FullNode, mcid cid.Cid, blo
 		if err != nil {
 			return nil, nil, nil, fmt.Errorf("failed to locate message: %w", err)
 		}
+		if msgInfo == nil {
+			return nil, nil, nil, fmt.Errorf("failed to locate message: not found")
+		}
 
 		log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode)
 

extern/storage-sealing/states_sealing.go

@@ -358,8 +358,11 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
 	}
 
 	params, pcd, tok, err := m.preCommitParams(ctx, sector)
-	if params == nil || err != nil {
+	if err != nil {
-		return err
+		return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("preCommitParams: %w", err)})
+	}
+	if params == nil {
+		return nil // event was sent in preCommitParams
 	}
 
 	deposit, err := collateralSendAmount(ctx.Context(), m.api, m.maddr, cfg, pcd)
@@ -403,9 +406,12 @@ func (m *Sealing) handleSubmitPreCommitBatch(ctx statemachine.Context, sector Se
 	}
 
 	params, deposit, _, err := m.preCommitParams(ctx, sector)
-	if params == nil || err != nil {
+	if err != nil {
 		return ctx.Send(SectorChainPreCommitFailed{xerrors.Errorf("preCommitParams: %w", err)})
 	}
+	if params == nil {
+		return nil // event was sent in preCommitParams
+	}
 
 	res, err := m.precommiter.AddPreCommit(ctx.Context(), sector, deposit, params)
 	if err != nil {

itests/api_test.go

@@ -121,6 +121,7 @@ func (ts *apiSuite) testSearchMsg(t *testing.T) {
 
 	searchRes, err := full.StateSearchMsg(ctx, types.EmptyTSK, sm.Cid(), lapi.LookbackNoLimit, true)
 	require.NoError(t, err)
+	require.NotNil(t, searchRes)
 
 	require.Equalf(t, res.TipSet, searchRes.TipSet, "search ts: %s, different from wait ts: %s", searchRes.TipSet, res.TipSet)
 }

itests/wdpost_test.go

@@ -213,12 +213,18 @@ func TestWindowPostBaseFeeNoBurn(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	sched := kit.DefaultTestUpgradeSchedule
+	lastUpgradeHeight := sched[len(sched)-1].Height
+
 	och := build.UpgradeClausHeight
-	build.UpgradeClausHeight = 10
+	build.UpgradeClausHeight = lastUpgradeHeight + 1
 
 	client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs())
 	ens.InterconnectAll().BeginMining(blocktime)
 
+	// Wait till all upgrades are done and we've passed the clause epoch.
+	client.WaitTillChain(ctx, kit.HeightAtLeast(build.UpgradeClausHeight+1))
+
 	maddr, err := miner.ActorAddress(ctx)
 	require.NoError(t, err)
 
@@ -268,6 +274,12 @@ func TestWindowPostBaseFeeBurn(t *testing.T) {
 	client, miner, ens := kit.EnsembleMinimal(t, kit.MockProofs(), opts)
 	ens.InterconnectAll().BeginMining(blocktime)
 
+	// Ideally we'd be a bit more precise here, but getting the information we need from the
+	// test framework is more work than it's worth.
+	//
+	// We just need to wait till all upgrades are done.
+	client.WaitTillChain(ctx, kit.HeightAtLeast(20))
+
 	maddr, err := miner.ActorAddress(ctx)
 	require.NoError(t, err)
 

markets/retrievaladapter/provider.go

@@ -54,7 +54,7 @@ func (rpn *retrievalProviderNode) GetMinerWorkerAddress(ctx context.Context, min
 
 func (rpn *retrievalProviderNode) UnsealSector(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (io.ReadCloser, error) {
 	log.Debugf("get sector %d, offset %d, length %d", sectorID, offset, length)
-	si, err := rpn.sectorsStatus(ctx, sectorID, true)
+	si, err := rpn.sectorsStatus(ctx, sectorID, false)
 	if err != nil {
 		return nil, err
 	}

node/impl/client/client.go

@@ -436,7 +436,19 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid, piece *cid.Cid)
 		if piece != nil && !piece.Equals(*p.PieceCID) {
 			continue
 		}
-		out = append(out, a.makeRetrievalQuery(ctx, p, root, piece, rm.QueryParams{}))
+
+		// do not rely on local data with respect to peer id
+		// fetch an up-to-date miner peer id from chain
+		mi, err := a.StateMinerInfo(ctx, p.Address, types.EmptyTSK)
+		if err != nil {
+			return nil, err
+		}
+		pp := rm.RetrievalPeer{
+			Address: p.Address,
+			ID:      *mi.PeerId,
+		}
+
+		out = append(out, a.makeRetrievalQuery(ctx, pp, root, piece, rm.QueryParams{}))
 	}
 
 	return out, nil

node/repo/fsrepo.go

@@ -327,6 +327,21 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain
 			return
 		}
 
+		//
+		// Tri-state environment variable LOTUS_CHAIN_BADGERSTORE_DISABLE_FSYNC
+		// - unset == the default (currently fsync enabled)
+		// - set with a false-y value == fsync enabled no matter what a future default is
+		// - set with any other value == fsync is disabled ignored defaults (recommended for day-to-day use)
+		//
+		if nosyncBs, nosyncBsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_DISABLE_FSYNC"); nosyncBsSet {
+			nosyncBs = strings.ToLower(nosyncBs)
+			if nosyncBs == "" || nosyncBs == "0" || nosyncBs == "false" || nosyncBs == "no" {
+				opts.SyncWrites = true
+			} else {
+				opts.SyncWrites = false
+			}
+		}
+
 		bs, err := badgerbs.Open(opts)
 		if err != nil {
 			fsr.bsErr = err

storage/miner_sealing.go

@@ -134,7 +134,7 @@ func (m *Miner) SectorsStatus(ctx context.Context, sid abi.SectorNumber, showOnC
 		LastErr: info.LastErr,
 		Log:     log,
 		// on chain info
-		SealProof:          0,
+		SealProof:          info.SectorType,
 		Activation:         0,
 		Expiration:         0,
 		DealWeight:         big.Zero(),