Merge pull request #8243 from filecoin-project/release/v1.14.4

chore: build: v1.14.4

CHANGELOG.md

@@ -1,5 +1,44 @@
 # Lotus changelog
 
+# 1.14.4 / 2022-03-03
+
+This is a *highly recommended* optional release for storage providers that are doing snap deals. This fix the bug 
+that causes some snap deal sectors are stuck in `FinalizeReplicaUpdate`. In addition, SPs should be able to force 
+update sectors status without getting blocked by `normal shutdown of state machine`. 
+
+# v1.14.3 / 2022-02-28
+
+This is an **optional** release, that includes a fix to properly register the `--really-do-it` flag for abort-upgrade.
+
+# 1.14.2 / 2022-02-24
+
+This is an **optional** release of lotus, that's had a couple more improvements w.r.t Snap experience for storage providers in preparation of the[upcoming OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550). 
+
+Note that the network is STILL scheduled to upgrade to v15 on March 1st at 2022-03-01T15:00:00Z. All node operators, including storage providers, must upgrade to at least Lotus v1.14.0 before that time. Storage providers must update their daemons, miners, and worker(s).
+
+Wanna know how to Snap your deal? Check [this](https://github.com/filecoin-project/lotus/discussions/8141) out! 
+
+## Bug Fixes
+- fix lotus-bench for sealing jobs (#8173)
+- fix:sealing:really-do-it flag for abort upgrade (#8181)
+- fix:proving:post check sector handles snap deals replica faults (#8177)
+- fix: sealing: missing file type (#8180)
+
+## Others
+- Retract force-pushed v1.14.0 to work around stale gomod caches (#8159): We originally tagged v1.14.0 off the wrong 
+  commit and fixed that by a force push, in which is a really bad practise since it messes up the go mod. Therefore, 
+  we want to retract it and users may use v1.14.1&^.
+
+## Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| @zenground0 | 2 | +73/-58 | 12 |
+| @eben.xie | 1 | +7/-0 | 1 |
+| @jennijuju | 1 | +4/-0 | 1 |
+| @jennijuju | 1 | +2/-1 | 1 |
+| @ribasushi | 1 | +2/-0 | 1 |
+
 # 1.14.1 / 2022-02-18
 
 This is an **optional** release of lotus, that fixes the incorrect *comment* of network v15 OhSnap upgrade **date**. Note the actual upgrade epoch in [v1.14.0](https://github.com/filecoin-project/lotus/releases/tag/v1.14.0) was correct.
@@ -22,7 +61,7 @@ It is recommended that storage providers download the new params before updating
 - run `./lotus-shed fetch-params` with the appropriate `proving-params` flag
 - Upgrade the Lotus daemon and miner **when the previous step is complete**
 
-All node operators, including storage providers, should be aware that a pre-migration will begin at 2022-03-01T13:30:00Z (150 minutes before the real upgrade). The pre-migration will take between 20 and 50 minutes, depending on hardware specs. During this time, expect slower block validation times, increased CPU and memory usage, and longer delays for API queries.
+All node operators, including storage providers, should be aware that a pre-migration will begin at 2022-03-01T13:30:00Z (90 minutes before the real upgrade). The pre-migration will take between 20 and 50 minutes, depending on hardware specs. During this time, expect slower block validation times, increased CPU and memory usage, and longer delays for API queries.
   
 ## New Features and Changes
 - Integrate actor v7-rc1:

api/api_storage.go

@@ -253,7 +253,7 @@ type StorageMiner interface {
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
 
 	ComputeProof(ctx context.Context, ssi []builtin.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtin.PoStProof, error) //perm:read
 }

api/proxy_gen.go

@@ -619,7 +619,7 @@ type StorageMinerStruct struct {
 
 		ActorSectorSize func(p0 context.Context, p1 address.Address) (abi.SectorSize, error) `perm:"read"`
 
-		CheckProvable func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) `perm:"admin"`
+		CheckProvable func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) `perm:"admin"`
 
 		ComputeProof func(p0 context.Context, p1 []builtin.ExtendedSectorInfo, p2 abi.PoStRandomness, p3 abi.ChainEpoch, p4 abinetwork.Version) ([]builtin.PoStProof, error) `perm:"read"`
 
@@ -3704,14 +3704,14 @@ func (s *StorageMinerStub) ActorSectorSize(p0 context.Context, p1 address.Addres
 	return *new(abi.SectorSize), ErrNotSupported
 }
 
-func (s *StorageMinerStruct) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) {
+func (s *StorageMinerStruct) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) {
 	if s.Internal.CheckProvable == nil {
 		return *new(map[abi.SectorNumber]string), ErrNotSupported
 	}
-	return s.Internal.CheckProvable(p0, p1, p2, p3)
+	return s.Internal.CheckProvable(p0, p1, p2, p3, p4)
 }
 
-func (s *StorageMinerStub) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) {
+func (s *StorageMinerStub) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 []bool, p4 bool) (map[abi.SectorNumber]string, error) {
 	return *new(map[abi.SectorNumber]string), ErrNotSupported
 }
 

api/version.go

@@ -57,7 +57,7 @@ var (
 	FullAPIVersion0 = newVer(1, 5, 0)
 	FullAPIVersion1 = newVer(2, 2, 0)
 
-	MinerAPIVersion0  = newVer(1, 3, 0)
+	MinerAPIVersion0  = newVer(1, 4, 0)
 	WorkerAPIVersion0 = newVer(1, 5, 0)
 )
 

build/version.go

@@ -37,7 +37,7 @@ func BuildTypeString() string {
 }
 
 // BuildVersion is the local build version
-const BuildVersion = "1.14.1"
+const BuildVersion = "1.14.4"
 
 func UserVersion() string {
 	if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" {

cmd/lotus-bench/main.go

@@ -276,6 +276,13 @@ var sealBenchCmd = &cli.Command{
 			if err != nil {
 				return xerrors.Errorf("failed to run seals: %w", err)
 			}
+			for _, s := range extendedSealedSectors {
+				sealedSectors = append(sealedSectors, proof.SectorInfo{
+					SealedCID:    s.SealedCID,
+					SectorNumber: s.SectorNumber,
+					SealProof:    s.SealProof,
+				})
+			}
 		} else {
 			// TODO: implement sbfs.List() and use that for all cases (preexisting sectorbuilder or not)
 

cmd/lotus-miner/info.go

@@ -527,6 +527,7 @@ var stateList = []stateMeta{
 	{col: color.FgRed, state: sealing.SnapDealsDealsExpired},
 	{col: color.FgRed, state: sealing.ReplicaUpdateFailed},
 	{col: color.FgRed, state: sealing.ReleaseSectorKeyFailed},
+	{col: color.FgRed, state: sealing.FinalizeReplicaUpdateFailed},
 }
 
 func init() {

cmd/lotus-miner/proving.go

@@ -437,6 +437,7 @@ var provingCheckProvableCmd = &cli.Command{
 			}
 
 			var tocheck []storage.SectorRef
+			var update []bool
 			for _, info := range sectorInfos {
 				si := abi.SectorID{
 					Miner:  abi.ActorID(mid),
@@ -454,9 +455,10 @@ var provingCheckProvableCmd = &cli.Command{
 					ProofType: info.SealProof,
 					ID:        si,
 				})
+				update = append(update, info.SectorKeyCID != nil)
 			}
 
-			bad, err := sapi.CheckProvable(ctx, info.WindowPoStProofType, tocheck, cctx.Bool("slow"))
+			bad, err := sapi.CheckProvable(ctx, info.WindowPoStProofType, tocheck, update, cctx.Bool("slow"))
 			if err != nil {
 				return err
 			}

cmd/lotus-miner/sectors.go

@@ -1525,11 +1525,23 @@ var sectorsSnapAbortCmd = &cli.Command{
 	Name:      "abort-upgrade",
 	Usage:     "Abort the attempted (SnapDeals) upgrade of a CC sector, reverting it to as before",
 	ArgsUsage: "<sectorNum>",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "really-do-it",
+			Usage: "pass this flag if you know what you are doing",
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.Args().Len() != 1 {
 			return lcli.ShowHelp(cctx, xerrors.Errorf("must pass sector number"))
 		}
 
+		really := cctx.Bool("really-do-it")
+		if !really {
+			//nolint:golint
+			return fmt.Errorf("--really-do-it must be specified for this action to have an effect; you have been warned")
+		}
+
 		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
 		if err != nil {
 			return err

documentation/en/api-v0-methods-miner.md

@@ -310,6 +310,7 @@ Inputs:
 [
   8,
   null,
+  null,
   true
 ]
 ```

documentation/en/cli-lotus-miner.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-miner [global options] command [command options] [arguments...]
 
 VERSION:
-   1.14.1
+   1.14.4
 
 COMMANDS:
    init     Initialize a lotus miner repo
@@ -1757,7 +1757,8 @@ USAGE:
    lotus-miner sectors abort-upgrade [command options] <sectorNum>
 
 OPTIONS:
-   --help, -h  show help (default: false)
+   --really-do-it  pass this flag if you know what you are doing (default: false)
+   --help, -h      show help (default: false)
    
 ```
 

documentation/en/cli-lotus-worker.md

@@ -7,7 +7,7 @@ USAGE:
    lotus-worker [global options] command [command options] [arguments...]
 
 VERSION:
-   1.14.1
+   1.14.4
 
 COMMANDS:
    run         Start lotus worker

documentation/en/cli-lotus.md

@@ -7,7 +7,7 @@ USAGE:
    lotus [global options] command [command options] [arguments...]
 
 VERSION:
-   1.14.1
+   1.14.4
 
 COMMANDS:
    daemon   Start a lotus daemon process

extern/sector-storage/faults.go

@@ -19,11 +19,11 @@ import (
 
 // FaultTracker TODO: Track things more actively
 type FaultTracker interface {
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error)
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, rg storiface.RGetter) (map[abi.SectorID]string, error)
 }
 
 // CheckProvable returns unprovable sectors
-func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) {
+func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, rg storiface.RGetter) (map[abi.SectorID]string, error) {
 	var bad = make(map[abi.SectorID]string)
 
 	ssize, err := pp.SectorSize()
@@ -32,72 +32,76 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
 	}
 
 	// TODO: More better checks
-	for _, sector := range sectors {
+	for i, sector := range sectors {
 		err := func() error {
 			ctx, cancel := context.WithCancel(ctx)
 			defer cancel()
+			var fReplica string
+			var fCache string
 
-			locked, err := m.index.StorageTryLock(ctx, sector.ID, storiface.FTSealed|storiface.FTCache, storiface.FTNone)
-			if err != nil {
-				return xerrors.Errorf("acquiring sector lock: %w", err)
-			}
-
-			if !locked {
-				log.Warnw("CheckProvable Sector FAULT: can't acquire read lock", "sector", sector)
-				bad[sector.ID] = fmt.Sprint("can't acquire read lock")
-				return nil
-			}
-
-			lp, _, err := m.localStore.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
-			if err != nil {
-				log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err)
-				bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err)
-				return nil
-			}
-
-			// temporary hack to make the check work with snapdeals
-			// will go away in https://github.com/filecoin-project/lotus/pull/7971
-			if lp.Sealed == "" || lp.Cache == "" {
-				// maybe it's update
+			if update[i] {
 				lockedUpdate, err := m.index.StorageTryLock(ctx, sector.ID, storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTNone)
 				if err != nil {
 					return xerrors.Errorf("acquiring sector lock: %w", err)
 				}
-				if lockedUpdate {
-					lp, _, err = m.localStore.AcquireSector(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
-					if err != nil {
-						log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err)
-						bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err)
-						return nil
-					}
-					lp.Sealed, lp.Cache = lp.Update, lp.UpdateCache
+				if !lockedUpdate {
+					log.Warnw("CheckProvable Sector FAULT: can't acquire read lock on update replica", "sector", sector)
+					bad[sector.ID] = fmt.Sprint("can't acquire read lock")
+					return nil
 				}
+				lp, _, err := m.localStore.AcquireSector(ctx, sector, storiface.FTUpdate|storiface.FTUpdateCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
+				if err != nil {
+					log.Warnw("CheckProvable Sector FAULT: acquire sector update replica in checkProvable", "sector", sector, "error", err)
+					bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err)
+					return nil
+				}
+				fReplica, fCache = lp.Update, lp.UpdateCache
+			} else {
+				locked, err := m.index.StorageTryLock(ctx, sector.ID, storiface.FTSealed|storiface.FTCache, storiface.FTNone)
+				if err != nil {
+					return xerrors.Errorf("acquiring sector lock: %w", err)
+				}
+
+				if !locked {
+					log.Warnw("CheckProvable Sector FAULT: can't acquire read lock", "sector", sector)
+					bad[sector.ID] = fmt.Sprint("can't acquire read lock")
+					return nil
+				}
+
+				lp, _, err := m.localStore.AcquireSector(ctx, sector, storiface.FTSealed|storiface.FTCache, storiface.FTNone, storiface.PathStorage, storiface.AcquireMove)
+				if err != nil {
+					log.Warnw("CheckProvable Sector FAULT: acquire sector in checkProvable", "sector", sector, "error", err)
+					bad[sector.ID] = fmt.Sprintf("acquire sector failed: %s", err)
+					return nil
+				}
+				fReplica, fCache = lp.Sealed, lp.Cache
+
 			}
 
-			if lp.Sealed == "" || lp.Cache == "" {
-				log.Warnw("CheckProvable Sector FAULT: cache and/or sealed paths not found", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache)
-				bad[sector.ID] = fmt.Sprintf("cache and/or sealed paths not found, cache %q, sealed %q", lp.Cache, lp.Sealed)
+			if fReplica == "" || fCache == "" {
+				log.Warnw("CheckProvable Sector FAULT: cache and/or sealed paths not found", "sector", sector, "sealed", fReplica, "cache", fCache)
+				bad[sector.ID] = fmt.Sprintf("cache and/or sealed paths not found, cache %q, sealed %q", fCache, fReplica)
 				return nil
 			}
 
 			toCheck := map[string]int64{
-				lp.Sealed:                        1,
-				filepath.Join(lp.Cache, "p_aux"): 0,
+				fReplica:                       1,
+				filepath.Join(fCache, "p_aux"): 0,
 			}
 
-			addCachePathsForSectorSize(toCheck, lp.Cache, ssize)
+			addCachePathsForSectorSize(toCheck, fCache, ssize)
 
 			for p, sz := range toCheck {
 				st, err := os.Stat(p)
 				if err != nil {
-					log.Warnw("CheckProvable Sector FAULT: sector file stat error", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "err", err)
+					log.Warnw("CheckProvable Sector FAULT: sector file stat error", "sector", sector, "sealed", fReplica, "cache", fCache, "file", p, "err", err)
 					bad[sector.ID] = fmt.Sprintf("%s", err)
 					return nil
 				}
 
 				if sz != 0 {
 					if st.Size() != int64(ssize)*sz {
-						log.Warnw("CheckProvable Sector FAULT: sector file is wrong size", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "file", p, "size", st.Size(), "expectSize", int64(ssize)*sz)
+						log.Warnw("CheckProvable Sector FAULT: sector file is wrong size", "sector", sector, "sealed", fReplica, "cache", fCache, "file", p, "size", st.Size(), "expectSize", int64(ssize)*sz)
 						bad[sector.ID] = fmt.Sprintf("%s is wrong size (got %d, expect %d)", p, st.Size(), int64(ssize)*sz)
 						return nil
 					}
@@ -118,14 +122,14 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
 					sector.ID.Number,
 				})
 				if err != nil {
-					log.Warnw("CheckProvable Sector FAULT: generating challenges", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err)
+					log.Warnw("CheckProvable Sector FAULT: generating challenges", "sector", sector, "sealed", fReplica, "cache", fCache, "err", err)
 					bad[sector.ID] = fmt.Sprintf("generating fallback challenges: %s", err)
 					return nil
 				}
 
 				commr, err := rg(ctx, sector.ID)
 				if err != nil {
-					log.Warnw("CheckProvable Sector FAULT: getting commR", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err)
+					log.Warnw("CheckProvable Sector FAULT: getting commR", "sector", sector, "sealed", fReplica, "cache", fCache, "err", err)
 					bad[sector.ID] = fmt.Sprintf("getting commR: %s", err)
 					return nil
 				}
@@ -136,12 +140,12 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof,
 						SectorNumber: sector.ID.Number,
 						SealedCID:    commr,
 					},
-					CacheDirPath:     lp.Cache,
+					CacheDirPath:     fCache,
 					PoStProofType:    wpp,
-					SealedSectorPath: lp.Sealed,
+					SealedSectorPath: fReplica,
 				}, ch.Challenges[sector.ID.Number])
 				if err != nil {
-					log.Warnw("CheckProvable Sector FAULT: generating vanilla proof", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err)
+					log.Warnw("CheckProvable Sector FAULT: generating vanilla proof", "sector", sector, "sealed", fReplica, "cache", fCache, "err", err)
 					bad[sector.ID] = fmt.Sprintf("generating vanilla proof: %s", err)
 					return nil
 				}

extern/sector-storage/mock/mock.go

@@ -505,7 +505,7 @@ func (mgr *SectorMgr) Remove(ctx context.Context, sector storage.SectorRef) erro
 	return nil
 }
 
-func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) {
+func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef, update []bool, rg storiface.RGetter) (map[abi.SectorID]string, error) {
 	bad := map[abi.SectorID]string{}
 
 	for _, sid := range ids {

extern/sector-storage/stores/http_handler.go

@@ -294,6 +294,10 @@ func ftFromString(t string) (storiface.SectorFileType, error) {
 		return storiface.FTSealed, nil
 	case storiface.FTCache.String():
 		return storiface.FTCache, nil
+	case storiface.FTUpdate.String():
+		return storiface.FTUpdate, nil
+	case storiface.FTUpdateCache.String():
+		return storiface.FTUpdateCache, nil
 	default:
 		return 0, xerrors.Errorf("unknown sector file type: '%s'", t)
 	}

extern/storage-sealing/fsm.go

@@ -19,7 +19,13 @@ import (
 func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface{}, uint64, error) {
 	next, processed, err := m.plan(events, user.(*SectorInfo))
 	if err != nil || next == nil {
-		return nil, processed, err
+		l := Log{
+			Timestamp: uint64(time.Now().Unix()),
+			Message:   fmt.Sprintf("state machine error: %s", err),
+			Kind:      fmt.Sprintf("error;%T", err),
+		}
+		user.(*SectorInfo).logAppend(l)
+		return nil, processed, nil
 	}
 
 	return func(ctx statemachine.Context, si SectorInfo) error {
@@ -175,6 +181,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	),
 	FinalizeReplicaUpdate: planOne(
 		on(SectorFinalized{}, UpdateActivating),
+		on(SectorFinalizeFailed{}, FinalizeReplicaUpdateFailed),
 	),
 	UpdateActivating: planOne(
 		on(SectorUpdateActive{}, ReleaseSectorKey),
@@ -267,6 +274,9 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	ReleaseSectorKeyFailed: planOne(
 		on(SectorUpdateActive{}, ReleaseSectorKey),
 	),
+	FinalizeReplicaUpdateFailed: planOne(
+		on(SectorRetryFinalize{}, FinalizeReplicaUpdate),
+	),
 
 	// Post-seal
 
@@ -309,6 +319,21 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	FailedUnrecoverable: final,
 }
 
+func (state *SectorInfo) logAppend(l Log) {
+	if len(state.Log) > 8000 {
+		log.Warnw("truncating sector log", "sector", state.SectorNumber)
+		state.Log[2000] = Log{
+			Timestamp: uint64(time.Now().Unix()),
+			Message:   "truncating log (above 8000 entries)",
+			Kind:      fmt.Sprintf("truncate"),
+		}
+
+		state.Log = append(state.Log[:2000], state.Log[6000:]...)
+	}
+
+	state.Log = append(state.Log, l)
+}
+
 func (m *Sealing) logEvents(events []statemachine.Event, state *SectorInfo) {
 	for _, event := range events {
 		log.Debugw("sector event", "sector", state.SectorNumber, "type", fmt.Sprintf("%T", event.User), "event", event.User)
@@ -337,18 +362,7 @@ func (m *Sealing) logEvents(events []statemachine.Event, state *SectorInfo) {
 			l.Trace = fmt.Sprintf("%+v", err)
 		}
 
-		if len(state.Log) > 8000 {
-			log.Warnw("truncating sector log", "sector", state.SectorNumber)
-			state.Log[2000] = Log{
-				Timestamp: uint64(time.Now().Unix()),
-				Message:   "truncating log (above 8000 entries)",
-				Kind:      fmt.Sprintf("truncate"),
-			}
-
-			state.Log = append(state.Log[:2000], state.Log[6000:]...)
-		}
-
-		state.Log = append(state.Log, l)
+		state.logAppend(l)
 	}
 }
 
@@ -536,6 +550,8 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		return m.handleSubmitReplicaUpdateFailed, processed, nil
 	case ReleaseSectorKeyFailed:
 		return m.handleReleaseSectorKeyFailed, 0, err
+	case FinalizeReplicaUpdateFailed:
+		return m.handleFinalizeFailed, processed, nil
 	case AbortUpgrade:
 		return m.handleAbortUpgrade, processed, nil
 

extern/storage-sealing/sector_state.go

@@ -3,64 +3,65 @@ package sealing
 type SectorState string
 
 var ExistSectorStateList = map[SectorState]struct{}{
-	Empty:                   {},
-	WaitDeals:               {},
-	Packing:                 {},
-	AddPiece:                {},
-	AddPieceFailed:          {},
-	GetTicket:               {},
-	PreCommit1:              {},
-	PreCommit2:              {},
-	PreCommitting:           {},
-	PreCommitWait:           {},
-	SubmitPreCommitBatch:    {},
-	PreCommitBatchWait:      {},
-	WaitSeed:                {},
-	Committing:              {},
-	CommitFinalize:          {},
-	CommitFinalizeFailed:    {},
-	SubmitCommit:            {},
-	CommitWait:              {},
-	SubmitCommitAggregate:   {},
-	CommitAggregateWait:     {},
-	FinalizeSector:          {},
-	Proving:                 {},
-	FailedUnrecoverable:     {},
-	SealPreCommit1Failed:    {},
-	SealPreCommit2Failed:    {},
-	PreCommitFailed:         {},
-	ComputeProofFailed:      {},
-	CommitFailed:            {},
-	PackingFailed:           {},
-	FinalizeFailed:          {},
-	DealsExpired:            {},
-	RecoverDealIDs:          {},
-	Faulty:                  {},
-	FaultReported:           {},
-	FaultedFinal:            {},
-	Terminating:             {},
-	TerminateWait:           {},
-	TerminateFinality:       {},
-	TerminateFailed:         {},
-	Removing:                {},
-	RemoveFailed:            {},
-	Removed:                 {},
-	SnapDealsWaitDeals:      {},
-	SnapDealsAddPiece:       {},
-	SnapDealsPacking:        {},
-	UpdateReplica:           {},
-	ProveReplicaUpdate:      {},
-	SubmitReplicaUpdate:     {},
-	ReplicaUpdateWait:       {},
-	UpdateActivating:        {},
-	ReleaseSectorKey:        {},
-	FinalizeReplicaUpdate:   {},
-	SnapDealsAddPieceFailed: {},
-	SnapDealsDealsExpired:   {},
-	SnapDealsRecoverDealIDs: {},
-	ReplicaUpdateFailed:     {},
-	ReleaseSectorKeyFailed:  {},
-	AbortUpgrade:            {},
+	Empty:                       {},
+	WaitDeals:                   {},
+	Packing:                     {},
+	AddPiece:                    {},
+	AddPieceFailed:              {},
+	GetTicket:                   {},
+	PreCommit1:                  {},
+	PreCommit2:                  {},
+	PreCommitting:               {},
+	PreCommitWait:               {},
+	SubmitPreCommitBatch:        {},
+	PreCommitBatchWait:          {},
+	WaitSeed:                    {},
+	Committing:                  {},
+	CommitFinalize:              {},
+	CommitFinalizeFailed:        {},
+	SubmitCommit:                {},
+	CommitWait:                  {},
+	SubmitCommitAggregate:       {},
+	CommitAggregateWait:         {},
+	FinalizeSector:              {},
+	Proving:                     {},
+	FailedUnrecoverable:         {},
+	SealPreCommit1Failed:        {},
+	SealPreCommit2Failed:        {},
+	PreCommitFailed:             {},
+	ComputeProofFailed:          {},
+	CommitFailed:                {},
+	PackingFailed:               {},
+	FinalizeFailed:              {},
+	DealsExpired:                {},
+	RecoverDealIDs:              {},
+	Faulty:                      {},
+	FaultReported:               {},
+	FaultedFinal:                {},
+	Terminating:                 {},
+	TerminateWait:               {},
+	TerminateFinality:           {},
+	TerminateFailed:             {},
+	Removing:                    {},
+	RemoveFailed:                {},
+	Removed:                     {},
+	SnapDealsWaitDeals:          {},
+	SnapDealsAddPiece:           {},
+	SnapDealsPacking:            {},
+	UpdateReplica:               {},
+	ProveReplicaUpdate:          {},
+	SubmitReplicaUpdate:         {},
+	ReplicaUpdateWait:           {},
+	UpdateActivating:            {},
+	ReleaseSectorKey:            {},
+	FinalizeReplicaUpdate:       {},
+	SnapDealsAddPieceFailed:     {},
+	SnapDealsDealsExpired:       {},
+	SnapDealsRecoverDealIDs:     {},
+	ReplicaUpdateFailed:         {},
+	ReleaseSectorKeyFailed:      {},
+	FinalizeReplicaUpdateFailed: {},
+	AbortUpgrade:                {},
 }
 
 // cmd/lotus-miner/info.go defines CLI colors corresponding to these states
@@ -124,12 +125,13 @@ const (
 	RecoverDealIDs       SectorState = "RecoverDealIDs"
 
 	// snap deals error modes
-	SnapDealsAddPieceFailed SectorState = "SnapDealsAddPieceFailed"
-	SnapDealsDealsExpired   SectorState = "SnapDealsDealsExpired"
-	SnapDealsRecoverDealIDs SectorState = "SnapDealsRecoverDealIDs"
-	AbortUpgrade            SectorState = "AbortUpgrade"
-	ReplicaUpdateFailed     SectorState = "ReplicaUpdateFailed"
-	ReleaseSectorKeyFailed  SectorState = "ReleaseSectorKeyFailed"
+	SnapDealsAddPieceFailed     SectorState = "SnapDealsAddPieceFailed"
+	SnapDealsDealsExpired       SectorState = "SnapDealsDealsExpired"
+	SnapDealsRecoverDealIDs     SectorState = "SnapDealsRecoverDealIDs"
+	AbortUpgrade                SectorState = "AbortUpgrade"
+	ReplicaUpdateFailed         SectorState = "ReplicaUpdateFailed"
+	ReleaseSectorKeyFailed      SectorState = "ReleaseSectorKeyFailed"
+	FinalizeReplicaUpdateFailed SectorState = "FinalizeReplicaUpdateFailed"
 
 	Faulty        SectorState = "Faulty"        // sector is corrupted or gone for some reason
 	FaultReported SectorState = "FaultReported" // sector has been declared as a fault on chain

go.mod

@@ -2,6 +2,8 @@ module github.com/filecoin-project/lotus
 
 go 1.16
 
+retract v1.14.0 // Accidentally force-pushed tag, use v1.14.1+ instead.
+
 require (
 	contrib.go.opencensus.io/exporter/jaeger v0.2.1
 	contrib.go.opencensus.io/exporter/prometheus v0.4.0

node/impl/storminer.go

@@ -960,7 +960,7 @@ func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error
 	return backup(sm.DS, fpath)
 }
 
-func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) {
+func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef, update []bool, expensive bool) (map[abi.SectorNumber]string, error) {
 	var rg storiface.RGetter
 	if expensive {
 		rg = func(ctx context.Context, id abi.SectorID) (cid.Cid, error) {
@@ -976,7 +976,7 @@ func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredP
 		}
 	}
 
-	bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors, rg)
+	bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors, update, rg)
 	if err != nil {
 		return nil, err
 	}

storage/wdpost_run.go

@@ -205,6 +205,7 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B
 
 	sectors := make(map[abi.SectorNumber]struct{})
 	var tocheck []storage.SectorRef
+	var update []bool
 	for _, info := range sectorInfos {
 		sectors[info.SectorNumber] = struct{}{}
 		tocheck = append(tocheck, storage.SectorRef{
@@ -214,9 +215,10 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B
 				Number: info.SectorNumber,
 			},
 		})
+		update = append(update, info.SectorKeyCID != nil)
 	}
 
-	bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck, nil)
+	bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck, update, nil)
 	if err != nil {
 		return bitfield.BitField{}, xerrors.Errorf("checking provable sectors: %w", err)
 	}

storage/wdpost_run_test.go

@@ -168,7 +168,7 @@ func (m mockVerif) GenerateWinningPoStSectorChallenge(context.Context, abi.Regis
 type mockFaultTracker struct {
 }
 
-func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) {
+func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, rg storiface.RGetter) (map[abi.SectorID]string, error) {
 	// Returns "bad" sectors so just return empty map meaning all sectors are good
 	return map[abi.SectorID]string{}, nil
 }