Lotus version 1.5.3

Chore/backport nerpa reset to 1 5 3

CHANGELOG.md

@@ -1,37 +1,38 @@
 # Lotus changelog
 
+# 1.5.3-rc2 / 2021-03-17
+
+This is the second release candidate for Lotus 1.5.3, which introduces small fixes to the Storage FSM.
+
+## Changes 
+
+- storagefsm: Fix double unlock with ready WaitDeals sectors (https://github.com/filecoin-project/lotus/pull/5783)
+- backupds: Allow larger values in write log (https://github.com/filecoin-project/lotus/pull/5776)
+- storagefsm: Don't log the SectorRestart event (https://github.com/filecoin-project/lotus/pull/5779)
+
+# 1.5.2 / 2021-03-11
+
+This is an hotfix release of Lotus that fixes a critical bug introduced in v1.5.1 in the miner windowPoSt logic. This upgrade is only affecting miner nodes.
+
+## Changes
+- fix window post rand check (https://github.com/filecoin-project/lotus/pull/5773)
+- wdpost: Always use head tipset to get randomness (https://github.com/filecoin-project/lotus/pull/5774)
+
 # 1.5.1 / 2021-03-10
 
 This is an optional release of Lotus that introduces an important fix to the WindowPoSt computation process. The change is to wait for some confidence before drawing beacon randomness for the proof. Without this, invalid proofs might be generated as the result of a null tipset. 
 
 ## Splitstore
 
-This release also introduces the splitstore, a new optional blockstore that
-segregates the monolithic blockstore into cold and hot regions. The
-hot region contains objects from the last 4-5 finalities plus all
-reachable objects from two finalities away. All other objects are
-moved to the cold region using a compaction process that executes
-every finality, once 5 finalities have elapsed.
-The splitstore allows us to separate the two regions quite
-effectively, using two separate badger blockstores. The separation
-means that the live working set is much smaller, which results in
-potentially significant performance improvements. In addition, it
-means that the coldstore can be moved to a separate (bigger, slower,
-cheaper) disk without loss of performance.
-The design also allows us to use different implementations for the two
-blockstores; for example, an append-only blockstore could be used for
-coldstore and a faster memory mapped blockstore could be used for the
-hotstore (eg LMDB). We plan to experiment with these options in the
-future.
-Once the splitstore has been enabled, the existing monolithic
-blockstore becomes the coldstore. On the first head change
-notification, the splitstore will warm up the hotstore by copying all
-reachable objects from the current tipset into the hotstore.  All new
-writes go into the hotstore, with the splitstore tracking the write
-epoch. Once 5 finalities have elapsed, and every finality thereafter,
-the splitstore compacts by moving cold objects into the coldstore.
-There is also experimental support for garbage collection, whereby
-unreachable objects are simply discarded.
+This release also introduces the splitstore, a new optional blockstore that segregates the monolithic blockstore into cold and hot regions. The hot region contains objects from the last 4-5 finalities plus all reachable objects from two finalities away. All other objects are moved to the cold region using a compaction process that executes every finality, once 5 finalities have elapsed.
+
+The splitstore allows us to separate the two regions quite effectively, using two separate badger blockstores. The separation
+means that the live working set is much smaller, which results in potentially significant performance improvements. In addition, it means that the coldstore can be moved to a separate (bigger, slower, cheaper) disk without loss of performance.
+
+The design also allows us to use different implementations for the two blockstores; for example, an append-only blockstore could be used for coldstore and a faster memory mapped blockstore could be used for the hotstore (eg LMDB). We plan to experiment with these options in the future.
+
+Once the splitstore has been enabled, the existing monolithic blockstore becomes the coldstore. On the first head change notification, the splitstore will warm up the hotstore by copying all reachable objects from the current tipset into the hotstore.  All new writes go into the hotstore, with the splitstore tracking the write epoch. Once 5 finalities have elapsed, and every finality thereafter, the splitstore compacts by moving cold objects into the coldstore. There is also experimental support for garbage collection, whereby nunreachable objects are simply discarded.
+
 To enable the splitstore, add the following to config.toml:
 
 ```

Makefile

@@ -63,20 +63,23 @@ CLEAN+=build/.update-modules
 deps: $(BUILD_DEPS)
 .PHONY: deps
 
+build-devnets: build lotus-seed lotus-shed lotus-wallet lotus-gateway
+.PHONY: build-devnets
+
 debug: GOFLAGS+=-tags=debug
-debug: lotus lotus-miner lotus-worker lotus-seed
+debug: build-devnets
 
 2k: GOFLAGS+=-tags=2k
-2k: lotus lotus-miner lotus-worker lotus-seed
+2k: build-devnets
 
 calibnet: GOFLAGS+=-tags=calibnet
-calibnet: lotus lotus-miner lotus-worker lotus-seed
+calibnet: build-devnets
 
 nerpanet: GOFLAGS+=-tags=nerpanet
-nerpanet: lotus lotus-miner lotus-worker lotus-seed
+nerpanet: build-devnets
 
 butterflynet: GOFLAGS+=-tags=butterflynet
-butterflynet: lotus lotus-miner lotus-worker lotus-seed
+butterflynet: build-devnets
 
 lotus: $(BUILD_DEPS)
 	rm -f lotus

build/bootstrap/nerpanet.pi

@@ -1,4 +1,4 @@
-/dns4/bootstrap-0.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWNfuGjtzWTVz8eJGZ2C3aJg2xLqorhsagu4LTWw6CwpK9
-/dns4/bootstrap-1.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWDfsxYk7dC6NNsHqZqqyMJCzkjZuXhjsmqBk3TUCBZLga
-/dns4/bootstrap-2.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWRZAGHmCCaa2gkYmnC4Q2TEwHGFSh6Fh1FFJ7RSXak5yN
-/dns4/bootstrap-3.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWBFxEigSKLvxJVdw3JziC9ePHHnyAn5LifWSqg2kttcth
+/dns4/bootstrap-2.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWQcL6ReWmR6ASWx4iT7EiAmxKDQpvgq1MKNTQZp5NPnWW
+/dns4/bootstrap-0.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWGyJCwCm7EfupM15CFPXM4c7zRVHwwwjcuy9umaGeztMX
+/dns4/bootstrap-3.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWNK9RmfksKXSCQj7ZwAM7L6roqbN4kwJteihq7yPvSgPs
+/dns4/bootstrap-1.nerpa.interplanetary.dev/tcp/1347/p2p/12D3KooWCWSaH6iUyXYspYxELjDfzToBsyVGVz3QvC7ysXv7wESo

build/params_nerpanet.go

@@ -24,21 +24,21 @@ const UpgradeSmokeHeight = -1
 const UpgradeIgnitionHeight = -2
 const UpgradeRefuelHeight = -3
 
-const UpgradeTapeHeight = -4
-
 const UpgradeLiftoffHeight = -5
-const UpgradeActorsV2Height = 120 // critical: the network can bootstrap from v1 only
 
-const UpgradeKumquatHeight = -6
+const UpgradeActorsV2Height = 30 // critical: the network can bootstrap from v1 only
+const UpgradeTapeHeight = 60
 
-const UpgradeCalicoHeight = 306000
-const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 12)
+const UpgradeKumquatHeight = 90
 
-const UpgradeOrangeHeight = 307500
+const UpgradeCalicoHeight = 100
+const UpgradePersianHeight = UpgradeCalicoHeight + (builtin2.EpochsInHour * 1)
 
-const UpgradeClausHeight = 307600
+const UpgradeClausHeight = 250
 
-const UpgradeActorsV3Height = 308000
+const UpgradeOrangeHeight = 300
+
+const UpgradeActorsV3Height = 600
 
 func init() {
 	// Minimum block production power is set to 4 TiB

build/version.go

@@ -29,7 +29,7 @@ func buildType() string {
 }
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "1.5.1"
+const BuildVersion = "1.5.3"
 
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit

chain/stmgr/forks.go

@@ -852,6 +852,10 @@ func UpgradeLiftoff(ctx context.Context, sm *StateManager, _ MigrationCache, cb
 }
 
 func UpgradeCalico(ctx context.Context, sm *StateManager, _ MigrationCache, cb ExecCallback, root cid.Cid, epoch abi.ChainEpoch, ts *types.TipSet) (cid.Cid, error) {
+	if build.BuildType != build.BuildMainnet {
+		return root, nil
+	}
+
 	store := sm.cs.ActorStore(ctx)
 	var stateRoot types.StateRoot
 	if err := store.Get(ctx, root, &stateRoot); err != nil {

cmd/lotus-seed/genesis.go

@@ -334,11 +334,6 @@ var genesisSetVRKCmd = &cli.Command{
 			return err
 		}
 
-		csvf, err := homedir.Expand(cctx.Args().Get(1))
-		if err != nil {
-			return err
-		}
-
 		var template genesis.Template
 		b, err := ioutil.ReadFile(genf)
 		if err != nil {
@@ -363,6 +358,10 @@ var genesisSetVRKCmd = &cli.Command{
 				Meta:    am.ActorMeta(),
 			}
 		} else if cctx.IsSet("multisig") {
+			csvf, err := homedir.Expand(cctx.String("multisig"))
+			if err != nil {
+				return err
+			}
 
 			entries, err := parseMultisigCsv(csvf)
 			if err != nil {
@@ -431,11 +430,6 @@ var genesisSetRemainderCmd = &cli.Command{
 			return err
 		}
 
-		csvf, err := homedir.Expand(cctx.Args().Get(1))
-		if err != nil {
-			return err
-		}
-
 		var template genesis.Template
 		b, err := ioutil.ReadFile(genf)
 		if err != nil {
@@ -460,6 +454,10 @@ var genesisSetRemainderCmd = &cli.Command{
 				Meta:    am.ActorMeta(),
 			}
 		} else if cctx.IsSet("multisig") {
+			csvf, err := homedir.Expand(cctx.String("multisig"))
+			if err != nil {
+				return err
+			}
 
 			entries, err := parseMultisigCsv(csvf)
 			if err != nil {

extern/storage-sealing/fsm.go

@@ -191,10 +191,7 @@ var fsmPlanners = map[SectorState]func(events []statemachine.Event, state *Secto
 	FailedUnrecoverable: final,
 }
 
-func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(statemachine.Context, SectorInfo) error, uint64, error) {
-	/////
-	// First process all events
-
+func (m *Sealing) logEvents(events []statemachine.Event, state *SectorInfo) {
 	for _, event := range events {
 		e, err := json.Marshal(event)
 		if err != nil {
@@ -202,6 +199,10 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 			continue
 		}
 
+		if event.User == (SectorRestart{}) {
+			continue // don't log on every fsm restart
+		}
+
 		l := Log{
 			Timestamp: uint64(time.Now().Unix()),
 			Message:   string(e),
@@ -225,6 +226,13 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 
 		state.Log = append(state.Log, l)
 	}
+}
+
+func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(statemachine.Context, SectorInfo) error, uint64, error) {
+	/////
+	// First process all events
+
+	m.logEvents(events, state)
 
 	if m.notifee != nil {
 		defer func(before SectorInfo) {

extern/storage-sealing/input.go

@@ -93,7 +93,6 @@ func (m *Sealing) maybeStartSealing(ctx statemachine.Context, sector SectorInfo,
 	if sector.CreationTime != 0 {
 		cfg, err := m.getConfig()
 		if err != nil {
-			m.inputLk.Unlock()
 			return false, xerrors.Errorf("getting storage config: %w", err)
 		}
 
@@ -101,7 +100,6 @@ func (m *Sealing) maybeStartSealing(ctx statemachine.Context, sector SectorInfo,
 		sealTime := time.Unix(sector.CreationTime, 0).Add(cfg.WaitDealsDelay)
 
 		if now.After(sealTime) {
-			m.inputLk.Unlock()
 			log.Infow("starting to seal deal sector", "sector", sector.SectorNumber, "trigger", "wait-timeout")
 			return true, ctx.Send(SectorStartPacking{})
 		}

gen/main.go

@@ -12,7 +12,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 	"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
-	"github.com/filecoin-project/lotus/lib/backupds"
 	"github.com/filecoin-project/lotus/node/hello"
 	"github.com/filecoin-project/lotus/paychmgr"
 )
@@ -105,12 +104,4 @@ func main() {
 		fmt.Println(err)
 		os.Exit(1)
 	}
-
-	err = gen.WriteTupleEncodersToFile("./lib/backupds/cbor_gen.go", "backupds",
-		backupds.Entry{},
-	)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
 }

go.mod

@@ -39,7 +39,7 @@ require (
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261
 	github.com/filecoin-project/go-state-types v0.1.0
 	github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe
-	github.com/filecoin-project/go-statestore v0.1.0
+	github.com/filecoin-project/go-statestore v0.1.1-0.20210311122610-6c7a5aedbdea
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
 	github.com/filecoin-project/specs-actors v0.9.13
 	github.com/filecoin-project/specs-actors/v2 v2.3.4

go.sum

@@ -239,7 +239,6 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349/
 github.com/filecoin-project/go-amt-ipld/v3 v3.0.0 h1:Ou/q82QeHGOhpkedvaxxzpBYuqTxLCcj5OChkDNx4qc=
 github.com/filecoin-project/go-amt-ipld/v3 v3.0.0/go.mod h1:Qa95YNAbtoVCTSVtX38aAC1ptBnJfPma1R/zZsKmx4o=
 github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
-github.com/filecoin-project/go-bitfield v0.2.3 h1:pedK/7maYF06Z+BYJf2OeFFqIDEh6SP6mIOlLFpYXGs=
 github.com/filecoin-project/go-bitfield v0.2.3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
 github.com/filecoin-project/go-bitfield v0.2.3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
 github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW8p9au0C68JPgk=
@@ -283,8 +282,9 @@ github.com/filecoin-project/go-state-types v0.1.0 h1:9r2HCSMMCmyMfGyMKxQtv0GKp6V
 github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw=
 github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
-github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ=
 github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
+github.com/filecoin-project/go-statestore v0.1.1-0.20210311122610-6c7a5aedbdea h1:EvmiCrHRrPF2mxVMIRdtsxZGByqwT24aJLw7mdzEFxE=
+github.com/filecoin-project/go-statestore v0.1.1-0.20210311122610-6c7a5aedbdea/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
 github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4=
@@ -1935,7 +1935,6 @@ howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCU
 modernc.org/cc v1.0.0 h1:nPibNuDEx6tvYrUAtvDTTw98rx5juGsa5zuDnKwEEQQ=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
 modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8=
-modernc.org/golex v1.0.0 h1:wWpDlbK8ejRfSyi0frMyhilD3JBvtcx2AdGDnU+JtsE=
 modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/golex v1.0.1 h1:EYKY1a3wStt0RzHaH8mdSRNg78Ub0OHxYfCRWw35YtM=

lib/backupds/cbor.go

@@ -1,21 +1,12 @@
-// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
-
 package backupds
 
 import (
 	"fmt"
 	"io"
-	"sort"
 
-	cid "github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
-	xerrors "golang.org/x/xerrors"
 )
 
-var _ = xerrors.Errorf
-var _ = cid.Undef
-var _ = sort.Sort
-
 var lengthBufEntry = []byte{131}
 
 func (t *Entry) MarshalCBOR(w io.Writer) error {
@@ -29,11 +20,6 @@ func (t *Entry) MarshalCBOR(w io.Writer) error {
 
 	scratch := make([]byte, 9)
 
-	// t.Key ([]uint8) (slice)
-	if len(t.Key) > cbg.ByteArrayMaxLen {
-		return xerrors.Errorf("Byte array in field t.Key was too long")
-	}
-
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Key))); err != nil {
 		return err
 	}
@@ -42,11 +28,6 @@ func (t *Entry) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
-	// t.Value ([]uint8) (slice)
-	if len(t.Value) > cbg.ByteArrayMaxLen {
-		return xerrors.Errorf("Byte array in field t.Value was too long")
-	}
-
 	if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajByteString, uint64(len(t.Value))); err != nil {
 		return err
 	}
@@ -93,9 +74,6 @@ func (t *Entry) UnmarshalCBOR(r io.Reader) error {
 		return err
 	}
 
-	if extra > cbg.ByteArrayMaxLen {
-		return fmt.Errorf("t.Key: byte array too large (%d)", extra)
-	}
 	if maj != cbg.MajByteString {
 		return fmt.Errorf("expected byte array")
 	}
@@ -114,9 +92,6 @@ func (t *Entry) UnmarshalCBOR(r io.Reader) error {
 		return err
 	}
 
-	if extra > cbg.ByteArrayMaxLen {
-		return fmt.Errorf("t.Value: byte array too large (%d)", extra)
-	}
 	if maj != cbg.MajByteString {
 		return fmt.Errorf("expected byte array")
 	}
@@ -151,7 +126,7 @@ func (t *Entry) UnmarshalCBOR(r io.Reader) error {
 			return fmt.Errorf("wrong type for int64 field: %d", maj)
 		}
 
-		t.Timestamp = int64(extraI)
+		t.Timestamp = extraI
 	}
 	return nil
 }

storage/wdpost_run.go

@@ -481,7 +481,12 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty
 		return nil, xerrors.Errorf("failed to marshal address to cbor: %w", err)
 	}
 
-	rand, err := s.api.ChainGetRandomnessFromBeacon(ctx, ts.Key(), crypto.DomainSeparationTag_WindowedPoStChallengeSeed, di.Challenge, buf.Bytes())
+	headTs, err := s.api.ChainHead(ctx)
+	if err != nil {
+		return nil, xerrors.Errorf("getting current head: %w", err)
+	}
+
+	rand, err := s.api.ChainGetRandomnessFromBeacon(ctx, headTs.Key(), crypto.DomainSeparationTag_WindowedPoStChallengeSeed, di.Challenge, buf.Bytes())
 	if err != nil {
 		return nil, xerrors.Errorf("failed to get chain randomness from beacon for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err)
 	}
@@ -589,7 +594,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty
 				return nil, err
 			}
 
-			postOut, ps, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, abi.PoStRandomness(rand))
+			postOut, ps, err := s.prover.GenerateWindowPoSt(ctx, abi.ActorID(mid), sinfos, append(abi.PoStRandomness{}, rand...))
 			elapsed := time.Since(tsStart)
 
 			log.Infow("computing window post", "batch", batchIdx, "elapsed", elapsed)