Merge pull request #4219 from filecoin-project/merge-master-to-next

Merge branch 'master' into merge-master-to-next

build/params_testnet.go

@@ -5,6 +5,8 @@
 package build
 
 import (
+	"os"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
@@ -39,7 +41,9 @@ func init() {
 		abi.RegisteredSealProof_StackedDrg64GiBV1,
 	)
 
-	SetAddressNetwork(address.Mainnet)
+	if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" {
+		SetAddressNetwork(address.Mainnet)
+	}
 
 	Devnet = false
 }

chain/actors/builtin/miner/diff_deadlines.go

@@ -0,0 +1,180 @@
+package miner
+
+import (
+	"errors"
+
+	"github.com/filecoin-project/go-bitfield"
+	"github.com/filecoin-project/go-state-types/exitcode"
+)
+
+type DeadlinesDiff map[uint64]*DeadlineDiff
+
+func DiffDeadlines(pre, cur State) (*DeadlinesDiff, error) {
+	changed, err := pre.DeadlinesChanged(cur)
+	if err != nil {
+		return nil, err
+	}
+	if !changed {
+		return nil, nil
+	}
+
+	numDl, err := pre.NumDeadlines()
+	if err != nil {
+		return nil, err
+	}
+	dlDiff := make(DeadlinesDiff, numDl)
+	if err := pre.ForEachDeadline(func(idx uint64, preDl Deadline) error {
+		curDl, err := cur.LoadDeadline(idx)
+		if err != nil {
+			return err
+		}
+
+		diff, err := DiffDeadline(preDl, curDl)
+		if err != nil {
+			return err
+		}
+
+		dlDiff[idx] = diff
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+	return &dlDiff, nil
+}
+
+type DeadlineDiff map[uint64]*PartitionDiff
+
+func DiffDeadline(pre, cur Deadline) (*DeadlineDiff, error) {
+	changed, err := pre.PartitionsChanged(cur)
+	if err != nil {
+		return nil, err
+	}
+	if !changed {
+		return nil, nil
+	}
+
+	partDiff := make(DeadlineDiff)
+	if err := pre.ForEachPartition(func(idx uint64, prePart Partition) error {
+		// try loading current partition at this index
+		curPart, err := cur.LoadPartition(idx)
+		if err != nil {
+			if errors.Is(err, exitcode.ErrNotFound) {
+				// TODO correctness?
+				return nil // the partition was removed.
+			}
+			return err
+		}
+
+		// compare it with the previous partition
+		diff, err := DiffPartition(prePart, curPart)
+		if err != nil {
+			return err
+		}
+
+		partDiff[idx] = diff
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	// all previous partitions have been walked.
+	// all partitions in cur and not in prev are new... can they be faulty already?
+	// TODO is this correct?
+	if err := cur.ForEachPartition(func(idx uint64, curPart Partition) error {
+		if _, found := partDiff[idx]; found {
+			return nil
+		}
+		faults, err := curPart.FaultySectors()
+		if err != nil {
+			return err
+		}
+		recovering, err := curPart.RecoveringSectors()
+		if err != nil {
+			return err
+		}
+		partDiff[idx] = &PartitionDiff{
+			Removed:    bitfield.New(),
+			Recovered:  bitfield.New(),
+			Faulted:    faults,
+			Recovering: recovering,
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return &partDiff, nil
+}
+
+type PartitionDiff struct {
+	Removed    bitfield.BitField
+	Recovered  bitfield.BitField
+	Faulted    bitfield.BitField
+	Recovering bitfield.BitField
+}
+
+func DiffPartition(pre, cur Partition) (*PartitionDiff, error) {
+	prevLiveSectors, err := pre.LiveSectors()
+	if err != nil {
+		return nil, err
+	}
+	curLiveSectors, err := cur.LiveSectors()
+	if err != nil {
+		return nil, err
+	}
+
+	removed, err := bitfield.SubtractBitField(prevLiveSectors, curLiveSectors)
+	if err != nil {
+		return nil, err
+	}
+
+	prevRecoveries, err := pre.RecoveringSectors()
+	if err != nil {
+		return nil, err
+	}
+
+	curRecoveries, err := cur.RecoveringSectors()
+	if err != nil {
+		return nil, err
+	}
+
+	recovering, err := bitfield.SubtractBitField(curRecoveries, prevRecoveries)
+	if err != nil {
+		return nil, err
+	}
+
+	prevFaults, err := pre.FaultySectors()
+	if err != nil {
+		return nil, err
+	}
+
+	curFaults, err := cur.FaultySectors()
+	if err != nil {
+		return nil, err
+	}
+
+	faulted, err := bitfield.SubtractBitField(curFaults, prevFaults)
+	if err != nil {
+		return nil, err
+	}
+
+	// all current good sectors
+	curActiveSectors, err := cur.ActiveSectors()
+	if err != nil {
+		return nil, err
+	}
+
+	// sectors that were previously fault and are now currently active are considered recovered.
+	recovered, err := bitfield.IntersectBitField(prevFaults, curActiveSectors)
+	if err != nil {
+		return nil, err
+	}
+
+	return &PartitionDiff{
+		Removed:    removed,
+		Recovered:  recovered,
+		Faulted:    faulted,
+		Recovering: recovering,
+	}, nil
+}

chain/vm/runtime.go

@@ -49,6 +49,9 @@ func (m *Message) ValueReceived() abi.TokenAmount {
 	return m.msg.Value
 }
 
+// EnableGasTracing, if true, outputs gas tracing in execution traces.
+var EnableGasTracing = false
+
 type Runtime struct {
 	rt0.Message
 	rt0.Syscalls
@@ -477,7 +480,7 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError {
 }
 
 func (rt *Runtime) finilizeGasTracing() {
-	if enableTracing {
+	if EnableGasTracing {
 		if rt.lastGasCharge != nil {
 			rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime)
 		}
@@ -509,11 +512,9 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) {
 
 }
 
-var enableTracing = false
-
 func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError {
 	toUse := gas.Total()
-	if enableTracing {
+	if EnableGasTracing {
 		var callers [10]uintptr
 
 		cout := 0 //gruntime.Callers(2+skip, callers[:])

chain/vm/runtime_test.go

@@ -45,3 +45,23 @@ func TestRuntimePutErrors(t *testing.T) {
 	rt.StorePut(&NotAVeryGoodMarshaler{})
 	t.Error("expected panic")
 }
+
+func BenchmarkRuntime_CreateRuntimeChargeGas_TracingDisabled(b *testing.B) {
+	var (
+		cst = cbor.NewCborStore(nil)
+		gch = newGasCharge("foo", 1000, 1000)
+	)
+
+	b.ResetTimer()
+
+	EnableGasTracing = false
+	noop := func() bool { return EnableGasTracing }
+	for n := 0; n < b.N; n++ {
+		// flip the value and access it to make sure
+		// the compiler doesn't optimize away
+		EnableGasTracing = true
+		_ = noop()
+		EnableGasTracing = false
+		_ = (&Runtime{cst: cst}).chargeGasInternal(gch, 0)
+	}
+}

chain/vm/vm.go

@@ -236,7 +236,7 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
 	}
 
 	rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac)
-	if enableTracing {
+	if EnableGasTracing {
 		rt.lastGasChargeTime = start
 		if parent != nil {
 			rt.lastGasChargeTime = parent.lastGasChargeTime

cli/state.go

@@ -70,6 +70,7 @@ var stateCmd = &cli.Command{
 		stateMsgCostCmd,
 		stateMinerInfo,
 		stateMarketCmd,
+		stateExecTraceCmd,
 	},
 }
 
@@ -313,6 +314,74 @@ var stateActiveSectorsCmd = &cli.Command{
 	},
 }
 
+var stateExecTraceCmd = &cli.Command{
+	Name:      "exec-trace",
+	Usage:     "Get the execution trace of a given message",
+	ArgsUsage: "<messageCid>",
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return ShowHelp(cctx, fmt.Errorf("must pass message cid"))
+		}
+
+		mcid, err := cid.Decode(cctx.Args().First())
+		if err != nil {
+			return fmt.Errorf("message cid was invalid: %s", err)
+		}
+
+		capi, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		msg, err := capi.ChainGetMessage(ctx, mcid)
+		if err != nil {
+			return err
+		}
+
+		lookup, err := capi.StateSearchMsg(ctx, mcid)
+		if err != nil {
+			return err
+		}
+
+		ts, err := capi.ChainGetTipSet(ctx, lookup.TipSet)
+		if err != nil {
+			return err
+		}
+
+		pts, err := capi.ChainGetTipSet(ctx, ts.Parents())
+		if err != nil {
+			return err
+		}
+
+		cso, err := capi.StateCompute(ctx, pts.Height(), nil, pts.Key())
+		if err != nil {
+			return err
+		}
+
+		var trace *api.InvocResult
+		for _, t := range cso.Trace {
+			if t.Msg.From == msg.From && t.Msg.Nonce == msg.Nonce {
+				trace = t
+				break
+			}
+		}
+		if trace == nil {
+			return fmt.Errorf("failed to find message in tipset trace output")
+		}
+
+		out, err := json.MarshalIndent(trace, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(out))
+		return nil
+	},
+}
+
 var stateReplaySetCmd = &cli.Command{
 	Name:      "replay",
 	Usage:     "Replay a particular message within a tipset",

cmd/lotus-pcr/main.go

@@ -37,6 +37,7 @@ import (
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/market"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/tools/stats"
@@ -342,6 +343,18 @@ var runCmd = &cli.Command{
 			Usage:   "process ProveCommitSector messages",
 			Value:   true,
 		},
+		&cli.BoolFlag{
+			Name:    "windowed-post",
+			EnvVars: []string{"LOTUS_PCR_WINDOWED_POST"},
+			Usage:   "process SubmitWindowedPoSt messages and refund gas fees",
+			Value:   false,
+		},
+		&cli.BoolFlag{
+			Name:    "storage-deals",
+			EnvVars: []string{"LOTUS_PCR_STORAGE_DEALS"},
+			Usage:   "process PublishStorageDeals messages and refund gas fees",
+			Value:   false,
+		},
 		&cli.IntFlag{
 			Name:    "head-delay",
 			EnvVars: []string{"LOTUS_PCR_HEAD_DELAY"},
@@ -378,6 +391,18 @@ var runCmd = &cli.Command{
 			Usage:   "percent of refund to issue",
 			Value:   110,
 		},
+		&cli.StringFlag{
+			Name:    "pre-fee-cap-max",
+			EnvVars: []string{"LOTUS_PCR_PRE_FEE_CAP_MAX"},
+			Usage:   "messages with a fee cap larger than this will be skipped when processing pre commit messages",
+			Value:   "0.0000000001",
+		},
+		&cli.StringFlag{
+			Name:    "prove-fee-cap-max",
+			EnvVars: []string{"LOTUS_PCR_PROVE_FEE_CAP_MAX"},
+			Usage:   "messages with a prove cap larger than this will be skipped when processing pre commit messages",
+			Value:   "0.0000000001",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		go func() {
@@ -421,6 +446,8 @@ var runCmd = &cli.Command{
 		dryRun := cctx.Bool("dry-run")
 		preCommitEnabled := cctx.Bool("pre-commit")
 		proveCommitEnabled := cctx.Bool("prove-commit")
+		windowedPoStEnabled := cctx.Bool("windowed-post")
+		publishStorageDealsEnabled := cctx.Bool("storage-deals")
 		aggregateTipsets := cctx.Int("aggregate-tipsets")
 		minerRecoveryEnabled := cctx.Bool("miner-recovery")
 		minerRecoveryPeriod := abi.ChainEpoch(int64(cctx.Int("miner-recovery-period")))
@@ -428,6 +455,16 @@ var runCmd = &cli.Command{
 		minerRecoveryCutoff := uint64(cctx.Int("miner-recovery-cutoff"))
 		minerRecoveryBonus := uint64(cctx.Int("miner-recovery-bonus"))
 
+		preFeeCapMax, err := types.ParseFIL(cctx.String("pre-fee-cap-max"))
+		if err != nil {
+			return err
+		}
+
+		proveFeeCapMax, err := types.ParseFIL(cctx.String("prove-fee-cap-max"))
+		if err != nil {
+			return err
+		}
+
 		rf := &refunder{
 			api:                        api,
 			wallet:                     from,
@@ -438,6 +475,10 @@ var runCmd = &cli.Command{
 			dryRun:                     dryRun,
 			preCommitEnabled:           preCommitEnabled,
 			proveCommitEnabled:         proveCommitEnabled,
+			windowedPoStEnabled:        windowedPoStEnabled,
+			publishStorageDealsEnabled: publishStorageDealsEnabled,
+			preFeeCapMax:               types.BigInt(preFeeCapMax),
+			proveFeeCapMax:             types.BigInt(proveFeeCapMax),
 		}
 
 		var refunds *MinersRefund = NewMinersRefund()
@@ -589,7 +630,12 @@ type refunder struct {
 	dryRun                     bool
 	preCommitEnabled           bool
 	proveCommitEnabled         bool
+	windowedPoStEnabled        bool
+	publishStorageDealsEnabled bool
 	threshold                  big.Int
+
+	preFeeCapMax   big.Int
+	proveFeeCapMax big.Int
 }
 
 func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, owner, worker, control bool) (*MinersRefund, error) {
@@ -817,6 +863,147 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet
 	return refunds, nil
 }
 
+func (r *refunder) processTipsetStorageMarketActor(ctx context.Context, tipset *types.TipSet, msg api.Message, recp *types.MessageReceipt) (bool, string, types.BigInt, error) {
+
+	m := msg.Message
+	refundValue := types.NewInt(0)
+	var messageMethod string
+
+	switch m.Method {
+	case builtin0.MethodsMarket.PublishStorageDeals:
+		if !r.publishStorageDealsEnabled {
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		messageMethod = "PublishStorageDeals"
+
+		if recp.ExitCode != exitcode.Ok {
+			log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee)
+	}
+
+	return true, messageMethod, refundValue, nil
+}
+
+func (r *refunder) processTipsetStorageMinerActor(ctx context.Context, tipset *types.TipSet, msg api.Message, recp *types.MessageReceipt) (bool, string, types.BigInt, error) {
+
+	m := msg.Message
+	refundValue := types.NewInt(0)
+	var messageMethod string
+
+	switch m.Method {
+	case builtin0.MethodsMiner.SubmitWindowedPoSt:
+		if !r.windowedPoStEnabled {
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		messageMethod = "SubmitWindowedPoSt"
+
+		if recp.ExitCode != exitcode.Ok {
+			log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		refundValue = types.BigMul(types.NewInt(uint64(recp.GasUsed)), tipset.Blocks()[0].ParentBaseFee)
+	case builtin0.MethodsMiner.ProveCommitSector:
+		if !r.proveCommitEnabled {
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		messageMethod = "ProveCommitSector"
+
+		if recp.ExitCode != exitcode.Ok {
+			log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		if m.GasFeeCap.GreaterThan(r.proveFeeCapMax) {
+			log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.proveFeeCapMax)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		var sn abi.SectorNumber
+
+		var proveCommitSector miner0.ProveCommitSectorParams
+		if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil {
+			log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		sn = proveCommitSector.SectorNumber
+
+		// We use the parent tipset key because precommit information is removed when ProveCommitSector is executed
+		precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, sn, tipset.Parents())
+		if err != nil {
+			log.Warnw("failed to get precommit info for sector", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		precommitTipset, err := r.api.ChainGetTipSetByHeight(ctx, precommitChainInfo.PreCommitEpoch, tipset.Key())
+		if err != nil {
+			log.Warnf("failed to lookup precommit epoch", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitChainInfo.Info, precommitTipset.Key())
+		if err != nil {
+			log.Warnw("failed to get initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		collateral = big.Sub(collateral, precommitChainInfo.PreCommitDeposit)
+		if collateral.LessThan(big.Zero()) {
+			log.Debugw("skipping zero pledge collateral difference", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		refundValue = collateral
+		if r.refundPercent > 0 {
+			refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent)))
+		}
+	case builtin0.MethodsMiner.PreCommitSector:
+		if !r.preCommitEnabled {
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		messageMethod = "PreCommitSector"
+
+		if recp.ExitCode != exitcode.Ok {
+			log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recp.ExitCode)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		if m.GasFeeCap.GreaterThan(r.preFeeCapMax) {
+			log.Debugw("skipping high fee cap message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "gas_fee_cap", m.GasFeeCap, "fee_cap_max", r.preFeeCapMax)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		var precommitInfo miner.SectorPreCommitInfo
+		if err := precommitInfo.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil {
+			log.Warnw("failed to decode precommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitInfo, tipset.Key())
+		if err != nil {
+			log.Warnw("failed to calculate initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", precommitInfo.SectorNumber)
+			return false, messageMethod, types.NewInt(0), nil
+		}
+
+		refundValue = collateral
+		if r.refundPercent > 0 {
+			refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent)))
+		}
+	default:
+		return false, messageMethod, types.NewInt(0), nil
+	}
+
+	return true, messageMethod, refundValue, nil
+}
+
 func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) {
 	cids := tipset.Cids()
 	if len(cids) == 0 {
@@ -852,91 +1039,28 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu
 			continue
 		}
 
-		if !builtin.IsStorageMinerActor(a.Code) {
-			continue
-		}
-
 		var messageMethod string
 
-		switch m.Method {
+		if m.To == market.Address {
-		case builtin0.MethodsMiner.ProveCommitSector:
+			var err error
-			if !r.proveCommitEnabled {
+			var processed bool
-				continue
+			processed, messageMethod, refundValue, err = r.processTipsetStorageMarketActor(ctx, tipset, msg, recps[i])
-			}
-
-			messageMethod = "ProveCommitSector"
-
-			if recps[i].ExitCode != exitcode.Ok {
-				log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recps[i].ExitCode)
-				continue
-			}
-
-			var sn abi.SectorNumber
-
-			var proveCommitSector miner0.ProveCommitSectorParams
-			if err := proveCommitSector.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil {
-				log.Warnw("failed to decode provecommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To)
-				continue
-			}
-
-			sn = proveCommitSector.SectorNumber
-
-			// We use the parent tipset key because precommit information is removed when ProveCommitSector is executed
-			precommitChainInfo, err := r.api.StateSectorPreCommitInfo(ctx, m.To, sn, tipset.Parents())
 			if err != nil {
-				log.Warnw("failed to get precommit info for sector", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn)
 				continue
 			}
-
+			if !processed {
-			precommitTipset, err := r.api.ChainGetTipSetByHeight(ctx, precommitChainInfo.PreCommitEpoch, tipset.Key())
+				continue
+			}
+		} else if builtin.IsStorageMinerActor(a.Code) {
+			var err error
+			var processed bool
+			processed, messageMethod, refundValue, err = r.processTipsetStorageMinerActor(ctx, tipset, msg, recps[i])
 			if err != nil {
-				log.Warnf("failed to lookup precommit epoch", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn)
 				continue
 			}
-
+			if !processed {
-			collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitChainInfo.Info, precommitTipset.Key())
-			if err != nil {
-				log.Warnw("failed to get initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn)
-			}
-
-			collateral = big.Sub(collateral, precommitChainInfo.PreCommitDeposit)
-			if collateral.LessThan(big.Zero()) {
-				log.Debugw("skipping zero pledge collateral difference", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", sn)
 				continue
 			}
-
-			refundValue = collateral
-		case builtin0.MethodsMiner.PreCommitSector:
-			if !r.preCommitEnabled {
-				continue
-			}
-
-			messageMethod = "PreCommitSector"
-
-			if recps[i].ExitCode != exitcode.Ok {
-				log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recps[i].ExitCode)
-				continue
-			}
-
-			var precommitInfo miner.SectorPreCommitInfo
-			if err := precommitInfo.UnmarshalCBOR(bytes.NewBuffer(m.Params)); err != nil {
-				log.Warnw("failed to decode precommit params", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To)
-				continue
-			}
-
-			collateral, err := r.api.StateMinerInitialPledgeCollateral(ctx, m.To, precommitInfo, tipset.Key())
-			if err != nil {
-				log.Warnw("failed to calculate initial pledge collateral", "err", err, "method", messageMethod, "cid", msg.Cid, "miner", m.To, "sector_number", precommitInfo.SectorNumber)
-				continue
-			}
-
-			refundValue = collateral
-		default:
-			continue
-		}
-
-		if r.refundPercent > 0 {
-			refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent)))
 		}
 
 		log.Debugw(

cmd/lotus-shed/nonce-fix.go

@@ -5,6 +5,8 @@ import (
 	"math"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/urfave/cli/v2"
 
 	"github.com/filecoin-project/lotus/chain/types"
@@ -32,6 +34,10 @@ var noncefix = &cli.Command{
 		&cli.BoolFlag{
 			Name: "auto",
 		},
+		&cli.Int64Flag{
+			Name:  "gas-fee-cap",
+			Usage: "specify gas fee cap for nonce filling messages",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := lcli.GetFullNodeAPI(cctx)
@@ -84,15 +90,32 @@ var noncefix = &cli.Command{
 		}
 		fmt.Printf("Creating %d filler messages (%d ~ %d)\n", end-start, start, end)
 
+		ts, err := api.ChainHead(ctx)
+		if err != nil {
+			return err
+		}
+
+		feeCap := big.Mul(ts.Blocks()[0].ParentBaseFee, big.NewInt(2)) // default fee cap to 2 * parent base fee
+		if fcf := cctx.Int64("gas-fee-cap"); fcf != 0 {
+			feeCap = abi.NewTokenAmount(fcf)
+		}
+
 		for i := start; i < end; i++ {
 			msg := &types.Message{
-				From:  addr,
+				From:       addr,
-				To:    addr,
+				To:         addr,
-				Value: types.NewInt(1),
+				Value:      types.NewInt(0),
-				Nonce: i,
+				Nonce:      i,
+				GasLimit:   1000000,
+				GasFeeCap:  feeCap,
+				GasPremium: abi.NewTokenAmount(5),
+			}
+			smsg, err := api.WalletSignMessage(ctx, addr, msg)
+			if err != nil {
+				return err
 			}
 
-			_, err = api.MpoolPushMessage(ctx, msg, nil)
+			_, err = api.MpoolPush(ctx, smsg)
 			if err != nil {
 				return err
 			}

cmd/lotus-storage-miner/storage.go

@@ -31,6 +31,10 @@ const metaFile = "sectorstore.json"
 var storageCmd = &cli.Command{
 	Name:  "storage",
 	Usage: "manage sector storage",
+	Description: `Sectors can be stored across many filesystem paths. These
+commands provide ways to manage the storage the miner will used to store sectors
+long term for proving (references as 'store') as well as how sectors will be
+stored while moving through the sealing pipeline (references as 'seal').`,
 	Subcommands: []*cli.Command{
 		storageAttachCmd,
 		storageListCmd,
@@ -41,6 +45,25 @@ var storageCmd = &cli.Command{
 var storageAttachCmd = &cli.Command{
 	Name:  "attach",
 	Usage: "attach local storage path",
+	Description: `Storage can be attached to the miner using this command. The storage volume
+list is stored local to the miner in $LOTUS_MINER_PATH/storage.json. We do not
+recommend manually modifying this value without further understanding of the
+storage system.
+
+Each storage volume contains a configuration file which describes the
+capabilities of the volume. When the '--init' flag is provided, this file will
+be created using the additional flags.
+
+Weight
+A high weight value means data will be more likely to be stored in this path
+
+Seal
+Data for the sealing process will be stored here
+
+Store
+Finalized sectors that will be moved here for long term storage and be proven
+over time
+   `,
 	Flags: []cli.Flag{
 		&cli.BoolFlag{
 			Name:  "init",

cmd/tvx/extract.go

@@ -198,8 +198,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 			Preroot:    root,
 			Epoch:      execTs.Height(),
 			Message:    m,
-			CircSupply: &circSupplyDetail.FilCirculating,
+			CircSupply: circSupplyDetail.FilCirculating,
-			BaseFee:    &basefee,
+			BaseFee:    basefee,
 		})
 		if err != nil {
 			return fmt.Errorf("failed to execute precursor message: %w", err)
@@ -229,8 +229,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 			Preroot:    preroot,
 			Epoch:      execTs.Height(),
 			Message:    msg,
-			CircSupply: &circSupplyDetail.FilCirculating,
+			CircSupply: circSupplyDetail.FilCirculating,
-			BaseFee:    &basefee,
+			BaseFee:    basefee,
 		})
 		if err != nil {
 			return fmt.Errorf("failed to execute message: %w", err)
@@ -260,8 +260,8 @@ func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
 			Preroot:    preroot,
 			Epoch:      execTs.Height(),
 			Message:    msg,
-			CircSupply: &circSupplyDetail.FilCirculating,
+			CircSupply: circSupplyDetail.FilCirculating,
-			BaseFee:    &basefee,
+			BaseFee:    basefee,
 		})
 		if err != nil {
 			return fmt.Errorf("failed to execute message: %w", err)

conformance/driver.go

@@ -2,6 +2,7 @@ package conformance
 
 import (
 	"context"
+	gobig "math/big"
 	"os"
 
 	"github.com/filecoin-project/lotus/chain/state"
@@ -14,6 +15,7 @@ import (
 	"github.com/filecoin-project/lotus/lib/blockstore"
 
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
 
 	"github.com/filecoin-project/test-vectors/schema"
@@ -80,7 +82,7 @@ type ExecuteTipsetResult struct {
 func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset) (*ExecuteTipsetResult, error) {
 	var (
 		syscalls = mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier))
-		vmRand   = new(testRand)
+		vmRand   = NewFixedRand()
 
 		cs = store.NewChainStore(bs, ds, syscalls)
 		sm = stmgr.NewStateManager(cs)
@@ -143,8 +145,12 @@ type ExecuteMessageParams struct {
 	Preroot    cid.Cid
 	Epoch      abi.ChainEpoch
 	Message    *types.Message
-	CircSupply *abi.TokenAmount
+	CircSupply abi.TokenAmount
-	BaseFee    *abi.TokenAmount
+	BaseFee    abi.TokenAmount
+
+	// Rand is an optional vm.Rand implementation to use. If nil, the driver
+	// will use a vm.Rand that returns a fixed value for all calls.
+	Rand vm.Rand
 }
 
 // ExecuteMessage executes a conformance test vector message in a temporary VM.
@@ -155,14 +161,8 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP
 		_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
 	}
 
-	basefee := DefaultBaseFee
+	if params.Rand == nil {
-	if params.BaseFee != nil {
+		params.Rand = NewFixedRand()
-		basefee = *params.BaseFee
-	}
-
-	circSupply := DefaultCirculatingSupply
-	if params.CircSupply != nil {
-		circSupply = *params.CircSupply
 	}
 
 	// dummy state manager; only to reference the GetNetworkVersion method,
@@ -172,13 +172,13 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageP
 	vmOpts := &vm.VMOpts{
 		StateBase: params.Preroot,
 		Epoch:     params.Epoch,
-		Rand:      &testRand{}, // TODO always succeeds; need more flexibility.
 		Bstore:    bs,
 		Syscalls:  mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility.
 		CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) {
-			return circSupply, nil
+			return params.CircSupply, nil
 		},
-		BaseFee:     basefee,
+		Rand:        params.Rand,
+		BaseFee:     params.BaseFee,
 		NtwkVersion: sm.GetNtwkVersion,
 	}
 
@@ -231,3 +231,22 @@ func toChainMsg(msg *types.Message) (ret types.ChainMsg) {
 	}
 	return ret
 }
+
+// BaseFeeOrDefault converts a basefee as passed in a test vector (go *big.Int
+// type) to an abi.TokenAmount, or if nil it returns the DefaultBaseFee.
+func BaseFeeOrDefault(basefee *gobig.Int) abi.TokenAmount {
+	if basefee == nil {
+		return DefaultBaseFee
+	}
+	return big.NewFromGo(basefee)
+}
+
+// CircSupplyOrDefault converts a circulating supply as passed in a test vector
+// (go *big.Int type) to an abi.TokenAmount, or if nil it returns the
+// DefaultCirculatingSupply.
+func CircSupplyOrDefault(circSupply *gobig.Int) abi.TokenAmount {
+	if circSupply == nil {
+		return DefaultBaseFee
+	}
+	return big.NewFromGo(circSupply)
+}

conformance/rand_fixed.go

@@ -0,0 +1,28 @@
+package conformance
+
+import (
+	"context"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
+
+	"github.com/filecoin-project/lotus/chain/vm"
+)
+
+type fixedRand struct{}
+
+var _ vm.Rand = (*fixedRand)(nil)
+
+// NewFixedRand creates a test vm.Rand that always returns fixed bytes value
+// of utf-8 string 'i_am_random_____i_am_random_____'.
+func NewFixedRand() vm.Rand {
+	return &fixedRand{}
+}
+
+func (r *fixedRand) GetChainRandomness(_ context.Context, _ crypto.DomainSeparationTag, _ abi.ChainEpoch, _ []byte) ([]byte, error) {
+	return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes.
+}
+
+func (r *fixedRand) GetBeaconRandomness(_ context.Context, _ crypto.DomainSeparationTag, _ abi.ChainEpoch, _ []byte) ([]byte, error) {
+	return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes.
+}

conformance/runner.go

@@ -13,9 +13,7 @@ import (
 
 	"github.com/fatih/color"
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/exitcode"
-	"github.com/filecoin-project/test-vectors/schema"
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
 	ds "github.com/ipfs/go-datastore"
@@ -24,6 +22,8 @@ import (
 	"github.com/ipfs/go-merkledag"
 	"github.com/ipld/go-car"
 
+	"github.com/filecoin-project/test-vectors/schema"
+
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/lib/blockstore"
@@ -46,18 +46,6 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) {
 	// Create a new Driver.
 	driver := NewDriver(ctx, vector.Selector, DriverOpts{DisableVMFlush: true})
 
-	var circSupply *abi.TokenAmount
-	if cs := vector.Pre.CircSupply; cs != nil {
-		ta := big.NewFromGo(cs)
-		circSupply = &ta
-	}
-
-	var basefee *abi.TokenAmount
-	if bf := vector.Pre.BaseFee; bf != nil {
-		ta := big.NewFromGo(bf)
-		basefee = &ta
-	}
-
 	// Apply every message.
 	for i, m := range vector.ApplyMessages {
 		msg, err := types.DecodeMessage(m.Bytes)
@@ -76,8 +64,8 @@ func ExecuteMessageVector(r Reporter, vector *schema.TestVector) {
 			Preroot:    root,
 			Epoch:      abi.ChainEpoch(epoch),
 			Message:    msg,
-			CircSupply: circSupply,
+			BaseFee:    BaseFeeOrDefault(vector.Pre.BaseFee),
-			BaseFee:    basefee,
+			CircSupply: CircSupplyOrDefault(vector.Pre.CircSupply),
 		})
 		if err != nil {
 			r.Fatalf("fatal failure when executing message: %s", err)

conformance/stubs.go

@@ -6,28 +6,16 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/runtime/proof"
 
 	"github.com/filecoin-project/go-address"
+
 	"github.com/filecoin-project/lotus/chain/state"
 	"github.com/filecoin-project/lotus/chain/vm"
 
-	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/specs-actors/actors/runtime"
 
 	cbor "github.com/ipfs/go-ipld-cbor"
 )
 
-type testRand struct{}
-
-var _ vm.Rand = (*testRand)(nil)
-
-func (r *testRand) GetChainRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
-	return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes.
-}
-
-func (r *testRand) GetBeaconRandomness(ctx context.Context, pers crypto.DomainSeparationTag, round abi.ChainEpoch, entropy []byte) ([]byte, error) {
-	return []byte("i_am_random_____i_am_random_____"), nil // 32 bytes.
-}
-
 type testSyscalls struct {
 	runtime.Syscalls
 }

extern/sector-storage/stores/local.go

@@ -30,10 +30,15 @@ type StoragePath struct {
 
 // LocalStorageMeta [path]/sectorstore.json
 type LocalStorageMeta struct {
-	ID     ID
+	ID ID
+
+	// A high weight means data is more likely to be stored in this path
 	Weight uint64 // 0 = readonly
 
-	CanSeal  bool
+	// Intermediate data for the sealing process will be stored here
+	CanSeal bool
+
+	// Finalized sectors that will be proved over time will be stored here
 	CanStore bool
 }