Merge remote-tracking branch 'origin/master' into feat/compact-sectors-numbers-cmd

cmd/chain-noise/main.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
@@ -27,6 +27,16 @@ func main() {
 				Hidden:  true,
 				Value:   "~/.lotus", // TODO: Consider XDG_DATA_HOME
 			},
+			&cli.IntFlag{
+				Name:  "limit",
+				Usage: "spam transaction count limit, <= 0 is no limit",
+				Value: 0,
+			},
+			&cli.IntFlag{
+				Name:  "rate",
+				Usage: "spam transaction rate, count per second",
+				Value: 5,
+			},
 		},
 		Commands: []*cli.Command{runCmd},
 	}
@@ -52,11 +62,17 @@ var runCmd = &cli.Command{
 		defer closer()
 		ctx := lcli.ReqContext(cctx)
 
-		return sendSmallFundsTxs(ctx, api, addr, 5)
+		rate := cctx.Int("rate")
+		if rate <= 0 {
+			rate = 5
+		}
+		limit := cctx.Int("limit")
+
+		return sendSmallFundsTxs(ctx, api, addr, rate, limit)
 	},
 }
 
-func sendSmallFundsTxs(ctx context.Context, api api.FullNode, from address.Address, rate int) error {
+func sendSmallFundsTxs(ctx context.Context, api v0api.FullNode, from address.Address, rate, limit int) error {
 	var sendSet []address.Address
 	for i := 0; i < 20; i++ {
 		naddr, err := api.WalletNew(ctx, types.KTSecp256k1)
@@ -66,9 +82,14 @@ func sendSmallFundsTxs(ctx context.Context, api api.FullNode, from address.Addre
 
 		sendSet = append(sendSet, naddr)
 	}
+	count := limit
 
 	tick := build.Clock.Ticker(time.Second / time.Duration(rate))
 	for {
+		if count <= 0 && limit > 0 {
+			fmt.Printf("%d messages sent.\n", limit)
+			return nil
+		}
 		select {
 		case <-tick.C:
 			msg := &types.Message{
@@ -81,6 +102,7 @@ func sendSmallFundsTxs(ctx context.Context, api api.FullNode, from address.Addre
 			if err != nil {
 				return err
 			}
+			count--
 			fmt.Println("Message sent: ", smsg.Cid())
 		case <-ctx.Done():
 			return nil

cmd/lotus-bench/caching_verifier.go

@@ -8,6 +8,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
+	proof5 "github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
 	"github.com/ipfs/go-datastore"
 	"github.com/minio/blake2b-simd"
 	cbg "github.com/whyrusleeping/cbor-gen"
@@ -96,4 +97,8 @@ func (cv *cachingVerifier) GenerateWinningPoStSectorChallenge(ctx context.Contex
 	return cv.backend.GenerateWinningPoStSectorChallenge(ctx, proofType, a, rnd, u)
 }
 
+func (cv cachingVerifier) VerifyAggregateSeals(aggregate proof5.AggregateSealVerifyProofAndInfos) (bool, error) {
+	return cv.backend.VerifyAggregateSeals(aggregate)
+}
+
 var _ ffiwrapper.Verifier = (*cachingVerifier)(nil)

cmd/lotus-bench/import.go

@@ -16,21 +16,29 @@ import (
 	"sort"
 	"time"
 
+	ocprom "contrib.go.opencensus.io/exporter/prometheus"
 	"github.com/cockroachdb/pebble"
 	"github.com/cockroachdb/pebble/bloom"
+	"github.com/ipfs/go-cid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/blockstore"
+	badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
-	"github.com/filecoin-project/lotus/lib/blockstore"
+	lcli "github.com/filecoin-project/lotus/cli"
 	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
 	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
-	metricsprometheus "github.com/ipfs/go-metrics-prometheus"
-	"github.com/ipld/go-car"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/filecoin-project/lotus/node/repo"
 
 	"github.com/filecoin-project/go-state-types/abi"
+	metricsprometheus "github.com/ipfs/go-metrics-prometheus"
+	"github.com/ipld/go-car"
+
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 
 	bdg "github.com/dgraph-io/badger/v2"
@@ -51,14 +59,30 @@ type TipSetExec struct {
 
 var importBenchCmd = &cli.Command{
 	Name:  "import",
-	Usage: "benchmark chain import and validation",
+	Usage: "Benchmark chain import and validation",
 	Subcommands: []*cli.Command{
 		importAnalyzeCmd,
 	},
 	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "start-tipset",
+			Usage: "start validation at the given tipset key; in format cid1,cid2,cid3...",
+		},
+		&cli.StringFlag{
+			Name:  "end-tipset",
+			Usage: "halt validation at the given tipset key; in format cid1,cid2,cid3...",
+		},
+		&cli.StringFlag{
+			Name:  "genesis-tipset",
+			Usage: "genesis tipset key; in format cid1,cid2,cid3...",
+		},
 		&cli.Int64Flag{
-			Name:  "height",
-			Usage: "halt validation after given height",
+			Name:  "start-height",
+			Usage: "start validation at given height; beware that chain traversal by height is very slow",
+		},
+		&cli.Int64Flag{
+			Name:  "end-height",
+			Usage: "halt validation after given height; beware that chain traversal by height is very slow",
 		},
 		&cli.IntFlag{
 			Name:  "batch-seal-verify-threads",
@@ -86,32 +110,52 @@ var importBenchCmd = &cli.Command{
 			Name:  "global-profile",
 			Value: true,
 		},
-		&cli.Int64Flag{
-			Name: "start-at",
-		},
 		&cli.BoolFlag{
 			Name: "only-import",
 		},
 		&cli.BoolFlag{
 			Name: "use-pebble",
 		},
+		&cli.BoolFlag{
+			Name: "use-native-badger",
+		},
+		&cli.StringFlag{
+			Name: "car",
+			Usage: "path to CAR file; required for import; on validation, either " +
+				"a CAR path or the --head flag are required",
+		},
+		&cli.StringFlag{
+			Name: "head",
+			Usage: "tipset key of the head, useful when benchmarking validation " +
+				"on an existing chain store, where a CAR is not available; " +
+				"if both --car and --head are provided, --head takes precedence " +
+				"over the CAR root; the format is cid1,cid2,cid3...",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		metricsprometheus.Inject() //nolint:errcheck
 		vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads")
-		if !cctx.Args().Present() {
-			fmt.Println("must pass car file of chain to benchmark importing")
-			return nil
-		}
-
-		cfi, err := os.Open(cctx.Args().First())
-		if err != nil {
-			return err
-		}
-		defer cfi.Close() //nolint:errcheck // read only file
 
 		go func() {
-			http.Handle("/debug/metrics/prometheus", promhttp.Handler())
+			// Prometheus globals are exposed as interfaces, but the prometheus
+			// OpenCensus exporter expects a concrete *Registry. The concrete type of
+			// the globals are actually *Registry, so we downcast them, staying
+			// defensive in case things change under the hood.
+			registry, ok := prometheus.DefaultRegisterer.(*prometheus.Registry)
+			if !ok {
+				log.Warnf("failed to export default prometheus registry; some metrics will be unavailable; unexpected type: %T", prometheus.DefaultRegisterer)
+				return
+			}
+			exporter, err := ocprom.NewExporter(ocprom.Options{
+				Registry:  registry,
+				Namespace: "lotus",
+			})
+			if err != nil {
+				log.Fatalf("could not create the prometheus stats exporter: %v", err)
+			}
+
+			http.Handle("/debug/metrics", exporter)
+
 			http.ListenAndServe("localhost:6060", nil) //nolint:errcheck
 		}()
 
@@ -126,17 +170,17 @@ var importBenchCmd = &cli.Command{
 			tdir = tmp
 		}
 
-		bdgOpt := badger.DefaultOptions
-		bdgOpt.GcInterval = 0
-		bdgOpt.Options = bdg.DefaultOptions("")
-		bdgOpt.Options.SyncWrites = false
-		bdgOpt.Options.Truncate = true
-		bdgOpt.Options.DetectConflicts = false
+		var (
+			ds  datastore.Batching
+			bs  blockstore.Blockstore
+			err error
+		)
 
-		var bds datastore.Batching
-		if cctx.Bool("use-pebble") {
+		switch {
+		case cctx.Bool("use-pebble"):
+			log.Info("using pebble")
 			cache := 512
-			bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{
+			ds, err = pebbleds.NewDatastore(tdir, &pebble.Options{
 				// Pebble has a single combined cache area and the write
 				// buffers are taken from this too. Assign all available
 				// memory allowance for cache.
@@ -155,30 +199,45 @@ var importBenchCmd = &cli.Command{
 				},
 				Logger: log,
 			})
-		} else {
-			bds, err = badger.NewDatastore(tdir, &bdgOpt)
+
+		case cctx.Bool("use-native-badger"):
+			log.Info("using native badger")
+			var opts badgerbs.Options
+			if opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, tdir, false); err != nil {
+				return err
+			}
+			opts.SyncWrites = false
+			bs, err = badgerbs.Open(opts)
+
+		default: // legacy badger via datastore.
+			log.Info("using legacy badger")
+			bdgOpt := badger.DefaultOptions
+			bdgOpt.GcInterval = 0
+			bdgOpt.Options = bdg.DefaultOptions("")
+			bdgOpt.Options.SyncWrites = false
+			bdgOpt.Options.Truncate = true
+			bdgOpt.Options.DetectConflicts = false
+
+			ds, err = badger.NewDatastore(tdir, &bdgOpt)
 		}
+
 		if err != nil {
 			return err
 		}
-		defer bds.Close() //nolint:errcheck
 
-		bds = measure.New("dsbench", bds)
+		if ds != nil {
+			ds = measure.New("dsbench", ds)
+			defer ds.Close() //nolint:errcheck
+			bs = blockstore.FromDatastore(ds)
+		}
 
-		bs := blockstore.NewBlockstore(bds)
-		cacheOpts := blockstore.DefaultCacheOpts()
-		cacheOpts.HasBloomFilterSize = 0
-
-		cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, cacheOpts)
-		if err != nil {
-			return err
+		if c, ok := bs.(io.Closer); ok {
+			defer c.Close() //nolint:errcheck
 		}
-		bs = cbs
-		ds := datastore.NewMapDatastore()
 
 		var verifier ffiwrapper.Verifier = ffiwrapper.ProofVerifier
 		if cctx.IsSet("syscall-cache") {
-			scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &bdgOpt)
+			scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &badger.DefaultOptions)
 			if err != nil {
 				return xerrors.Errorf("opening syscall-cache datastore: %w", err)
 			}
@@ -193,11 +252,221 @@ var importBenchCmd = &cli.Command{
 			return nil
 		}
 
-		cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier), nil)
+		metadataDs := datastore.NewMapDatastore()
+		cs := store.NewChainStore(bs, bs, metadataDs, vm.Syscalls(verifier), nil)
+		defer cs.Close() //nolint:errcheck
+
 		stm := stmgr.NewStateManager(cs)
 
+		var carFile *os.File
+		// open the CAR file if one is provided.
+		if path := cctx.String("car"); path != "" {
+			var err error
+			if carFile, err = os.Open(path); err != nil {
+				return xerrors.Errorf("failed to open provided CAR file: %w", err)
+			}
+		}
+
+		startTime := time.Now()
+
+		// register a gauge that reports how long since the measurable
+		// operation began.
+		promauto.NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "lotus_bench_time_taken_secs",
+		}, func() float64 {
+			return time.Since(startTime).Seconds()
+		})
+
+		defer func() {
+			end := time.Now().Format(time.RFC3339)
+
+			resp, err := http.Get("http://localhost:6060/debug/metrics")
+			if err != nil {
+				log.Warnf("failed to scape prometheus: %s", err)
+			}
+
+			metricsfi, err := os.Create("bench.metrics")
+			if err != nil {
+				log.Warnf("failed to write prometheus data: %s", err)
+			}
+
+			_, _ = io.Copy(metricsfi, resp.Body) //nolint:errcheck
+			_ = metricsfi.Close()                //nolint:errcheck
+
+			writeProfile := func(name string) {
+				if file, err := os.Create(fmt.Sprintf("%s.%s.%s.pprof", name, startTime.Format(time.RFC3339), end)); err == nil {
+					if err := pprof.Lookup(name).WriteTo(file, 0); err != nil {
+						log.Warnf("failed to write %s pprof: %s", name, err)
+					}
+					_ = file.Close()
+				} else {
+					log.Warnf("failed to create %s pprof file: %s", name, err)
+				}
+			}
+
+			writeProfile("heap")
+			writeProfile("allocs")
+		}()
+
+		var head *types.TipSet
+		// --- IMPORT ---
+		if !cctx.Bool("no-import") {
+			if cctx.Bool("global-profile") {
+				prof, err := os.Create("bench.import.pprof")
+				if err != nil {
+					return err
+				}
+				defer prof.Close() //nolint:errcheck
+
+				if err := pprof.StartCPUProfile(prof); err != nil {
+					return err
+				}
+			}
+
+			// import is NOT suppressed; do it.
+			if carFile == nil { // a CAR is compulsory for the import.
+				return fmt.Errorf("no CAR file provided for import")
+			}
+
+			head, err = cs.Import(carFile)
+			if err != nil {
+				return err
+			}
+
+			pprof.StopCPUProfile()
+		}
+
+		if cctx.Bool("only-import") {
+			return nil
+		}
+
+		// --- VALIDATION ---
+		//
+		// we are now preparing for the validation benchmark.
+		// a HEAD needs to be set; --head takes precedence over the root
+		// of the CAR, if both are provided.
+		if h := cctx.String("head"); h != "" {
+			cids, err := lcli.ParseTipSetString(h)
+			if err != nil {
+				return xerrors.Errorf("failed to parse head tipset key: %w", err)
+			}
+
+			head, err = cs.LoadTipSet(types.NewTipSetKey(cids...))
+			if err != nil {
+				return err
+			}
+		} else if carFile != nil && head == nil {
+			cr, err := car.NewCarReader(carFile)
+			if err != nil {
+				return err
+			}
+			head, err = cs.LoadTipSet(types.NewTipSetKey(cr.Header.Roots...))
+			if err != nil {
+				return err
+			}
+		} else if h == "" && carFile == nil {
+			return xerrors.Errorf("neither --car nor --head flags supplied")
+		}
+
+		log.Infof("chain head is tipset: %s", head.Key())
+
+		var genesis *types.TipSet
+		log.Infof("getting genesis block")
+		if tsk := cctx.String("genesis-tipset"); tsk != "" {
+			var cids []cid.Cid
+			if cids, err = lcli.ParseTipSetString(tsk); err != nil {
+				return xerrors.Errorf("failed to parse genesis tipset key: %w", err)
+			}
+			genesis, err = cs.LoadTipSet(types.NewTipSetKey(cids...))
+		} else {
+			log.Warnf("getting genesis by height; this will be slow; pass in the genesis tipset through --genesis-tipset")
+			// fallback to the slow path of walking the chain.
+			genesis, err = cs.GetTipsetByHeight(context.TODO(), 0, head, true)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if err = cs.SetGenesis(genesis.Blocks()[0]); err != nil {
+			return err
+		}
+
+		// Resolve the end tipset, falling back to head if not provided.
+		end := head
+		if tsk := cctx.String("end-tipset"); tsk != "" {
+			var cids []cid.Cid
+			if cids, err = lcli.ParseTipSetString(tsk); err != nil {
+				return xerrors.Errorf("failed to end genesis tipset key: %w", err)
+			}
+			end, err = cs.LoadTipSet(types.NewTipSetKey(cids...))
+		} else if h := cctx.Int64("end-height"); h != 0 {
+			log.Infof("getting end tipset at height %d...", h)
+			end, err = cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(h), head, true)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		// Resolve the start tipset, if provided; otherwise, fallback to
+		// height 1 for a start point.
+		var (
+			startEpoch = abi.ChainEpoch(1)
+			start      *types.TipSet
+		)
+
+		if tsk := cctx.String("start-tipset"); tsk != "" {
+			var cids []cid.Cid
+			if cids, err = lcli.ParseTipSetString(tsk); err != nil {
+				return xerrors.Errorf("failed to start genesis tipset key: %w", err)
+			}
+			start, err = cs.LoadTipSet(types.NewTipSetKey(cids...))
+		} else if h := cctx.Int64("start-height"); h != 0 {
+			log.Infof("getting start tipset at height %d...", h)
+			// lookback from the end tipset (which falls back to head if not supplied).
+			start, err = cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(h), end, true)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if start != nil {
+			startEpoch = start.Height()
+			if err := cs.ForceHeadSilent(context.Background(), start); err != nil {
+				// if err := cs.SetHead(start); err != nil {
+				return err
+			}
+		}
+
+		inverseChain := append(make([]*types.TipSet, 0, end.Height()), end)
+		for ts := end; ts.Height() > startEpoch; {
+			if h := ts.Height(); h%100 == 0 {
+				log.Infof("walking back the chain; loaded tipset at height %d...", h)
+			}
+			next, err := cs.LoadTipSet(ts.Parents())
+			if err != nil {
+				return err
+			}
+
+			inverseChain = append(inverseChain, next)
+			ts = next
+		}
+
+		var enc *json.Encoder
+		if cctx.Bool("export-traces") {
+			ibj, err := os.Create("bench.json")
+			if err != nil {
+				return err
+			}
+			defer ibj.Close() //nolint:errcheck
+
+			enc = json.NewEncoder(ibj)
+		}
+
 		if cctx.Bool("global-profile") {
-			prof, err := os.Create("import-bench.prof")
+			prof, err := os.Create("bench.validation.pprof")
 			if err != nil {
 				return err
 			}
@@ -208,84 +477,8 @@ var importBenchCmd = &cli.Command{
 			}
 		}
 
-		var head *types.TipSet
-		if !cctx.Bool("no-import") {
-			head, err = cs.Import(cfi)
-			if err != nil {
-				return err
-			}
-		} else {
-			cr, err := car.NewCarReader(cfi)
-			if err != nil {
-				return err
-			}
-			head, err = cs.LoadTipSet(types.NewTipSetKey(cr.Header.Roots...))
-			if err != nil {
-				return err
-			}
-		}
-
-		if cctx.Bool("only-import") {
-			return nil
-		}
-
-		gb, err := cs.GetTipsetByHeight(context.TODO(), 0, head, true)
-		if err != nil {
-			return err
-		}
-
-		err = cs.SetGenesis(gb.Blocks()[0])
-		if err != nil {
-			return err
-		}
-
-		startEpoch := abi.ChainEpoch(1)
-		if cctx.IsSet("start-at") {
-			startEpoch = abi.ChainEpoch(cctx.Int64("start-at"))
-			start, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(cctx.Int64("start-at")), head, true)
-			if err != nil {
-				return err
-			}
-
-			err = cs.SetHead(start)
-			if err != nil {
-				return err
-			}
-		}
-
-		if h := cctx.Int64("height"); h != 0 {
-			tsh, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(h), head, true)
-			if err != nil {
-				return err
-			}
-			head = tsh
-		}
-
-		ts := head
-		tschain := []*types.TipSet{ts}
-		for ts.Height() > startEpoch {
-			next, err := cs.LoadTipSet(ts.Parents())
-			if err != nil {
-				return err
-			}
-
-			tschain = append(tschain, next)
-			ts = next
-		}
-
-		var enc *json.Encoder
-		if cctx.Bool("export-traces") {
-			ibj, err := os.Create("import-bench.json")
-			if err != nil {
-				return err
-			}
-			defer ibj.Close() //nolint:errcheck
-
-			enc = json.NewEncoder(ibj)
-		}
-
-		for i := len(tschain) - 1; i >= 1; i-- {
-			cur := tschain[i]
+		for i := len(inverseChain) - 1; i >= 1; i-- {
+			cur := inverseChain[i]
 			start := time.Now()
 			log.Infof("computing state (height: %d, ts=%s)", cur.Height(), cur.Cids())
 			st, trace, err := stm.ExecutionTrace(context.TODO(), cur)
@@ -304,7 +497,7 @@ var importBenchCmd = &cli.Command{
 					return xerrors.Errorf("failed to write out tipsetexec: %w", err)
 				}
 			}
-			if tschain[i-1].ParentState() != st {
+			if inverseChain[i-1].ParentState() != st {
 				stripCallers(tse.Trace)
 				lastTrace := tse.Trace
 				d, err := json.MarshalIndent(lastTrace, "", "  ")
@@ -320,23 +513,7 @@ var importBenchCmd = &cli.Command{
 
 		pprof.StopCPUProfile()
 
-		if true {
-			resp, err := http.Get("http://localhost:6060/debug/metrics/prometheus")
-			if err != nil {
-				return err
-			}
-
-			metricsfi, err := os.Create("import-bench.metrics")
-			if err != nil {
-				return err
-			}
-
-			io.Copy(metricsfi, resp.Body) //nolint:errcheck
-			metricsfi.Close()             //nolint:errcheck
-		}
-
 		return nil
-
 	},
 }
 

cmd/lotus-bench/main.go

@@ -31,7 +31,7 @@ import (
 
 	lapi "github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/genesis"
 )
@@ -39,8 +39,12 @@ import (
 var log = logging.Logger("lotus-bench")
 
 type BenchResults struct {
-	SectorSize abi.SectorSize
+	EnvVar map[string]string
 
+	SectorSize   abi.SectorSize
+	SectorNumber int
+
+	SealingSum     SealingResult
 	SealingResults []SealingResult
 
 	PostGenerateCandidates time.Duration
@@ -55,6 +59,26 @@ type BenchResults struct {
 	VerifyWindowPostHot  time.Duration
 }
 
+func (bo *BenchResults) SumSealingTime() error {
+	if len(bo.SealingResults) <= 0 {
+		return xerrors.Errorf("BenchResults SealingResults len <= 0")
+	}
+	if len(bo.SealingResults) != bo.SectorNumber {
+		return xerrors.Errorf("BenchResults SealingResults len(%d) != bo.SectorNumber(%d)", len(bo.SealingResults), bo.SectorNumber)
+	}
+
+	for _, sealing := range bo.SealingResults {
+		bo.SealingSum.AddPiece += sealing.AddPiece
+		bo.SealingSum.PreCommit1 += sealing.PreCommit1
+		bo.SealingSum.PreCommit2 += sealing.PreCommit2
+		bo.SealingSum.Commit1 += sealing.Commit1
+		bo.SealingSum.Commit2 += sealing.Commit2
+		bo.SealingSum.Verify += sealing.Verify
+		bo.SealingSum.Unseal += sealing.Unseal
+	}
+	return nil
+}
+
 type SealingResult struct {
 	AddPiece   time.Duration
 	PreCommit1 time.Duration
@@ -94,12 +118,13 @@ func main() {
 }
 
 var sealBenchCmd = &cli.Command{
-	Name: "sealing",
+	Name:  "sealing",
+	Usage: "Benchmark seal and winning post and window post",
 	Flags: []cli.Flag{
 		&cli.StringFlag{
 			Name:  "storage-dir",
 			Value: "~/.lotus-bench",
-			Usage: "Path to the storage directory that will store sectors long term",
+			Usage: "path to the storage directory that will store sectors long term",
 		},
 		&cli.StringFlag{
 			Name:  "sector-size",
@@ -131,22 +156,26 @@ var sealBenchCmd = &cli.Command{
 			Name:  "skip-unseal",
 			Usage: "skip the unseal portion of the benchmark",
 		},
+		&cli.StringFlag{
+			Name:  "ticket-preimage",
+			Usage: "ticket random",
+		},
 		&cli.StringFlag{
 			Name:  "save-commit2-input",
-			Usage: "Save commit2 input to a file",
+			Usage: "save commit2 input to a file",
 		},
 		&cli.IntFlag{
 			Name:  "num-sectors",
+			Usage: "select number of sectors to seal",
 			Value: 1,
 		},
 		&cli.IntFlag{
 			Name:  "parallel",
+			Usage: "num run in parallel",
 			Value: 1,
 		},
 	},
 	Action: func(c *cli.Context) error {
-		policy.AddSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1)
-
 		if c.Bool("no-gpu") {
 			err := os.Setenv("BELLMAN_NO_GPU", "1")
 			if err != nil {
@@ -211,18 +240,10 @@ var sealBenchCmd = &cli.Command{
 		}
 		sectorSize := abi.SectorSize(sectorSizeInt)
 
-		spt, err := ffiwrapper.SealProofTypeFromSectorSize(sectorSize)
-		if err != nil {
-			return err
-		}
-
-		cfg := &ffiwrapper.Config{
-			SealProofType: spt,
-		}
-
 		// Only fetch parameters if actually needed
-		if !c.Bool("skip-commit2") {
-			if err := paramfetch.GetParams(lcli.ReqContext(c), build.ParametersJSON(), uint64(sectorSize)); err != nil {
+		skipc2 := c.Bool("skip-commit2")
+		if !skipc2 {
+			if err := paramfetch.GetParams(lcli.ReqContext(c), build.ParametersJSON(), build.SrsJSON(), uint64(sectorSize)); err != nil {
 				return xerrors.Errorf("getting params: %w", err)
 			}
 		}
@@ -231,11 +252,13 @@ var sealBenchCmd = &cli.Command{
 			Root: sbdir,
 		}
 
-		sb, err := ffiwrapper.New(sbfs, cfg)
+		sb, err := ffiwrapper.New(sbfs)
 		if err != nil {
 			return err
 		}
 
+		sectorNumber := c.Int("num-sectors")
+
 		var sealTimings []SealingResult
 		var sealedSectors []saproof2.SectorInfo
 
@@ -246,18 +269,11 @@ var sealBenchCmd = &cli.Command{
 				PreCommit2: 1,
 				Commit:     1,
 			}
-			sealTimings, sealedSectors, err = runSeals(sb, sbfs, c.Int("num-sectors"), parCfg, mid, sectorSize, []byte(c.String("ticket-preimage")), c.String("save-commit2-input"), c.Bool("skip-commit2"), c.Bool("skip-unseal"))
+			sealTimings, sealedSectors, err = runSeals(sb, sbfs, sectorNumber, parCfg, mid, sectorSize, []byte(c.String("ticket-preimage")), c.String("save-commit2-input"), skipc2, c.Bool("skip-unseal"))
 			if err != nil {
 				return xerrors.Errorf("failed to run seals: %w", err)
 			}
-		}
-
-		beforePost := time.Now()
-
-		var challenge [32]byte
-		rand.Read(challenge[:])
-
-		if robench != "" {
+		} else {
 			// TODO: implement sbfs.List() and use that for all cases (preexisting sectorbuilder or not)
 
 			// TODO: this assumes we only ever benchmark a preseal
@@ -290,12 +306,21 @@ var sealBenchCmd = &cli.Command{
 
 		bo := BenchResults{
 			SectorSize:     sectorSize,
+			SectorNumber:   sectorNumber,
 			SealingResults: sealTimings,
 		}
+		if err := bo.SumSealingTime(); err != nil {
+			return err
+		}
 
-		if !c.Bool("skip-commit2") {
+		var challenge [32]byte
+		rand.Read(challenge[:])
+
+		beforePost := time.Now()
+
+		if !skipc2 {
 			log.Info("generating winning post candidates")
-			wipt, err := spt.RegisteredWinningPoStProof()
+			wipt, err := spt(sectorSize).RegisteredWinningPoStProof()
 			if err != nil {
 				return err
 			}
@@ -420,6 +445,15 @@ var sealBenchCmd = &cli.Command{
 			bo.VerifyWindowPostHot = verifyWindowpost2.Sub(verifyWindowpost1)
 		}
 
+		bo.EnvVar = make(map[string]string)
+		for _, envKey := range []string{"BELLMAN_NO_GPU", "FIL_PROOFS_MAXIMIZE_CACHING", "FIL_PROOFS_USE_GPU_COLUMN_BUILDER",
+			"FIL_PROOFS_USE_GPU_TREE_BUILDER", "FIL_PROOFS_USE_MULTICORE_SDR", "BELLMAN_CUSTOM_GPU"} {
+			envValue, found := os.LookupEnv(envKey)
+			if found {
+				bo.EnvVar[envKey] = envValue
+			}
+		}
+
 		if c.Bool("json-out") {
 			data, err := json.MarshalIndent(bo, "", "  ")
 			if err != nil {
@@ -428,21 +462,25 @@ var sealBenchCmd = &cli.Command{
 
 			fmt.Println(string(data))
 		} else {
-			fmt.Printf("----\nresults (v27) (%d)\n", sectorSize)
+			fmt.Println("environment variable list:")
+			for envKey, envValue := range bo.EnvVar {
+				fmt.Printf("%s=%s\n", envKey, envValue)
+			}
+			fmt.Printf("----\nresults (v28) SectorSize:(%d), SectorNumber:(%d)\n", sectorSize, sectorNumber)
 			if robench == "" {
-				fmt.Printf("seal: addPiece: %s (%s)\n", bo.SealingResults[0].AddPiece, bps(bo.SectorSize, bo.SealingResults[0].AddPiece)) // TODO: average across multiple sealings
-				fmt.Printf("seal: preCommit phase 1: %s (%s)\n", bo.SealingResults[0].PreCommit1, bps(bo.SectorSize, bo.SealingResults[0].PreCommit1))
-				fmt.Printf("seal: preCommit phase 2: %s (%s)\n", bo.SealingResults[0].PreCommit2, bps(bo.SectorSize, bo.SealingResults[0].PreCommit2))
-				fmt.Printf("seal: commit phase 1: %s (%s)\n", bo.SealingResults[0].Commit1, bps(bo.SectorSize, bo.SealingResults[0].Commit1))
-				fmt.Printf("seal: commit phase 2: %s (%s)\n", bo.SealingResults[0].Commit2, bps(bo.SectorSize, bo.SealingResults[0].Commit2))
-				fmt.Printf("seal: verify: %s\n", bo.SealingResults[0].Verify)
+				fmt.Printf("seal: addPiece: %s (%s)\n", bo.SealingSum.AddPiece, bps(bo.SectorSize, bo.SectorNumber, bo.SealingSum.AddPiece))
+				fmt.Printf("seal: preCommit phase 1: %s (%s)\n", bo.SealingSum.PreCommit1, bps(bo.SectorSize, bo.SectorNumber, bo.SealingSum.PreCommit1))
+				fmt.Printf("seal: preCommit phase 2: %s (%s)\n", bo.SealingSum.PreCommit2, bps(bo.SectorSize, bo.SectorNumber, bo.SealingSum.PreCommit2))
+				fmt.Printf("seal: commit phase 1: %s (%s)\n", bo.SealingSum.Commit1, bps(bo.SectorSize, bo.SectorNumber, bo.SealingSum.Commit1))
+				fmt.Printf("seal: commit phase 2: %s (%s)\n", bo.SealingSum.Commit2, bps(bo.SectorSize, bo.SectorNumber, bo.SealingSum.Commit2))
+				fmt.Printf("seal: verify: %s\n", bo.SealingSum.Verify)
 				if !c.Bool("skip-unseal") {
-					fmt.Printf("unseal: %s  (%s)\n", bo.SealingResults[0].Unseal, bps(bo.SectorSize, bo.SealingResults[0].Unseal))
+					fmt.Printf("unseal: %s  (%s)\n", bo.SealingSum.Unseal, bps(bo.SectorSize, bo.SectorNumber, bo.SealingSum.Unseal))
 				}
 				fmt.Println("")
 			}
-			if !c.Bool("skip-commit2") {
-				fmt.Printf("generate candidates: %s (%s)\n", bo.PostGenerateCandidates, bps(bo.SectorSize*abi.SectorSize(len(bo.SealingResults)), bo.PostGenerateCandidates))
+			if !skipc2 {
+				fmt.Printf("generate candidates: %s (%s)\n", bo.PostGenerateCandidates, bps(bo.SectorSize, len(bo.SealingResults), bo.PostGenerateCandidates))
 				fmt.Printf("compute winning post proof (cold): %s\n", bo.PostWinningProofCold)
 				fmt.Printf("compute winning post proof (hot): %s\n", bo.PostWinningProofHot)
 				fmt.Printf("verify winning post proof (cold): %s\n", bo.VerifyWinningPostCold)
@@ -475,11 +513,13 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 	if numSectors%par.PreCommit1 != 0 {
 		return nil, nil, fmt.Errorf("parallelism factor must cleanly divide numSectors")
 	}
-
-	for i := abi.SectorNumber(1); i <= abi.SectorNumber(numSectors); i++ {
-		sid := abi.SectorID{
-			Miner:  mid,
-			Number: i,
+	for i := abi.SectorNumber(0); i < abi.SectorNumber(numSectors); i++ {
+		sid := storage.SectorRef{
+			ID: abi.SectorID{
+				Miner:  mid,
+				Number: i,
+			},
+			ProofType: spt(sectorSize),
 		}
 
 		start := time.Now()
@@ -494,7 +534,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 
 		pieces = append(pieces, pi)
 
-		sealTimings[i-1].AddPiece = time.Since(start)
+		sealTimings[i].AddPiece = time.Since(start)
 	}
 
 	sectorsPerWorker := numSectors / par.PreCommit1
@@ -503,13 +543,15 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 	for wid := 0; wid < par.PreCommit1; wid++ {
 		go func(worker int) {
 			sealerr := func() error {
-				start := 1 + (worker * sectorsPerWorker)
+				start := worker * sectorsPerWorker
 				end := start + sectorsPerWorker
 				for i := abi.SectorNumber(start); i < abi.SectorNumber(end); i++ {
-					ix := int(i - 1)
-					sid := abi.SectorID{
-						Miner:  mid,
-						Number: i,
+					sid := storage.SectorRef{
+						ID: abi.SectorID{
+							Miner:  mid,
+							Number: i,
+						},
+						ProofType: spt(sectorSize),
 					}
 
 					start := time.Now()
@@ -518,8 +560,8 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 					ticket := abi.SealRandomness(trand[:])
 
 					log.Infof("[%d] Running replication(1)...", i)
-					pieces := []abi.PieceInfo{pieces[ix]}
-					pc1o, err := sb.SealPreCommit1(context.TODO(), sid, ticket, pieces)
+					piece := []abi.PieceInfo{pieces[i]}
+					pc1o, err := sb.SealPreCommit1(context.TODO(), sid, ticket, piece)
 					if err != nil {
 						return xerrors.Errorf("commit: %w", err)
 					}
@@ -537,8 +579,8 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 					precommit2 := time.Now()
 					<-preCommit2Sema
 
-					sealedSectors[ix] = saproof2.SectorInfo{
-						SealProof:    sb.SealProofType(),
+					sealedSectors[i] = saproof2.SectorInfo{
+						SealProof:    sid.ProofType,
 						SectorNumber: i,
 						SealedCID:    cids.Sealed,
 					}
@@ -551,7 +593,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 					commitSema <- struct{}{}
 					commitStart := time.Now()
 					log.Infof("[%d] Generating PoRep for sector (1)", i)
-					c1o, err := sb.SealCommit1(context.TODO(), sid, ticket, seed.Value, pieces, cids)
+					c1o, err := sb.SealCommit1(context.TODO(), sid, ticket, seed.Value, piece, cids)
 					if err != nil {
 						return err
 					}
@@ -592,7 +634,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 						svi := saproof2.SealVerifyInfo{
 							SectorID:              abi.SectorID{Miner: mid, Number: i},
 							SealedCID:             cids.Sealed,
-							SealProof:             sb.SealProofType(),
+							SealProof:             sid.ProofType,
 							Proof:                 proof,
 							DealIDs:               nil,
 							Randomness:            ticket,
@@ -614,7 +656,7 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 					if !skipunseal {
 						log.Infof("[%d] Unsealing sector", i)
 						{
-							p, done, err := sbfs.AcquireSector(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, storiface.FTUnsealed, storiface.FTNone, storiface.PathSealing)
+							p, done, err := sbfs.AcquireSector(context.TODO(), sid, storiface.FTUnsealed, storiface.FTNone, storiface.PathSealing)
 							if err != nil {
 								return xerrors.Errorf("acquire unsealed sector for removing: %w", err)
 							}
@@ -625,19 +667,19 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 							}
 						}
 
-						err := sb.UnsealPiece(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, 0, abi.PaddedPieceSize(sectorSize).Unpadded(), ticket, cids.Unsealed)
+						err := sb.UnsealPiece(context.TODO(), sid, 0, abi.PaddedPieceSize(sectorSize).Unpadded(), ticket, cids.Unsealed)
 						if err != nil {
 							return err
 						}
 					}
 					unseal := time.Now()
 
-					sealTimings[ix].PreCommit1 = precommit1.Sub(start)
-					sealTimings[ix].PreCommit2 = precommit2.Sub(pc2Start)
-					sealTimings[ix].Commit1 = sealcommit1.Sub(commitStart)
-					sealTimings[ix].Commit2 = sealcommit2.Sub(sealcommit1)
-					sealTimings[ix].Verify = verifySeal.Sub(sealcommit2)
-					sealTimings[ix].Unseal = unseal.Sub(verifySeal)
+					sealTimings[i].PreCommit1 = precommit1.Sub(start)
+					sealTimings[i].PreCommit2 = precommit2.Sub(pc2Start)
+					sealTimings[i].Commit1 = sealcommit1.Sub(commitStart)
+					sealTimings[i].Commit2 = sealcommit2.Sub(sealcommit1)
+					sealTimings[i].Verify = verifySeal.Sub(sealcommit2)
+					sealTimings[i].Unseal = unseal.Sub(verifySeal)
 				}
 				return nil
 			}()
@@ -660,8 +702,9 @@ func runSeals(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, numSectors int, par
 }
 
 var proveCmd = &cli.Command{
-	Name:  "prove",
-	Usage: "Benchmark a proof computation",
+	Name:      "prove",
+	Usage:     "Benchmark a proof computation",
+	ArgsUsage: "[input.json]",
 	Flags: []cli.Flag{
 		&cli.BoolFlag{
 			Name:  "no-gpu",
@@ -695,7 +738,7 @@ var proveCmd = &cli.Command{
 			return xerrors.Errorf("unmarshalling input file: %w", err)
 		}
 
-		if err := paramfetch.GetParams(lcli.ReqContext(c), build.ParametersJSON(), c2in.SectorSize); err != nil {
+		if err := paramfetch.GetParams(lcli.ReqContext(c), build.ParametersJSON(), build.SrsJSON(), c2in.SectorSize); err != nil {
 			return xerrors.Errorf("getting params: %w", err)
 		}
 
@@ -708,23 +751,23 @@ var proveCmd = &cli.Command{
 			return err
 		}
 
-		spt, err := ffiwrapper.SealProofTypeFromSectorSize(abi.SectorSize(c2in.SectorSize))
+		sb, err := ffiwrapper.New(nil)
 		if err != nil {
 			return err
 		}
 
-		cfg := &ffiwrapper.Config{
-			SealProofType: spt,
-		}
-
-		sb, err := ffiwrapper.New(nil, cfg)
-		if err != nil {
-			return err
+		ref := storage.SectorRef{
+			ID: abi.SectorID{
+				Miner:  abi.ActorID(mid),
+				Number: abi.SectorNumber(c2in.SectorNum),
+			},
+			ProofType: spt(abi.SectorSize(c2in.SectorSize)),
 		}
 
+		fmt.Printf("----\nstart proof computation\n")
 		start := time.Now()
 
-		proof, err := sb.SealCommit2(context.TODO(), abi.SectorID{Miner: abi.ActorID(mid), Number: abi.SectorNumber(c2in.SectorNum)}, c2in.Phase1Out)
+		proof, err := sb.SealCommit2(context.TODO(), ref, c2in.Phase1Out)
 		if err != nil {
 			return err
 		}
@@ -733,17 +776,27 @@ var proveCmd = &cli.Command{
 
 		fmt.Printf("proof: %x\n", proof)
 
-		fmt.Printf("----\nresults (v27) (%d)\n", c2in.SectorSize)
+		fmt.Printf("----\nresults (v28) (%d)\n", c2in.SectorSize)
 		dur := sealCommit2.Sub(start)
 
-		fmt.Printf("seal: commit phase 2: %s (%s)\n", dur, bps(abi.SectorSize(c2in.SectorSize), dur))
+		fmt.Printf("seal: commit phase 2: %s (%s)\n", dur, bps(abi.SectorSize(c2in.SectorSize), 1, dur))
 		return nil
 	},
 }
 
-func bps(data abi.SectorSize, d time.Duration) string {
-	bdata := new(big.Int).SetUint64(uint64(data))
+func bps(sectorSize abi.SectorSize, sectorNum int, d time.Duration) string {
+	bdata := new(big.Int).SetUint64(uint64(sectorSize))
+	bdata = bdata.Mul(bdata, big.NewInt(int64(sectorNum)))
 	bdata = bdata.Mul(bdata, big.NewInt(time.Second.Nanoseconds()))
 	bps := bdata.Div(bdata, big.NewInt(d.Nanoseconds()))
 	return types.SizeStr(types.BigInt{Int: bps}) + "/s"
 }
+
+func spt(ssize abi.SectorSize) abi.RegisteredSealProof {
+	spt, err := miner.SealProofTypeFromSectorSize(ssize, build.NewestNetworkVersion)
+	if err != nil {
+		panic(err)
+	}
+
+	return spt
+}

cmd/lotus-chainwatch/processor/miner.go

@@ -14,8 +14,8 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/apibstore"
+	"github.com/filecoin-project/lotus/api/v0api"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/events/state"
@@ -202,7 +202,7 @@ func (p *Processor) processMiners(ctx context.Context, minerTips map[types.TipSe
 		log.Debugw("Processed Miners", "duration", time.Since(start).String())
 	}()
 
-	stor := store.ActorStore(ctx, apibstore.NewAPIBlockstore(p.node))
+	stor := store.ActorStore(ctx, blockstore.NewAPIBlockstore(p.node))
 
 	var out []minerActorInfo
 	// TODO add parallel calls if this becomes slow
@@ -649,7 +649,7 @@ func (p *Processor) getMinerStateAt(ctx context.Context, maddr address.Address,
 	if err != nil {
 		return nil, err
 	}
-	return miner.Load(store.ActorStore(ctx, apibstore.NewAPIBlockstore(p.node)), prevActor)
+	return miner.Load(store.ActorStore(ctx, blockstore.NewAPIBlockstore(p.node)), prevActor)
 }
 
 func (p *Processor) getMinerPreCommitChanges(ctx context.Context, m minerActorInfo) (*miner.PreCommitChanges, error) {
@@ -1026,7 +1026,7 @@ func (p *Processor) storeMinersPower(miners []minerActorInfo) error {
 }
 
 // load the power actor state clam as an adt.Map at the tipset `ts`.
-func getPowerActorState(ctx context.Context, api api.FullNode, ts types.TipSetKey) (power.State, error) {
+func getPowerActorState(ctx context.Context, api v0api.FullNode, ts types.TipSetKey) (power.State, error) {
 	powerActor, err := api.StateGetActor(ctx, power.Address, ts)
 	if err != nil {
 		return nil, err

cmd/lotus-chainwatch/processor/processor.go

@@ -17,7 +17,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
 
-	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/chain/types"
 	cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util"
 	"github.com/filecoin-project/lotus/lib/parmap"
@@ -28,7 +28,7 @@ var log = logging.Logger("processor")
 type Processor struct {
 	db *sql.DB
 
-	node     api.FullNode
+	node     v0api.FullNode
 	ctxStore *cw_util.APIIpldStore
 
 	genesisTs *types.TipSet
@@ -52,7 +52,7 @@ type actorInfo struct {
 	state string
 }
 
-func NewProcessor(ctx context.Context, db *sql.DB, node api.FullNode, batch int) *Processor {
+func NewProcessor(ctx context.Context, db *sql.DB, node v0api.FullNode, batch int) *Processor {
 	ctxStore := cw_util.NewAPIIpldStore(ctx, node)
 	return &Processor{
 		db:       db,
@@ -146,7 +146,7 @@ func (p *Processor) Start(ctx context.Context) {
 				go func() {
 					defer grp.Done()
 					if err := p.HandleMarketChanges(ctx, actorChanges[builtin2.StorageMarketActorCodeID]); err != nil {
-						log.Errorf("Failed to handle market changes: %w", err)
+						log.Errorf("Failed to handle market changes: %v", err)
 						return
 					}
 				}()
@@ -155,7 +155,7 @@ func (p *Processor) Start(ctx context.Context) {
 				go func() {
 					defer grp.Done()
 					if err := p.HandleMinerChanges(ctx, actorChanges[builtin2.StorageMinerActorCodeID]); err != nil {
-						log.Errorf("Failed to handle miner changes: %w", err)
+						log.Errorf("Failed to handle miner changes: %v", err)
 						return
 					}
 				}()
@@ -164,7 +164,7 @@ func (p *Processor) Start(ctx context.Context) {
 				go func() {
 					defer grp.Done()
 					if err := p.HandleRewardChanges(ctx, actorChanges[builtin2.RewardActorCodeID], nullRounds); err != nil {
-						log.Errorf("Failed to handle reward changes: %w", err)
+						log.Errorf("Failed to handle reward changes: %v", err)
 						return
 					}
 				}()
@@ -173,7 +173,7 @@ func (p *Processor) Start(ctx context.Context) {
 				go func() {
 					defer grp.Done()
 					if err := p.HandlePowerChanges(ctx, actorChanges[builtin2.StoragePowerActorCodeID]); err != nil {
-						log.Errorf("Failed to handle power actor changes: %w", err)
+						log.Errorf("Failed to handle power actor changes: %v", err)
 						return
 					}
 				}()
@@ -182,7 +182,7 @@ func (p *Processor) Start(ctx context.Context) {
 				go func() {
 					defer grp.Done()
 					if err := p.HandleMessageChanges(ctx, toProcess); err != nil {
-						log.Errorf("Failed to handle message changes: %w", err)
+						log.Errorf("Failed to handle message changes: %v", err)
 						return
 					}
 				}()
@@ -191,7 +191,7 @@ func (p *Processor) Start(ctx context.Context) {
 				go func() {
 					defer grp.Done()
 					if err := p.HandleCommonActorsChanges(ctx, actorChanges); err != nil {
-						log.Errorf("Failed to handle common actor changes: %w", err)
+						log.Errorf("Failed to handle common actor changes: %v", err)
 						return
 					}
 				}()

cmd/lotus-chainwatch/run.go

@@ -8,6 +8,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/filecoin-project/lotus/api/v0api"
+
 	_ "github.com/lib/pq"
 
 	"github.com/filecoin-project/go-jsonrpc"
@@ -15,7 +17,6 @@ import (
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/lotus/api"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/cmd/lotus-chainwatch/processor"
 	"github.com/filecoin-project/lotus/cmd/lotus-chainwatch/scheduler"
@@ -44,7 +45,7 @@ var runCmd = &cli.Command{
 			return err
 		}
 
-		var api api.FullNode
+		var api v0api.FullNode
 		var closer jsonrpc.ClientCloser
 		var err error
 		if tokenMaddr := cctx.String("api"); tokenMaddr != "" {

cmd/lotus-chainwatch/syncer/sync.go

@@ -13,7 +13,7 @@ import (
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
 
-	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 )
@@ -26,10 +26,10 @@ type Syncer struct {
 	lookbackLimit uint64
 
 	headerLk sync.Mutex
-	node     api.FullNode
+	node     v0api.FullNode
 }
 
-func NewSyncer(db *sql.DB, node api.FullNode, lookbackLimit uint64) *Syncer {
+func NewSyncer(db *sql.DB, node v0api.FullNode, lookbackLimit uint64) *Syncer {
 	return &Syncer{
 		db:            db,
 		node:          node,

cmd/lotus-chainwatch/util/api.go

@@ -5,13 +5,13 @@ import (
 	"net/http"
 
 	"github.com/filecoin-project/go-jsonrpc"
-	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/client"
+	"github.com/filecoin-project/lotus/api/v0api"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
 )
 
-func GetFullNodeAPIUsingCredentials(ctx context.Context, listenAddr, token string) (api.FullNode, jsonrpc.ClientCloser, error) {
+func GetFullNodeAPIUsingCredentials(ctx context.Context, listenAddr, token string) (v0api.FullNode, jsonrpc.ClientCloser, error) {
 	parsedAddr, err := ma.NewMultiaddr(listenAddr)
 	if err != nil {
 		return nil, nil, err
@@ -22,7 +22,7 @@ func GetFullNodeAPIUsingCredentials(ctx context.Context, listenAddr, token strin
 		return nil, nil, err
 	}
 
-	return client.NewFullNodeRPC(ctx, apiURI(addr), apiHeaders(token))
+	return client.NewFullNodeRPCV0(ctx, apiURI(addr), apiHeaders(token))
 }
 func apiURI(addr string) string {
 	return "ws://" + addr + "/rpc/v0"

cmd/lotus-chainwatch/util/contextStore.go

@@ -8,7 +8,7 @@ import (
 	"github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 
-	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/v0api"
 )
 
 // TODO extract this to a common location in lotus and reuse the code
@@ -16,10 +16,10 @@ import (
 // APIIpldStore is required for AMT and HAMT access.
 type APIIpldStore struct {
 	ctx context.Context
-	api api.FullNode
+	api v0api.FullNode
 }
 
-func NewAPIIpldStore(ctx context.Context, api api.FullNode) *APIIpldStore {
+func NewAPIIpldStore(ctx context.Context, api v0api.FullNode) *APIIpldStore {
 	return &APIIpldStore{
 		ctx: ctx,
 		api: api,

cmd/lotus-fountain/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"html/template"
 	"net"
 	"net/http"
 	"os"
@@ -14,7 +15,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
@@ -68,6 +69,10 @@ var runCmd = &cli.Command{
 			EnvVars: []string{"LOTUS_FOUNTAIN_AMOUNT"},
 			Value:   "50",
 		},
+		&cli.Float64Flag{
+			Name:  "captcha-threshold",
+			Value: 0.5,
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		sendPerRequest, err := types.ParseFIL(cctx.String("amount"))
@@ -87,7 +92,7 @@ var runCmd = &cli.Command{
 			return err
 		}
 
-		log.Info("Remote version: %s", v.Version)
+		log.Infof("Remote version: %s", v.Version)
 
 		from, err := address.NewFromString(cctx.String("from"))
 		if err != nil {
@@ -107,11 +112,13 @@ var runCmd = &cli.Command{
 				WalletRate:  15 * time.Minute,
 				WalletBurst: 2,
 			}),
+			recapThreshold: cctx.Float64("captcha-threshold"),
 		}
 
-		http.Handle("/", http.FileServer(rice.MustFindBox("site").HTTPBox()))
-		http.HandleFunc("/send", h.send)
-
+		box := rice.MustFindBox("site")
+		http.Handle("/", http.FileServer(box.HTTPBox()))
+		http.HandleFunc("/funds.html", prepFundsHtml(box))
+		http.Handle("/send", h)
 		fmt.Printf("Open http://%s\n", cctx.String("front"))
 
 		go func() {
@@ -123,22 +130,63 @@ var runCmd = &cli.Command{
 	},
 }
 
+func prepFundsHtml(box *rice.Box) http.HandlerFunc {
+	tmpl := template.Must(template.New("funds").Parse(box.MustString("funds.html")))
+	return func(w http.ResponseWriter, r *http.Request) {
+		err := tmpl.Execute(w, os.Getenv("RECAPTCHA_SITE_KEY"))
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadGateway)
+			return
+		}
+	}
+}
+
 type handler struct {
 	ctx context.Context
-	api api.FullNode
+	api v0api.FullNode
 
 	from           address.Address
 	sendPerRequest types.FIL
 
-	limiter *Limiter
+	limiter        *Limiter
+	recapThreshold float64
 }
 
-func (h *handler) send(w http.ResponseWriter, r *http.Request) {
+func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "only POST is allowed", http.StatusBadRequest)
+		return
+	}
+
+	reqIP := r.Header.Get("X-Real-IP")
+	if reqIP == "" {
+		h, _, err := net.SplitHostPort(r.RemoteAddr)
+		if err != nil {
+			log.Errorf("could not get ip from: %s, err: %s", r.RemoteAddr, err)
+		}
+		reqIP = h
+	}
+
+	capResp, err := VerifyToken(r.FormValue("g-recaptcha-response"), reqIP)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadGateway)
+		return
+	}
+	if !capResp.Success || capResp.Score < h.recapThreshold {
+		log.Infow("spam", "capResp", capResp)
+		http.Error(w, "spam protection", http.StatusUnprocessableEntity)
+		return
+	}
+
 	to, err := address.NewFromString(r.FormValue("address"))
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
+	if to == address.Undef {
+		http.Error(w, "empty address", http.StatusBadRequest)
+		return
+	}
 
 	// Limit based on wallet address
 	limiter := h.limiter.GetWalletLimiter(to.String())
@@ -148,15 +196,6 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Limit based on IP
-
-	reqIP := r.Header.Get("X-Real-IP")
-	if reqIP == "" {
-		h, _, err := net.SplitHostPort(r.RemoteAddr)
-		if err != nil {
-			log.Errorf("could not get ip from: %s, err: %s", r.RemoteAddr, err)
-		}
-		reqIP = h
-	}
 	if i := net.ParseIP(reqIP); i != nil && i.IsLoopback() {
 		log.Errorf("rate limiting localhost: %s", reqIP)
 	}

cmd/lotus-fountain/recaptcha.go

@@ -0,0 +1,73 @@
+// From https://github.com/lukasaron/recaptcha
+// BLS-3 Licensed
+// Copyright (c) 2020, Lukas Aron
+// Modified by Kubuxu
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+)
+
+// content type for communication with the verification server.
+const (
+	contentType = "application/json"
+)
+
+// VerifyURL defines the endpoint which is called when a token needs to be verified.
+var (
+	VerifyURL, _ = url.Parse("https://www.google.com/recaptcha/api/siteverify")
+)
+
+// Response defines the response format from the verification endpoint.
+type Response struct {
+	Success            bool      `json:"success"`          // status of the verification
+	TimeStamp          time.Time `json:"challenge_ts"`     // timestamp of the challenge load (ISO format)
+	HostName           string    `json:"hostname"`         // the hostname of the site where the reCAPTCHA was solved
+	Score              float64   `json:"score"`            // the score for this request (0.0 - 1.0)
+	Action             string    `json:"action"`           // the action name for this request
+	ErrorCodes         []string  `json:"error-codes"`      // error codes
+	AndroidPackageName string    `json:"apk_package_name"` // android related only
+}
+
+// VerifyToken function implements the basic logic of verification of ReCaptcha token that is usually created
+// on the user site (front-end) and then sent to verify on the server side (back-end).
+// To provide a successful verification process the secret key is required. Based on the security recommendations
+// the key has to be passed as an environmental variable SECRET_KEY.
+//
+// Token parameter is required, however remoteIP is optional.
+func VerifyToken(token, remoteIP string) (Response, error) {
+	resp := Response{}
+	if len(token) == 0 {
+		resp.ErrorCodes = []string{"no-token"}
+		return resp, nil
+	}
+
+	q := url.Values{}
+	q.Add("secret", os.Getenv("RECAPTCHA_SECRET_KEY"))
+	q.Add("response", token)
+	q.Add("remoteip", remoteIP)
+
+	var u *url.URL
+	{
+		verifyCopy := *VerifyURL
+		u = &verifyCopy
+	}
+	u.RawQuery = q.Encode()
+	r, err := http.Post(u.String(), contentType, nil)
+	if err != nil {
+		return resp, err
+	}
+
+	b, err := ioutil.ReadAll(r.Body)
+	_ = r.Body.Close() // close immediately after reading finished
+	if err != nil {
+		return resp, err
+	}
+
+	return resp, json.Unmarshal(b, &resp)
+}

cmd/lotus-fountain/site/funds.html

@@ -3,6 +3,13 @@
 <head>
     <title>Sending Funds - Lotus Fountain</title>
     <link rel="stylesheet" type="text/css" href="main.css">
+	<script src="https://www.google.com/recaptcha/api.js"></script>
+	<script>
+   		function onSubmit(token) {
+     	document.getElementById("funds-form").submit();
+   	}
+	</script>
+
 </head>
 <body>
 <div class="Index">
@@ -11,10 +18,13 @@
             [SENDING FUNDS]
         </div>
         <div class="Index-node">
-            <form action='/send' method='get'>
+            <form action='/send' method='post' id='funds-form'>
                 <span>Enter destination address:</span>
-                <input type='text' name='address' style="width: 300px">
-                <button type='submit'>Send Funds</button>
+				<input type='text' name='address' style="width: 300px">
+				<button class="g-recaptcha" 
+						data-sitekey="{{ . }}"
+						data-callback='onSubmit' 
+						data-action='submit'>Send Funds</button>
             </form>
         </div>
     </div>

cmd/lotus-gateway/api.go

@@ -1,390 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-bitfield"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/crypto"
-	"github.com/filecoin-project/go-state-types/dline"
-	"github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/sigs"
-	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
-	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
-	"github.com/filecoin-project/lotus/node/impl/full"
-	"github.com/ipfs/go-cid"
-)
-
-const (
-	LookbackCap            = time.Hour * 24
-	StateWaitLookbackLimit = abi.ChainEpoch(20)
-)
-
-var (
-	ErrLookbackTooLong = fmt.Errorf("lookbacks of more than %s are disallowed", LookbackCap)
-)
-
-// gatewayDepsAPI defines the API methods that the GatewayAPI depends on
-// (to make it easy to mock for tests)
-type gatewayDepsAPI interface {
-	Version(context.Context) (api.Version, error)
-	ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
-	ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error)
-	ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error)
-	ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error)
-	ChainHasObj(context.Context, cid.Cid) (bool, error)
-	ChainHead(ctx context.Context) (*types.TipSet, error)
-	ChainNotify(context.Context) (<-chan []*api.HeadChange, error)
-	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
-	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
-	MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
-	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
-	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error)
-	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)
-	StateGetReceipt(context.Context, cid.Cid, types.TipSetKey) (*types.MessageReceipt, error)
-	StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
-	StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error)
-	StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error)
-	StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error)
-	StateNetworkVersion(context.Context, types.TipSetKey) (network.Version, error)
-	StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error)
-	StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error)
-	StateMinerPower(context.Context, address.Address, types.TipSetKey) (*api.MinerPower, error)
-	StateMinerFaults(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)
-	StateMinerRecoveries(context.Context, address.Address, types.TipSetKey) (bitfield.BitField, error)
-	StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error)
-	StateMinerDeadlines(context.Context, address.Address, types.TipSetKey) ([]api.Deadline, error)
-	StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error)
-	StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error)
-	StateCirculatingSupply(context.Context, types.TipSetKey) (abi.TokenAmount, error)
-	StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error)
-	StateVMCirculatingSupplyInternal(context.Context, types.TipSetKey) (api.CirculatingSupply, error)
-}
-
-type GatewayAPI struct {
-	api                    gatewayDepsAPI
-	lookbackCap            time.Duration
-	stateWaitLookbackLimit abi.ChainEpoch
-}
-
-// NewGatewayAPI creates a new GatewayAPI with the default lookback cap
-func NewGatewayAPI(api gatewayDepsAPI) *GatewayAPI {
-	return newGatewayAPI(api, LookbackCap, StateWaitLookbackLimit)
-}
-
-// used by the tests
-func newGatewayAPI(api gatewayDepsAPI, lookbackCap time.Duration, stateWaitLookbackLimit abi.ChainEpoch) *GatewayAPI {
-	return &GatewayAPI{api: api, lookbackCap: lookbackCap, stateWaitLookbackLimit: stateWaitLookbackLimit}
-}
-
-func (a *GatewayAPI) checkTipsetKey(ctx context.Context, tsk types.TipSetKey) error {
-	if tsk.IsEmpty() {
-		return nil
-	}
-
-	ts, err := a.api.ChainGetTipSet(ctx, tsk)
-	if err != nil {
-		return err
-	}
-
-	return a.checkTipset(ts)
-}
-
-func (a *GatewayAPI) checkTipset(ts *types.TipSet) error {
-	at := time.Unix(int64(ts.Blocks()[0].Timestamp), 0)
-	if err := a.checkTimestamp(at); err != nil {
-		return fmt.Errorf("bad tipset: %w", err)
-	}
-	return nil
-}
-
-func (a *GatewayAPI) checkTipsetHeight(ts *types.TipSet, h abi.ChainEpoch) error {
-	tsBlock := ts.Blocks()[0]
-	heightDelta := time.Duration(uint64(tsBlock.Height-h)*build.BlockDelaySecs) * time.Second
-	timeAtHeight := time.Unix(int64(tsBlock.Timestamp), 0).Add(-heightDelta)
-
-	if err := a.checkTimestamp(timeAtHeight); err != nil {
-		return fmt.Errorf("bad tipset height: %w", err)
-	}
-	return nil
-}
-
-func (a *GatewayAPI) checkTimestamp(at time.Time) error {
-	if time.Since(at) > a.lookbackCap {
-		return ErrLookbackTooLong
-	}
-
-	return nil
-}
-
-func (a *GatewayAPI) Version(ctx context.Context) (api.Version, error) {
-	return a.api.Version(ctx)
-}
-
-func (a *GatewayAPI) ChainGetBlockMessages(ctx context.Context, c cid.Cid) (*api.BlockMessages, error) {
-	return a.api.ChainGetBlockMessages(ctx, c)
-}
-
-func (a *GatewayAPI) ChainHasObj(ctx context.Context, c cid.Cid) (bool, error) {
-	return a.api.ChainHasObj(ctx, c)
-}
-
-func (a *GatewayAPI) ChainHead(ctx context.Context) (*types.TipSet, error) {
-	// TODO: cache and invalidate cache when timestamp is up (or have internal ChainNotify)
-
-	return a.api.ChainHead(ctx)
-}
-
-func (a *GatewayAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) {
-	return a.api.ChainGetMessage(ctx, mc)
-}
-
-func (a *GatewayAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) {
-	return a.api.ChainGetTipSet(ctx, tsk)
-}
-
-func (a *GatewayAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) {
-	var ts *types.TipSet
-	if tsk.IsEmpty() {
-		head, err := a.api.ChainHead(ctx)
-		if err != nil {
-			return nil, err
-		}
-		ts = head
-	} else {
-		gts, err := a.api.ChainGetTipSet(ctx, tsk)
-		if err != nil {
-			return nil, err
-		}
-		ts = gts
-	}
-
-	// Check if the tipset key refers to a tipset that's too far in the past
-	if err := a.checkTipset(ts); err != nil {
-		return nil, err
-	}
-
-	// Check if the height is too far in the past
-	if err := a.checkTipsetHeight(ts, h); err != nil {
-		return nil, err
-	}
-
-	return a.api.ChainGetTipSetByHeight(ctx, h, tsk)
-}
-
-func (a *GatewayAPI) ChainGetNode(ctx context.Context, p string) (*api.IpldObject, error) {
-	return a.api.ChainGetNode(ctx, p)
-}
-
-func (a *GatewayAPI) ChainNotify(ctx context.Context) (<-chan []*api.HeadChange, error) {
-	return a.api.ChainNotify(ctx)
-}
-
-func (a *GatewayAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) {
-	return a.api.ChainReadObj(ctx, c)
-}
-
-func (a *GatewayAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return nil, err
-	}
-
-	return a.api.GasEstimateMessageGas(ctx, msg, spec, tsk)
-}
-
-func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) {
-	// TODO: additional anti-spam checks
-	return a.api.MpoolPushUntrusted(ctx, sm)
-}
-
-func (a *GatewayAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return types.NewInt(0), err
-	}
-
-	return a.api.MsigGetAvailableBalance(ctx, addr, tsk)
-}
-
-func (a *GatewayAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) {
-	if err := a.checkTipsetKey(ctx, start); err != nil {
-		return types.NewInt(0), err
-	}
-	if err := a.checkTipsetKey(ctx, end); err != nil {
-		return types.NewInt(0), err
-	}
-
-	return a.api.MsigGetVested(ctx, addr, start, end)
-}
-
-func (a *GatewayAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return address.Undef, err
-	}
-
-	return a.api.StateAccountKey(ctx, addr, tsk)
-}
-
-func (a *GatewayAPI) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return api.DealCollateralBounds{}, err
-	}
-
-	return a.api.StateDealProviderCollateralBounds(ctx, size, verified, tsk)
-}
-
-func (a *GatewayAPI) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return nil, err
-	}
-
-	return a.api.StateGetActor(ctx, actor, tsk)
-}
-
-func (a *GatewayAPI) StateGetReceipt(ctx context.Context, c cid.Cid, tsk types.TipSetKey) (*types.MessageReceipt, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return nil, err
-	}
-
-	return a.api.StateGetReceipt(ctx, c, tsk)
-}
-
-func (a *GatewayAPI) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return nil, err
-	}
-
-	return a.api.StateListMiners(ctx, tsk)
-}
-
-func (a *GatewayAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return address.Undef, err
-	}
-
-	return a.api.StateLookupID(ctx, addr, tsk)
-}
-
-func (a *GatewayAPI) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return api.MarketBalance{}, err
-	}
-
-	return a.api.StateMarketBalance(ctx, addr, tsk)
-}
-
-func (a *GatewayAPI) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return nil, err
-	}
-
-	return a.api.StateMarketStorageDeal(ctx, dealId, tsk)
-}
-
-func (a *GatewayAPI) StateNetworkVersion(ctx context.Context, tsk types.TipSetKey) (network.Version, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return network.VersionMax, err
-	}
-
-	return a.api.StateNetworkVersion(ctx, tsk)
-}
-
-func (a *GatewayAPI) StateWaitMsg(ctx context.Context, msg cid.Cid, confidence uint64) (*api.MsgLookup, error) {
-	return a.api.StateWaitMsgLimited(ctx, msg, confidence, a.stateWaitLookbackLimit)
-}
-
-func (a *GatewayAPI) StateReadState(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*api.ActorState, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return nil, err
-	}
-	return a.api.StateReadState(ctx, actor, tsk)
-}
-
-func (a *GatewayAPI) StateMinerPower(ctx context.Context, m address.Address, tsk types.TipSetKey) (*api.MinerPower, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return nil, err
-	}
-	return a.api.StateMinerPower(ctx, m, tsk)
-}
-
-func (a *GatewayAPI) StateMinerFaults(ctx context.Context, m address.Address, tsk types.TipSetKey) (bitfield.BitField, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return bitfield.BitField{}, err
-	}
-	return a.api.StateMinerFaults(ctx, m, tsk)
-}
-func (a *GatewayAPI) StateMinerRecoveries(ctx context.Context, m address.Address, tsk types.TipSetKey) (bitfield.BitField, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return bitfield.BitField{}, err
-	}
-	return a.api.StateMinerRecoveries(ctx, m, tsk)
-}
-
-func (a *GatewayAPI) StateMinerInfo(ctx context.Context, m address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return miner.MinerInfo{}, err
-	}
-	return a.api.StateMinerInfo(ctx, m, tsk)
-}
-
-func (a *GatewayAPI) StateMinerDeadlines(ctx context.Context, m address.Address, tsk types.TipSetKey) ([]api.Deadline, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return nil, err
-	}
-	return a.api.StateMinerDeadlines(ctx, m, tsk)
-}
-
-func (a *GatewayAPI) StateMinerAvailableBalance(ctx context.Context, m address.Address, tsk types.TipSetKey) (types.BigInt, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return types.BigInt{}, err
-	}
-	return a.api.StateMinerAvailableBalance(ctx, m, tsk)
-}
-
-func (a *GatewayAPI) StateMinerProvingDeadline(ctx context.Context, m address.Address, tsk types.TipSetKey) (*dline.Info, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return nil, err
-	}
-	return a.api.StateMinerProvingDeadline(ctx, m, tsk)
-}
-
-func (a *GatewayAPI) StateCirculatingSupply(ctx context.Context, tsk types.TipSetKey) (abi.TokenAmount, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return types.BigInt{}, err
-	}
-	return a.api.StateCirculatingSupply(ctx, tsk)
-
-}
-
-func (a *GatewayAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return nil, err
-	}
-	return a.api.StateVerifiedClientStatus(ctx, addr, tsk)
-}
-
-func (a *GatewayAPI) StateVMCirculatingSupplyInternal(ctx context.Context, tsk types.TipSetKey) (api.CirculatingSupply, error) {
-	if err := a.checkTipsetKey(ctx, tsk); err != nil {
-		return api.CirculatingSupply{}, err
-	}
-	return a.api.StateVMCirculatingSupplyInternal(ctx, tsk)
-}
-
-func (a *GatewayAPI) WalletVerify(ctx context.Context, k address.Address, msg []byte, sig *crypto.Signature) (bool, error) {
-	return sigs.Verify(sig, k, msg) == nil, nil
-}
-
-var _ api.GatewayAPI = (*GatewayAPI)(nil)
-var _ full.ChainModuleAPI = (*GatewayAPI)(nil)
-var _ full.GasModuleAPI = (*GatewayAPI)(nil)
-var _ full.MpoolModuleAPI = (*GatewayAPI)(nil)
-var _ full.StateModuleAPI = (*GatewayAPI)(nil)

cmd/lotus-gateway/api_test.go

@@ -1,237 +0,0 @@
-package main
-
-import (
-	"context"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/go-state-types/network"
-	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
-
-	"github.com/filecoin-project/lotus/build"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/filecoin-project/lotus/chain/types/mock"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/ipfs/go-cid"
-)
-
-func TestGatewayAPIChainGetTipSetByHeight(t *testing.T) {
-	ctx := context.Background()
-
-	lookbackTimestamp := uint64(time.Now().Unix()) - uint64(LookbackCap.Seconds())
-	type args struct {
-		h         abi.ChainEpoch
-		tskh      abi.ChainEpoch
-		genesisTS uint64
-	}
-	tests := []struct {
-		name   string
-		args   args
-		expErr bool
-	}{{
-		name: "basic",
-		args: args{
-			h:    abi.ChainEpoch(1),
-			tskh: abi.ChainEpoch(5),
-		},
-	}, {
-		name: "genesis",
-		args: args{
-			h:    abi.ChainEpoch(0),
-			tskh: abi.ChainEpoch(5),
-		},
-	}, {
-		name: "same epoch as tipset",
-		args: args{
-			h:    abi.ChainEpoch(5),
-			tskh: abi.ChainEpoch(5),
-		},
-	}, {
-		name: "tipset too old",
-		args: args{
-			// Tipset height is 5, genesis is at LookbackCap - 10 epochs.
-			// So resulting tipset height will be 5 epochs earlier than LookbackCap.
-			h:         abi.ChainEpoch(1),
-			tskh:      abi.ChainEpoch(5),
-			genesisTS: lookbackTimestamp - build.BlockDelaySecs*10,
-		},
-		expErr: true,
-	}, {
-		name: "lookup height too old",
-		args: args{
-			// Tipset height is 5, lookup height is 1, genesis is at LookbackCap - 3 epochs.
-			// So
-			// - lookup height will be 2 epochs earlier than LookbackCap.
-			// - tipset height will be 2 epochs later than LookbackCap.
-			h:         abi.ChainEpoch(1),
-			tskh:      abi.ChainEpoch(5),
-			genesisTS: lookbackTimestamp - build.BlockDelaySecs*3,
-		},
-		expErr: true,
-	}, {
-		name: "tipset and lookup height within acceptable range",
-		args: args{
-			// Tipset height is 5, lookup height is 1, genesis is at LookbackCap.
-			// So
-			// - lookup height will be 1 epoch later than LookbackCap.
-			// - tipset height will be 5 epochs later than LookbackCap.
-			h:         abi.ChainEpoch(1),
-			tskh:      abi.ChainEpoch(5),
-			genesisTS: lookbackTimestamp,
-		},
-	}}
-	for _, tt := range tests {
-		tt := tt
-		t.Run(tt.name, func(t *testing.T) {
-			mock := &mockGatewayDepsAPI{}
-			a := NewGatewayAPI(mock)
-
-			// Create tipsets from genesis up to tskh and return the highest
-			ts := mock.createTipSets(tt.args.tskh, tt.args.genesisTS)
-
-			got, err := a.ChainGetTipSetByHeight(ctx, tt.args.h, ts.Key())
-			if tt.expErr {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-				require.Equal(t, tt.args.h, got.Height())
-			}
-		})
-	}
-}
-
-type mockGatewayDepsAPI struct {
-	lk      sync.RWMutex
-	tipsets []*types.TipSet
-
-	gatewayDepsAPI // satisfies all interface requirements but will panic if
-	// methods are called. easier than filling out with panic stubs IMO
-}
-
-func (m *mockGatewayDepsAPI) ChainHasObj(context.Context, cid.Cid) (bool, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Message, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) StateListMiners(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) StateMarketBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (api.MarketBalance, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) StateMarketStorageDeal(ctx context.Context, dealId abi.DealID, tsk types.TipSetKey) (*api.MarketDeal, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) StateMinerInfo(ctx context.Context, actor address.Address, tsk types.TipSetKey) (miner.MinerInfo, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) ChainHead(ctx context.Context) (*types.TipSet, error) {
-	m.lk.RLock()
-	defer m.lk.RUnlock()
-
-	return m.tipsets[len(m.tipsets)-1], nil
-}
-
-func (m *mockGatewayDepsAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) {
-	m.lk.RLock()
-	defer m.lk.RUnlock()
-
-	for _, ts := range m.tipsets {
-		if ts.Key() == tsk {
-			return ts, nil
-		}
-	}
-
-	return nil, nil
-}
-
-// createTipSets creates tipsets from genesis up to tskh and returns the highest
-func (m *mockGatewayDepsAPI) createTipSets(h abi.ChainEpoch, genesisTimestamp uint64) *types.TipSet {
-	m.lk.Lock()
-	defer m.lk.Unlock()
-
-	targeth := h + 1 // add one for genesis block
-	if genesisTimestamp == 0 {
-		genesisTimestamp = uint64(time.Now().Unix()) - build.BlockDelaySecs*uint64(targeth)
-	}
-	var currts *types.TipSet
-	for currh := abi.ChainEpoch(0); currh < targeth; currh++ {
-		blks := mock.MkBlock(currts, 1, 1)
-		if currh == 0 {
-			blks.Timestamp = genesisTimestamp
-		}
-		currts = mock.TipSet(blks)
-		m.tipsets = append(m.tipsets, currts)
-	}
-
-	return m.tipsets[len(m.tipsets)-1]
-}
-
-func (m *mockGatewayDepsAPI) ChainGetTipSetByHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) {
-	m.lk.Lock()
-	defer m.lk.Unlock()
-
-	return m.tipsets[h], nil
-}
-
-func (m *mockGatewayDepsAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec, tsk types.TipSetKey) (*types.Message, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) MpoolPushUntrusted(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) StateLookupID(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) StateWaitMsgLimited(ctx context.Context, msg cid.Cid, confidence uint64, h abi.ChainEpoch) (*api.MsgLookup, error) {
-	panic("implement me")
-}
-
-func (m *mockGatewayDepsAPI) StateReadState(ctx context.Context, act address.Address, ts types.TipSetKey) (*api.ActorState, error) {
-	panic("implement me")
-}

cmd/lotus-gateway/endtoend_test.go

@@ -1,306 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"math"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/filecoin-project/lotus/cli"
-	clitest "github.com/filecoin-project/lotus/cli/test"
-
-	init2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/init"
-	multisig2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/multisig"
-
-	"github.com/stretchr/testify/require"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-jsonrpc"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/client"
-	"github.com/filecoin-project/lotus/api/test"
-	"github.com/filecoin-project/lotus/chain/actors/policy"
-	"github.com/filecoin-project/lotus/chain/stmgr"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/node"
-	builder "github.com/filecoin-project/lotus/node/test"
-)
-
-const maxLookbackCap = time.Duration(math.MaxInt64)
-const maxStateWaitLookbackLimit = stmgr.LookbackNoLimit
-
-func init() {
-	policy.SetSupportedProofTypes(abi.RegisteredSealProof_StackedDrg2KiBV1)
-	policy.SetConsensusMinerMinPower(abi.NewStoragePower(2048))
-	policy.SetMinVerifiedDealSize(abi.NewStoragePower(256))
-}
-
-// TestWalletMsig tests that API calls to wallet and msig can be made on a lite
-// node that is connected through a gateway to a full API node
-func TestWalletMsig(t *testing.T) {
-	_ = os.Setenv("BELLMAN_NO_GPU", "1")
-	clitest.QuietMiningLogs()
-
-	blocktime := 5 * time.Millisecond
-	ctx := context.Background()
-	nodes := startNodes(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit)
-	defer nodes.closer()
-
-	lite := nodes.lite
-	full := nodes.full
-
-	// The full node starts with a wallet
-	fullWalletAddr, err := full.WalletDefaultAddress(ctx)
-	require.NoError(t, err)
-
-	// Check the full node's wallet balance from the lite node
-	balance, err := lite.WalletBalance(ctx, fullWalletAddr)
-	require.NoError(t, err)
-	fmt.Println(balance)
-
-	// Create a wallet on the lite node
-	liteWalletAddr, err := lite.WalletNew(ctx, types.KTSecp256k1)
-	require.NoError(t, err)
-
-	// Send some funds from the full node to the lite node
-	err = sendFunds(ctx, full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18))
-	require.NoError(t, err)
-
-	// Send some funds from the lite node back to the full node
-	err = sendFunds(ctx, lite, liteWalletAddr, fullWalletAddr, types.NewInt(100))
-	require.NoError(t, err)
-
-	// Sign some data with the lite node wallet address
-	data := []byte("hello")
-	sig, err := lite.WalletSign(ctx, liteWalletAddr, data)
-	require.NoError(t, err)
-
-	// Verify the signature
-	ok, err := lite.WalletVerify(ctx, liteWalletAddr, data, sig)
-	require.NoError(t, err)
-	require.True(t, ok)
-
-	// Create some wallets on the lite node to use for testing multisig
-	var walletAddrs []address.Address
-	for i := 0; i < 4; i++ {
-		addr, err := lite.WalletNew(ctx, types.KTSecp256k1)
-		require.NoError(t, err)
-
-		walletAddrs = append(walletAddrs, addr)
-
-		err = sendFunds(ctx, lite, liteWalletAddr, addr, types.NewInt(1e15))
-		require.NoError(t, err)
-	}
-
-	// Create an msig with three of the addresses and threshold of two sigs
-	msigAddrs := walletAddrs[:3]
-	amt := types.NewInt(1000)
-	addProposal, err := lite.MsigCreate(ctx, 2, msigAddrs, abi.ChainEpoch(50), amt, liteWalletAddr, types.NewInt(0))
-	require.NoError(t, err)
-
-	res, err := lite.StateWaitMsg(ctx, addProposal, 1)
-	require.NoError(t, err)
-	require.EqualValues(t, 0, res.Receipt.ExitCode)
-
-	var execReturn init2.ExecReturn
-	err = execReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return))
-	require.NoError(t, err)
-
-	// Get available balance of msig: should be greater than zero and less
-	// than initial amount
-	msig := execReturn.IDAddress
-	msigBalance, err := lite.MsigGetAvailableBalance(ctx, msig, types.EmptyTSK)
-	require.NoError(t, err)
-	require.Greater(t, msigBalance.Int64(), int64(0))
-	require.Less(t, msigBalance.Int64(), amt.Int64())
-
-	// Propose to add a new address to the msig
-	addProposal, err = lite.MsigAddPropose(ctx, msig, walletAddrs[0], walletAddrs[3], false)
-	require.NoError(t, err)
-
-	res, err = lite.StateWaitMsg(ctx, addProposal, 1)
-	require.NoError(t, err)
-	require.EqualValues(t, 0, res.Receipt.ExitCode)
-
-	var proposeReturn multisig2.ProposeReturn
-	err = proposeReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return))
-	require.NoError(t, err)
-
-	// Approve proposal (proposer is first (implicit) signer, approver is
-	// second signer
-	txnID := uint64(proposeReturn.TxnID)
-	approval1, err := lite.MsigAddApprove(ctx, msig, walletAddrs[1], txnID, walletAddrs[0], walletAddrs[3], false)
-	require.NoError(t, err)
-
-	res, err = lite.StateWaitMsg(ctx, approval1, 1)
-	require.NoError(t, err)
-	require.EqualValues(t, 0, res.Receipt.ExitCode)
-
-	var approveReturn multisig2.ApproveReturn
-	err = approveReturn.UnmarshalCBOR(bytes.NewReader(res.Receipt.Return))
-	require.NoError(t, err)
-	require.True(t, approveReturn.Applied)
-}
-
-// TestMsigCLI tests that msig CLI calls can be made
-// on a lite node that is connected through a gateway to a full API node
-func TestMsigCLI(t *testing.T) {
-	_ = os.Setenv("BELLMAN_NO_GPU", "1")
-	clitest.QuietMiningLogs()
-
-	blocktime := 5 * time.Millisecond
-	ctx := context.Background()
-	nodes := startNodesWithFunds(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit)
-	defer nodes.closer()
-
-	lite := nodes.lite
-	clitest.RunMultisigTest(t, cli.Commands, lite)
-}
-
-func TestDealFlow(t *testing.T) {
-	_ = os.Setenv("BELLMAN_NO_GPU", "1")
-	clitest.QuietMiningLogs()
-
-	blocktime := 5 * time.Millisecond
-	ctx := context.Background()
-	nodes := startNodesWithFunds(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit)
-	defer nodes.closer()
-
-	test.MakeDeal(t, ctx, 6, nodes.lite, nodes.miner, false, false)
-}
-
-func TestCLIDealFlow(t *testing.T) {
-	_ = os.Setenv("BELLMAN_NO_GPU", "1")
-	clitest.QuietMiningLogs()
-
-	blocktime := 5 * time.Millisecond
-	ctx := context.Background()
-	nodes := startNodesWithFunds(ctx, t, blocktime, maxLookbackCap, maxStateWaitLookbackLimit)
-	defer nodes.closer()
-
-	clitest.RunClientTest(t, cli.Commands, nodes.lite)
-}
-
-type testNodes struct {
-	lite   test.TestNode
-	full   test.TestNode
-	miner  test.TestStorageNode
-	closer jsonrpc.ClientCloser
-}
-
-func startNodesWithFunds(
-	ctx context.Context,
-	t *testing.T,
-	blocktime time.Duration,
-	lookbackCap time.Duration,
-	stateWaitLookbackLimit abi.ChainEpoch,
-) *testNodes {
-	nodes := startNodes(ctx, t, blocktime, lookbackCap, stateWaitLookbackLimit)
-
-	// The full node starts with a wallet
-	fullWalletAddr, err := nodes.full.WalletDefaultAddress(ctx)
-	require.NoError(t, err)
-
-	// Create a wallet on the lite node
-	liteWalletAddr, err := nodes.lite.WalletNew(ctx, types.KTSecp256k1)
-	require.NoError(t, err)
-
-	// Send some funds from the full node to the lite node
-	err = sendFunds(ctx, nodes.full, fullWalletAddr, liteWalletAddr, types.NewInt(1e18))
-	require.NoError(t, err)
-
-	return nodes
-}
-
-func startNodes(
-	ctx context.Context,
-	t *testing.T,
-	blocktime time.Duration,
-	lookbackCap time.Duration,
-	stateWaitLookbackLimit abi.ChainEpoch,
-) *testNodes {
-	var closer jsonrpc.ClientCloser
-
-	// Create one miner and two full nodes.
-	// - Put a gateway server in front of full node 1
-	// - Start full node 2 in lite mode
-	// - Connect lite node -> gateway server -> full node
-	opts := append(
-		// Full node
-		test.OneFull,
-		// Lite node
-		test.FullNodeOpts{
-			Lite: true,
-			Opts: func(nodes []test.TestNode) node.Option {
-				fullNode := nodes[0]
-
-				// Create a gateway server in front of the full node
-				gapiImpl := newGatewayAPI(fullNode, lookbackCap, stateWaitLookbackLimit)
-				_, addr, err := builder.CreateRPCServer(gapiImpl)
-				require.NoError(t, err)
-
-				// Create a gateway client API that connects to the gateway server
-				var gapi api.GatewayAPI
-				gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil)
-				require.NoError(t, err)
-
-				// Provide the gateway API to dependency injection
-				return node.Override(new(api.GatewayAPI), gapi)
-			},
-		},
-	)
-	n, sn := builder.RPCMockSbBuilder(t, opts, test.OneMiner)
-
-	full := n[0]
-	lite := n[1]
-	miner := sn[0]
-
-	// Get the listener address for the full node
-	fullAddr, err := full.NetAddrsListen(ctx)
-	require.NoError(t, err)
-
-	// Connect the miner and the full node
-	err = miner.NetConnect(ctx, fullAddr)
-	require.NoError(t, err)
-
-	// Connect the miner and the lite node (so that the lite node can send
-	// data to the miner)
-	liteAddr, err := lite.NetAddrsListen(ctx)
-	require.NoError(t, err)
-	err = miner.NetConnect(ctx, liteAddr)
-	require.NoError(t, err)
-
-	// Start mining blocks
-	bm := test.NewBlockMiner(ctx, t, miner, blocktime)
-	bm.MineBlocks()
-
-	return &testNodes{lite: lite, full: full, miner: miner, closer: closer}
-}
-
-func sendFunds(ctx context.Context, fromNode test.TestNode, fromAddr address.Address, toAddr address.Address, amt types.BigInt) error {
-	msg := &types.Message{
-		From:  fromAddr,
-		To:    toAddr,
-		Value: amt,
-	}
-
-	sm, err := fromNode.MpoolPushMessage(ctx, msg, nil)
-	if err != nil {
-		return err
-	}
-
-	res, err := fromNode.StateWaitMsg(ctx, sm.Cid(), 1)
-	if err != nil {
-		return err
-	}
-	if res.Receipt.ExitCode != 0 {
-		return xerrors.Errorf("send funds failed with exit code %d", res.Receipt.ExitCode)
-	}
-
-	return nil
-}

cmd/lotus-gateway/main.go

@@ -2,23 +2,31 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"net"
-	"net/http"
 	"os"
 
-	"github.com/filecoin-project/go-jsonrpc"
-	"go.opencensus.io/tag"
+	"github.com/urfave/cli/v2"
+	"go.opencensus.io/stats/view"
+	"golang.org/x/xerrors"
 
+	logging "github.com/ipfs/go-log/v2"
+
+	"github.com/filecoin-project/go-jsonrpc"
+	"github.com/filecoin-project/go-state-types/abi"
+
+	manet "github.com/multiformats/go-multiaddr/net"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/api/client"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/gateway"
 	"github.com/filecoin-project/lotus/lib/lotuslog"
 	"github.com/filecoin-project/lotus/metrics"
-
-	logging "github.com/ipfs/go-log"
-	"go.opencensus.io/stats/view"
-
-	"github.com/gorilla/mux"
-	"github.com/urfave/cli/v2"
+	"github.com/filecoin-project/lotus/node"
 )
 
 var log = logging.Logger("gateway")
@@ -28,6 +36,7 @@ func main() {
 
 	local := []*cli.Command{
 		runCmd,
+		checkCmd,
 	}
 
 	app := &cli.App{
@@ -47,11 +56,60 @@ func main() {
 	app.Setup()
 
 	if err := app.Run(os.Args); err != nil {
-		log.Warnf("%+v", err)
+		log.Errorf("%+v", err)
+		os.Exit(1)
 		return
 	}
 }
 
+var checkCmd = &cli.Command{
+	Name:      "check",
+	Usage:     "performs a simple check to verify that a connection can be made to a gateway",
+	ArgsUsage: "[apiInfo]",
+	Description: `Any valid value for FULLNODE_API_INFO is a valid argument to the check command.
+
+   Examples
+   - ws://127.0.0.1:2346
+   - http://127.0.0.1:2346
+   - /ip4/127.0.0.1/tcp/2346`,
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		ainfo := cliutil.ParseApiInfo(cctx.Args().First())
+
+		darg, err := ainfo.DialArgs("v1")
+		if err != nil {
+			return err
+		}
+
+		api, closer, err := client.NewFullNodeRPCV1(ctx, darg, nil)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+
+		addr, err := address.NewIDAddress(100)
+		if err != nil {
+			return err
+		}
+
+		laddr, err := api.StateLookupID(ctx, addr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		if laddr != addr {
+			return fmt.Errorf("looked up addresses does not match returned address, %s != %s", addr, laddr)
+		}
+
+		return nil
+	},
+}
+
 var runCmd = &cli.Command{
 	Name:  "run",
 	Usage: "Start api server",
@@ -61,65 +119,75 @@ var runCmd = &cli.Command{
 			Usage: "host address and port the api server will listen on",
 			Value: "0.0.0.0:2346",
 		},
+		&cli.IntFlag{
+			Name:  "api-max-req-size",
+			Usage: "maximum API request size accepted by the JSON RPC server",
+		},
+		&cli.DurationFlag{
+			Name:  "api-max-lookback",
+			Usage: "maximum duration allowable for tipset lookbacks",
+			Value: gateway.DefaultLookbackCap,
+		},
+		&cli.Int64Flag{
+			Name:  "api-wait-lookback-limit",
+			Usage: "maximum number of blocks to search back through for message inclusion",
+			Value: int64(gateway.DefaultStateWaitLookbackLimit),
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		log.Info("Starting lotus gateway")
 
-		ctx := lcli.ReqContext(cctx)
-		ctx, cancel := context.WithCancel(ctx)
-		defer cancel()
-
 		// Register all metric views
 		if err := view.Register(
-			metrics.DefaultViews...,
+			metrics.ChainNodeViews...,
 		); err != nil {
 			log.Fatalf("Cannot register the view: %v", err)
 		}
 
-		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		api, closer, err := lcli.GetFullNodeAPIV1(cctx)
 		if err != nil {
 			return err
 		}
 		defer closer()
 
-		address := cctx.String("listen")
-		mux := mux.NewRouter()
+		var (
+			lookbackCap  = cctx.Duration("api-max-lookback")
+			address      = cctx.String("listen")
+			waitLookback = abi.ChainEpoch(cctx.Int64("api-wait-lookback-limit"))
+		)
 
-		log.Info("Setting up API endpoint at " + address)
-
-		rpcServer := jsonrpc.NewServer()
-		rpcServer.Register("Filecoin", metrics.MetricedGatewayAPI(NewGatewayAPI(api)))
-
-		mux.Handle("/rpc/v0", rpcServer)
-		mux.PathPrefix("/").Handler(http.DefaultServeMux)
-
-		/*ah := &auth.Handler{
-			Verify: nodeApi.AuthVerify,
-			Next:   mux.ServeHTTP,
-		}*/
-
-		srv := &http.Server{
-			Handler: mux,
-			BaseContext: func(listener net.Listener) context.Context {
-				ctx, _ := tag.New(context.Background(), tag.Upsert(metrics.APIInterface, "lotus-gateway"))
-				return ctx
-			},
+		serverOptions := make([]jsonrpc.ServerOption, 0)
+		if maxRequestSize := cctx.Int("api-max-req-size"); maxRequestSize != 0 {
+			serverOptions = append(serverOptions, jsonrpc.WithMaxRequestSize(int64(maxRequestSize)))
 		}
 
-		go func() {
-			<-ctx.Done()
-			log.Warn("Shutting down...")
-			if err := srv.Shutdown(context.TODO()); err != nil {
-				log.Errorf("shutting down RPC server failed: %s", err)
-			}
-			log.Warn("Graceful shutdown successful")
-		}()
+		log.Info("setting up API endpoint at " + address)
 
-		nl, err := net.Listen("tcp", address)
+		addr, err := net.ResolveTCPAddr("tcp", address)
 		if err != nil {
-			return err
+			return xerrors.Errorf("failed to resolve endpoint address: %w", err)
 		}
 
-		return srv.Serve(nl)
+		maddr, err := manet.FromNetAddr(addr)
+		if err != nil {
+			return xerrors.Errorf("failed to convert endpoint address to multiaddr: %w", err)
+		}
+
+		gwapi := gateway.NewNode(api, lookbackCap, waitLookback)
+		h, err := gateway.Handler(gwapi, serverOptions...)
+		if err != nil {
+			return xerrors.Errorf("failed to set up gateway HTTP handler")
+		}
+
+		stopFunc, err := node.ServeRPC(h, "lotus-gateway", maddr)
+		if err != nil {
+			return xerrors.Errorf("failed to serve rpc endpoint: %w", err)
+		}
+
+		<-node.MonitorShutdown(nil, node.ShutdownHandler{
+			Component: "rpc",
+			StopFunc:  stopFunc,
+		})
+		return nil
 	},
 }

cmd/lotus-health/main.go

@@ -8,13 +8,14 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/filecoin-project/lotus/api/v0api"
+
 	cid "github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/urfave/cli/v2"
 
 	"github.com/filecoin-project/go-jsonrpc"
 
-	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
@@ -180,7 +181,7 @@ func checkWindow(window CidWindow, t int) bool {
  * returns a slice of slices of Cids
  * len of slice <= `t` - threshold
  */
-func updateWindow(ctx context.Context, a api.FullNode, w CidWindow, t int, r int, to time.Duration) (CidWindow, error) {
+func updateWindow(ctx context.Context, a v0api.FullNode, w CidWindow, t int, r int, to time.Duration) (CidWindow, error) {
 	head, err := getHead(ctx, a, r, to)
 	if err != nil {
 		return nil, err
@@ -194,7 +195,7 @@ func updateWindow(ctx context.Context, a api.FullNode, w CidWindow, t int, r int
  * retries if API no available
  * returns tipset
  */
-func getHead(ctx context.Context, a api.FullNode, r int, t time.Duration) (*types.TipSet, error) {
+func getHead(ctx context.Context, a v0api.FullNode, r int, t time.Duration) (*types.TipSet, error) {
 	for i := 0; i < r; i++ {
 		head, err := a.ChainHead(ctx)
 		if err != nil && i == (r-1) {
@@ -226,7 +227,7 @@ func appendCIDsToWindow(w CidWindow, c []cid.Cid, t int) CidWindow {
 /*
  * wait for node to sync
  */
-func waitForSyncComplete(ctx context.Context, a api.FullNode, r int, t time.Duration) error {
+func waitForSyncComplete(ctx context.Context, a v0api.FullNode, r int, t time.Duration) error {
 	for {
 		select {
 		case <-ctx.Done():
@@ -248,7 +249,7 @@ func waitForSyncComplete(ctx context.Context, a api.FullNode, r int, t time.Dura
  * A thin wrapper around lotus cli GetFullNodeAPI
  * Adds retry logic
  */
-func getFullNodeAPI(ctx *cli.Context, r int, t time.Duration) (api.FullNode, jsonrpc.ClientCloser, error) {
+func getFullNodeAPI(ctx *cli.Context, r int, t time.Duration) (v0api.FullNode, jsonrpc.ClientCloser, error) {
 	for i := 0; i < r; i++ {
 		api, closer, err := lcli.GetFullNodeAPI(ctx)
 		if err != nil && i == (r-1) {

cmd/lotus-keygen/main.go

@@ -22,6 +22,11 @@ func main() {
 			Value:   "bls",
 			Usage:   "specify key type to generate (bls or secp256k1)",
 		},
+		&cli.StringFlag{
+			Name:    "out",
+			Aliases: []string{"o"},
+			Usage:   "specify key file name to generate",
+		},
 	}
 	app.Action = func(cctx *cli.Context) error {
 		memks := wallet.NewMemKeyStore()
@@ -50,7 +55,11 @@ func main() {
 			return err
 		}
 
-		fi, err := os.Create(fmt.Sprintf("%s.key", kaddr))
+		outFile := fmt.Sprintf("%s.key", kaddr)
+		if cctx.IsSet("out") {
+			outFile = fmt.Sprintf("%s.key", cctx.String("out"))
+		}
+		fi, err := os.Create(outFile)
 		if err != nil {
 			return err
 		}

cmd/lotus-seal-worker/cli.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var setCmd = &cli.Command{
+	Name:  "set",
+	Usage: "Manage worker settings",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "enabled",
+			Usage: "enable/disable new task processing",
+			Value: true,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetWorkerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		if err := api.SetEnabled(ctx, cctx.Bool("enabled")); err != nil {
+			return xerrors.Errorf("SetEnabled: %w", err)
+		}
+
+		return nil
+	},
+}
+
+var waitQuietCmd = &cli.Command{
+	Name:  "wait-quiet",
+	Usage: "Block until all running tasks exit",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetWorkerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		return api.WaitQuiet(ctx)
+	},
+}

cmd/lotus-seal-worker/info.go

@@ -2,12 +2,14 @@ package main
 
 import (
 	"fmt"
+	"sort"
 
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 )
 
 var infoCmd = &cli.Command{
@@ -32,15 +34,39 @@ var infoCmd = &cli.Command{
 		cli.VersionPrinter(cctx)
 		fmt.Println()
 
+		sess, err := api.ProcessSession(ctx)
+		if err != nil {
+			return xerrors.Errorf("getting session: %w", err)
+		}
+		fmt.Printf("Session: %s\n", sess)
+
+		enabled, err := api.Enabled(ctx)
+		if err != nil {
+			return xerrors.Errorf("checking worker status: %w", err)
+		}
+		fmt.Printf("Enabled: %t\n", enabled)
+
 		info, err := api.Info(ctx)
 		if err != nil {
 			return xerrors.Errorf("getting info: %w", err)
 		}
 
+		tt, err := api.TaskTypes(ctx)
+		if err != nil {
+			return xerrors.Errorf("getting task types: %w", err)
+		}
+
 		fmt.Printf("Hostname: %s\n", info.Hostname)
 		fmt.Printf("CPUs: %d; GPUs: %v\n", info.Resources.CPUs, info.Resources.GPUs)
 		fmt.Printf("RAM: %s; Swap: %s\n", types.SizeStr(types.NewInt(info.Resources.MemPhysical)), types.SizeStr(types.NewInt(info.Resources.MemSwap)))
 		fmt.Printf("Reserved memory: %s\n", types.SizeStr(types.NewInt(info.Resources.MemReserved)))
+
+		fmt.Printf("Task types: ")
+		for _, t := range ttList(tt) {
+			fmt.Printf("%s ", t.Short())
+		}
+		fmt.Println()
+
 		fmt.Println()
 
 		paths, err := api.Paths(ctx)
@@ -52,7 +78,6 @@ var infoCmd = &cli.Command{
 			fmt.Printf("%s:\n", path.ID)
 			fmt.Printf("\tWeight: %d; Use: ", path.Weight)
 			if path.CanSeal || path.CanStore {
-				fmt.Printf("Weight: %d; Use: ", path.Weight)
 				if path.CanSeal {
 					fmt.Print("Seal ")
 				}
@@ -69,3 +94,14 @@ var infoCmd = &cli.Command{
 		return nil
 	},
 }
+
+func ttList(tt map[sealtasks.TaskType]struct{}) []sealtasks.TaskType {
+	tasks := make([]sealtasks.TaskType, 0, len(tt))
+	for taskType := range tt {
+		tasks = append(tasks, taskType)
+	}
+	sort.Slice(tasks, func(i, j int) bool {
+		return tasks[i].Less(tasks[j])
+	})
+	return tasks
+}

cmd/lotus-seal-worker/main.go

@@ -28,11 +28,10 @@ import (
 	"github.com/filecoin-project/go-statestore"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/api/apistruct"
 	"github.com/filecoin-project/lotus/build"
 	lcli "github.com/filecoin-project/lotus/cli"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
-	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
 	"github.com/filecoin-project/lotus/lib/lotuslog"
@@ -50,7 +49,7 @@ const FlagWorkerRepo = "worker-repo"
 const FlagWorkerRepoDeprecation = "workerrepo"
 
 func main() {
-	build.RunningNodeType = build.NodeWorker
+	api.RunningNodeType = api.NodeWorker
 
 	lotuslog.SetupLogLevels()
 
@@ -58,12 +57,16 @@ func main() {
 		runCmd,
 		infoCmd,
 		storageCmd,
+		setCmd,
+		waitQuietCmd,
+		tasksCmd,
 	}
 
 	app := &cli.App{
-		Name:    "lotus-worker",
-		Usage:   "Remote miner worker",
-		Version: build.UserVersion(),
+		Name:                 "lotus-worker",
+		Usage:                "Remote miner worker",
+		Version:              build.UserVersion(),
+		EnableBashCompletion: true,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:    FlagWorkerRepo,
@@ -181,7 +184,7 @@ var runCmd = &cli.Command{
 		var closer func()
 		var err error
 		for {
-			nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, lcli.StorageMinerUseHttp)
+			nodeApi, closer, err = lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp)
 			if err == nil {
 				_, err = nodeApi.Version(ctx)
 				if err == nil {
@@ -208,8 +211,8 @@ var runCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
-		if v.APIVersion != build.MinerAPIVersion {
-			return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.Version{APIVersion: build.MinerAPIVersion})
+		if v.APIVersion != api.MinerAPIVersion0 {
+			return xerrors.Errorf("lotus-miner API version doesn't match: expected: %s", api.APIVersion{APIVersion: api.MinerAPIVersion0})
 		}
 		log.Infof("Remote version %s", v)
 
@@ -225,7 +228,7 @@ var runCmd = &cli.Command{
 		}
 
 		if cctx.Bool("commit") {
-			if err := paramfetch.GetParams(ctx, build.ParametersJSON(), uint64(ssize)); err != nil {
+			if err := paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), uint64(ssize)); err != nil {
 				return xerrors.Errorf("get params: %w", err)
 			}
 		}
@@ -306,7 +309,7 @@ var runCmd = &cli.Command{
 
 			{
 				// init datastore for r.Exists
-				_, err := lr.Datastore("/metadata")
+				_, err := lr.Datastore(context.Background(), "/metadata")
 				if err != nil {
 					return err
 				}
@@ -325,7 +328,7 @@ var runCmd = &cli.Command{
 				log.Error("closing repo", err)
 			}
 		}()
-		ds, err := lr.Datastore("/metadata")
+		ds, err := lr.Datastore(context.Background(), "/metadata")
 		if err != nil {
 			return err
 		}
@@ -354,17 +357,24 @@ var runCmd = &cli.Command{
 		}
 
 		// Setup remote sector store
-		spt, err := ffiwrapper.SealProofTypeFromSectorSize(ssize)
-		if err != nil {
-			return xerrors.Errorf("getting proof type: %w", err)
-		}
-
 		sminfo, err := lcli.GetAPIInfo(cctx, repo.StorageMiner)
 		if err != nil {
 			return xerrors.Errorf("could not get api info: %w", err)
 		}
 
-		remote := stores.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"))
+		remote := stores.NewRemote(localStore, nodeApi, sminfo.AuthHeader(), cctx.Int("parallel-fetch-limit"),
+			&stores.DefaultPartialFileHandler{})
+
+		fh := &stores.FetchHandler{Local: localStore, PfHandler: &stores.DefaultPartialFileHandler{}}
+		remoteHandler := func(w http.ResponseWriter, r *http.Request) {
+			if !auth.HasPerm(r.Context(), nil, api.PermAdmin) {
+				w.WriteHeader(401)
+				_ = json.NewEncoder(w).Encode(struct{ Error string }{"unauthorized: missing admin permission"})
+				return
+			}
+
+			fh.ServeHTTP(w, r)
+		}
 
 		// Create / expose the worker
 
@@ -372,7 +382,6 @@ var runCmd = &cli.Command{
 
 		workerApi := &worker{
 			LocalWorker: sectorstorage.NewLocalWorker(sectorstorage.WorkerConfig{
-				SealProof: spt,
 				TaskTypes: taskTypes,
 				NoSwap:    cctx.Bool("no-swap"),
 			}, remote, localStore, nodeApi, nodeApi, wsts),
@@ -386,11 +395,11 @@ var runCmd = &cli.Command{
 
 		readerHandler, readerServerOpt := rpcenc.ReaderParamDecoder()
 		rpcServer := jsonrpc.NewServer(readerServerOpt)
-		rpcServer.Register("Filecoin", apistruct.PermissionedWorkerAPI(metrics.MetricedWorkerAPI(workerApi)))
+		rpcServer.Register("Filecoin", api.PermissionedWorkerAPI(metrics.MetricedWorkerAPI(workerApi)))
 
 		mux.Handle("/rpc/v0", rpcServer)
 		mux.Handle("/rpc/streams/v0/push/{uuid}", readerHandler)
-		mux.PathPrefix("/remote").HandlerFunc((&stores.FetchHandler{Local: localStore}).ServeHTTP)
+		mux.PathPrefix("/remote").HandlerFunc(remoteHandler)
 		mux.PathPrefix("/").Handler(http.DefaultServeMux) // pprof
 
 		ah := &auth.Handler{
@@ -451,14 +460,24 @@ var runCmd = &cli.Command{
 			return xerrors.Errorf("getting miner session: %w", err)
 		}
 
+		waitQuietCh := func() chan struct{} {
+			out := make(chan struct{})
+			go func() {
+				workerApi.LocalWorker.WaitQuiet()
+				close(out)
+			}()
+			return out
+		}
+
 		go func() {
 			heartbeats := time.NewTicker(stores.HeartbeatInterval)
 			defer heartbeats.Stop()
 
-			var connected, reconnect bool
+			var redeclareStorage bool
+			var readyCh chan struct{}
 			for {
 				// If we're reconnecting, redeclare storage first
-				if reconnect {
+				if redeclareStorage {
 					log.Info("Redeclaring local storage")
 
 					if err := localStore.Redeclare(ctx); err != nil {
@@ -471,14 +490,13 @@ var runCmd = &cli.Command{
 						}
 						continue
 					}
-
-					connected = false
 				}
 
-				log.Info("Making sure no local tasks are running")
-
 				// TODO: we could get rid of this, but that requires tracking resources for restarted tasks correctly
-				workerApi.LocalWorker.WaitQuiet()
+				if readyCh == nil {
+					log.Info("Making sure no local tasks are running")
+					readyCh = waitQuietCh()
+				}
 
 				for {
 					curSession, err := nodeApi.Session(ctx)
@@ -489,29 +507,28 @@ var runCmd = &cli.Command{
 							minerSession = curSession
 							break
 						}
-
-						if !connected {
-							if err := nodeApi.WorkerConnect(ctx, "http://"+address+"/rpc/v0"); err != nil {
-								log.Errorf("Registering worker failed: %+v", err)
-								cancel()
-								return
-							}
-
-							log.Info("Worker registered successfully, waiting for tasks")
-							connected = true
-						}
 					}
 
 					select {
+					case <-readyCh:
+						if err := nodeApi.WorkerConnect(ctx, "http://"+address+"/rpc/v0"); err != nil {
+							log.Errorf("Registering worker failed: %+v", err)
+							cancel()
+							return
+						}
+
+						log.Info("Worker registered successfully, waiting for tasks")
+
+						readyCh = nil
+					case <-heartbeats.C:
 					case <-ctx.Done():
 						return // graceful shutdown
-					case <-heartbeats.C:
 					}
 				}
 
 				log.Errorf("LOTUS-MINER CONNECTION LOST")
 
-				reconnect = true
+				redeclareStorage = true
 			}
 		}()
 

cmd/lotus-seal-worker/rpc.go

@@ -2,10 +2,14 @@ package main
 
 import (
 	"context"
+	"sync/atomic"
 
+	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/lotus/api"
+	apitypes "github.com/filecoin-project/lotus/api/types"
 	"github.com/filecoin-project/lotus/build"
 	sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
 	"github.com/filecoin-project/lotus/extern/sector-storage/stores"
@@ -17,10 +21,12 @@ type worker struct {
 
 	localStore *stores.Local
 	ls         stores.LocalStorage
+
+	disabled int64
 }
 
-func (w *worker) Version(context.Context) (build.Version, error) {
-	return build.WorkerAPIVersion, nil
+func (w *worker) Version(context.Context) (api.Version, error) {
+	return api.WorkerAPIVersion0, nil
 }
 
 func (w *worker) StorageAddLocal(ctx context.Context, path string) error {
@@ -42,4 +48,38 @@ func (w *worker) StorageAddLocal(ctx context.Context, path string) error {
 	return nil
 }
 
+func (w *worker) SetEnabled(ctx context.Context, enabled bool) error {
+	disabled := int64(1)
+	if enabled {
+		disabled = 0
+	}
+	atomic.StoreInt64(&w.disabled, disabled)
+	return nil
+}
+
+func (w *worker) Enabled(ctx context.Context) (bool, error) {
+	return atomic.LoadInt64(&w.disabled) == 0, nil
+}
+
+func (w *worker) WaitQuiet(ctx context.Context) error {
+	w.LocalWorker.WaitQuiet() // uses WaitGroup under the hood so no ctx :/
+	return nil
+}
+
+func (w *worker) ProcessSession(ctx context.Context) (uuid.UUID, error) {
+	return w.LocalWorker.Session(ctx)
+}
+
+func (w *worker) Session(ctx context.Context) (uuid.UUID, error) {
+	if atomic.LoadInt64(&w.disabled) == 1 {
+		return uuid.UUID{}, xerrors.Errorf("worker disabled")
+	}
+
+	return w.LocalWorker.Session(ctx)
+}
+
+func (w *worker) Discover(ctx context.Context) (apitypes.OpenRPCDocument, error) {
+	return build.OpenRPCDiscoverJSON_Worker(), nil
+}
+
 var _ storiface.WorkerCalls = &worker{}

cmd/lotus-seal-worker/storage.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/docker/go-units"
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
@@ -46,6 +47,10 @@ var storageAttachCmd = &cli.Command{
 			Name:  "store",
 			Usage: "(for init) use path for long-term storage",
 		},
+		&cli.StringFlag{
+			Name:  "max-storage",
+			Usage: "(for init) limit storage space for sectors (expensive for very large paths!)",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		nodeApi, closer, err := lcli.GetWorkerAPI(cctx)
@@ -79,15 +84,24 @@ var storageAttachCmd = &cli.Command{
 				return err
 			}
 
+			var maxStor int64
+			if cctx.IsSet("max-storage") {
+				maxStor, err = units.RAMInBytes(cctx.String("max-storage"))
+				if err != nil {
+					return xerrors.Errorf("parsing max-storage: %w", err)
+				}
+			}
+
 			cfg := &stores.LocalStorageMeta{
-				ID:       stores.ID(uuid.New().String()),
-				Weight:   cctx.Uint64("weight"),
-				CanSeal:  cctx.Bool("seal"),
-				CanStore: cctx.Bool("store"),
+				ID:         stores.ID(uuid.New().String()),
+				Weight:     cctx.Uint64("weight"),
+				CanSeal:    cctx.Bool("seal"),
+				CanStore:   cctx.Bool("store"),
+				MaxStorage: uint64(maxStor),
 			}
 
 			if !(cfg.CanStore || cfg.CanSeal) {
-				return xerrors.Errorf("must specify at least one of --store of --seal")
+				return xerrors.Errorf("must specify at least one of --store or --seal")
 			}
 
 			b, err := json.MarshalIndent(cfg, "", "  ")

cmd/lotus-seal-worker/tasks.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"context"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/api"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
+)
+
+var tasksCmd = &cli.Command{
+	Name:  "tasks",
+	Usage: "Manage task processing",
+	Subcommands: []*cli.Command{
+		tasksEnableCmd,
+		tasksDisableCmd,
+	},
+}
+
+var allowSetting = map[sealtasks.TaskType]struct{}{
+	sealtasks.TTAddPiece:   {},
+	sealtasks.TTPreCommit1: {},
+	sealtasks.TTPreCommit2: {},
+	sealtasks.TTCommit2:    {},
+	sealtasks.TTUnseal:     {},
+}
+
+var settableStr = func() string {
+	var s []string
+	for _, tt := range ttList(allowSetting) {
+		s = append(s, tt.Short())
+	}
+	return strings.Join(s, "|")
+}()
+
+var tasksEnableCmd = &cli.Command{
+	Name:      "enable",
+	Usage:     "Enable a task type",
+	ArgsUsage: "[" + settableStr + "]",
+	Action:    taskAction(api.Worker.TaskEnable),
+}
+
+var tasksDisableCmd = &cli.Command{
+	Name:      "disable",
+	Usage:     "Disable a task type",
+	ArgsUsage: "[" + settableStr + "]",
+	Action:    taskAction(api.Worker.TaskDisable),
+}
+
+func taskAction(tf func(a api.Worker, ctx context.Context, tt sealtasks.TaskType) error) func(cctx *cli.Context) error {
+	return func(cctx *cli.Context) error {
+		if cctx.NArg() != 1 {
+			return xerrors.Errorf("expected 1 argument")
+		}
+
+		var tt sealtasks.TaskType
+		for taskType := range allowSetting {
+			if taskType.Short() == cctx.Args().First() {
+				tt = taskType
+				break
+			}
+		}
+
+		if tt == "" {
+			return xerrors.Errorf("unknown task type '%s'", cctx.Args().First())
+		}
+
+		api, closer, err := lcli.GetWorkerAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		return tf(api, ctx, tt)
+	}
+}

cmd/lotus-seed/genesis.go

@@ -9,6 +9,13 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/filecoin-project/go-state-types/network"
+
+	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/lotus/journal"
+	"github.com/filecoin-project/lotus/node/modules/testing"
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
@@ -32,6 +39,10 @@ var genesisCmd = &cli.Command{
 		genesisNewCmd,
 		genesisAddMinerCmd,
 		genesisAddMsigsCmd,
+		genesisSetVRKCmd,
+		genesisSetRemainderCmd,
+		genesisSetActorVersionCmd,
+		genesisCarCmd,
 	},
 }
 
@@ -48,6 +59,7 @@ var genesisNewCmd = &cli.Command{
 			return xerrors.New("seed genesis new [genesis.json]")
 		}
 		out := genesis.Template{
+			NetworkVersion:   build.NewestNetworkVersion,
 			Accounts:         []genesis.Actor{},
 			Miners:           []genesis.Miner{},
 			VerifregRootKey:  gen.DefaultVerifregRootkeyActor,
@@ -302,3 +314,267 @@ func parseMultisigCsv(csvf string) ([]GenAccountEntry, error) {
 
 	return entries, nil
 }
+
+var genesisSetVRKCmd = &cli.Command{
+	Name:  "set-vrk",
+	Usage: "Set the verified registry's root key",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "multisig",
+			Usage: "CSV file to parse the multisig that will be set as the root key",
+		},
+		&cli.StringFlag{
+			Name:  "account",
+			Usage: "pubkey address that will be set as the root key (must NOT be declared anywhere else, since it must be given ID 80)",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 1 {
+			return fmt.Errorf("must specify template file")
+		}
+
+		genf, err := homedir.Expand(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		var template genesis.Template
+		b, err := ioutil.ReadFile(genf)
+		if err != nil {
+			return xerrors.Errorf("read genesis template: %w", err)
+		}
+
+		if err := json.Unmarshal(b, &template); err != nil {
+			return xerrors.Errorf("unmarshal genesis template: %w", err)
+		}
+
+		if cctx.IsSet("account") {
+			addr, err := address.NewFromString(cctx.String("account"))
+			if err != nil {
+				return err
+			}
+
+			am := genesis.AccountMeta{Owner: addr}
+
+			template.VerifregRootKey = genesis.Actor{
+				Type:    genesis.TAccount,
+				Balance: big.Zero(),
+				Meta:    am.ActorMeta(),
+			}
+		} else if cctx.IsSet("multisig") {
+			csvf, err := homedir.Expand(cctx.String("multisig"))
+			if err != nil {
+				return err
+			}
+
+			entries, err := parseMultisigCsv(csvf)
+			if err != nil {
+				return xerrors.Errorf("parsing multisig csv file: %w", err)
+			}
+
+			if len(entries) == 0 {
+				return xerrors.Errorf("no msig entries in csv file: %w", err)
+			}
+
+			e := entries[0]
+			if len(e.Addresses) != e.N {
+				return fmt.Errorf("entry had mismatch between 'N' and number of addresses")
+			}
+
+			msig := &genesis.MultisigMeta{
+				Signers:         e.Addresses,
+				Threshold:       e.M,
+				VestingDuration: monthsToBlocks(e.VestingMonths),
+				VestingStart:    0,
+			}
+
+			act := genesis.Actor{
+				Type:    genesis.TMultisig,
+				Balance: abi.TokenAmount(e.Amount),
+				Meta:    msig.ActorMeta(),
+			}
+
+			template.VerifregRootKey = act
+		} else {
+			return xerrors.Errorf("must include either --account or --multisig flag")
+		}
+
+		b, err = json.MarshalIndent(&template, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		if err := ioutil.WriteFile(genf, b, 0644); err != nil {
+			return err
+		}
+		return nil
+	},
+}
+
+var genesisSetRemainderCmd = &cli.Command{
+	Name:  "set-remainder",
+	Usage: "Set the remainder actor",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "multisig",
+			Usage: "CSV file to parse the multisig that will be set as the remainder actor",
+		},
+		&cli.StringFlag{
+			Name:  "account",
+			Usage: "pubkey address that will be set as the remainder key (must NOT be declared anywhere else, since it must be given ID 90)",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 1 {
+			return fmt.Errorf("must specify template file")
+		}
+
+		genf, err := homedir.Expand(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		var template genesis.Template
+		b, err := ioutil.ReadFile(genf)
+		if err != nil {
+			return xerrors.Errorf("read genesis template: %w", err)
+		}
+
+		if err := json.Unmarshal(b, &template); err != nil {
+			return xerrors.Errorf("unmarshal genesis template: %w", err)
+		}
+
+		if cctx.IsSet("account") {
+			addr, err := address.NewFromString(cctx.String("account"))
+			if err != nil {
+				return err
+			}
+
+			am := genesis.AccountMeta{Owner: addr}
+
+			template.RemainderAccount = genesis.Actor{
+				Type:    genesis.TAccount,
+				Balance: big.Zero(),
+				Meta:    am.ActorMeta(),
+			}
+		} else if cctx.IsSet("multisig") {
+			csvf, err := homedir.Expand(cctx.String("multisig"))
+			if err != nil {
+				return err
+			}
+
+			entries, err := parseMultisigCsv(csvf)
+			if err != nil {
+				return xerrors.Errorf("parsing multisig csv file: %w", err)
+			}
+
+			if len(entries) == 0 {
+				return xerrors.Errorf("no msig entries in csv file: %w", err)
+			}
+
+			e := entries[0]
+			if len(e.Addresses) != e.N {
+				return fmt.Errorf("entry had mismatch between 'N' and number of addresses")
+			}
+
+			msig := &genesis.MultisigMeta{
+				Signers:         e.Addresses,
+				Threshold:       e.M,
+				VestingDuration: monthsToBlocks(e.VestingMonths),
+				VestingStart:    0,
+			}
+
+			act := genesis.Actor{
+				Type:    genesis.TMultisig,
+				Balance: abi.TokenAmount(e.Amount),
+				Meta:    msig.ActorMeta(),
+			}
+
+			template.RemainderAccount = act
+		} else {
+			return xerrors.Errorf("must include either --account or --multisig flag")
+		}
+
+		b, err = json.MarshalIndent(&template, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		if err := ioutil.WriteFile(genf, b, 0644); err != nil {
+			return err
+		}
+		return nil
+	},
+}
+
+var genesisSetActorVersionCmd = &cli.Command{
+	Name:      "set-network-version",
+	Usage:     "Set the version that this network will start from",
+	ArgsUsage: "<genesisFile> <actorVersion>",
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 2 {
+			return fmt.Errorf("must specify genesis file and network version (e.g. '0'")
+		}
+
+		genf, err := homedir.Expand(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		var template genesis.Template
+		b, err := ioutil.ReadFile(genf)
+		if err != nil {
+			return xerrors.Errorf("read genesis template: %w", err)
+		}
+
+		if err := json.Unmarshal(b, &template); err != nil {
+			return xerrors.Errorf("unmarshal genesis template: %w", err)
+		}
+
+		nv, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64)
+		if err != nil {
+			return xerrors.Errorf("parsing network version: %w", err)
+		}
+
+		if nv > uint64(build.NewestNetworkVersion) {
+			return xerrors.Errorf("invalid network version: %d", nv)
+		}
+
+		template.NetworkVersion = network.Version(nv)
+
+		b, err = json.MarshalIndent(&template, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		if err := ioutil.WriteFile(genf, b, 0644); err != nil {
+			return err
+		}
+		return nil
+	},
+}
+
+var genesisCarCmd = &cli.Command{
+	Name:        "car",
+	Description: "write genesis car file",
+	ArgsUsage:   "genesis template `FILE`",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "out",
+			Aliases: []string{"o"},
+			Value:   "genesis.car",
+			Usage:   "write output to `FILE`",
+		},
+	},
+	Action: func(c *cli.Context) error {
+		if c.Args().Len() != 1 {
+			return xerrors.Errorf("Please specify a genesis template. (i.e, the one created with `genesis new`)")
+		}
+		ofile := c.String("out")
+		jrnl := journal.NilJournal()
+		bstor := blockstore.WrapIDStore(blockstore.NewMemorySync())
+		sbldr := vm.Syscalls(ffiwrapper.ProofVerifier)
+		_, err := testing.MakeGenesis(ofile, c.Args().First())(bstor, sbldr, jrnl)()
+		return err
+	},
+}

cmd/lotus-seed/main.go

@@ -7,9 +7,9 @@ import (
 	"io/ioutil"
 	"os"
 
-	"github.com/docker/go-units"
-	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/go-state-types/network"
 
+	"github.com/docker/go-units"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
@@ -19,6 +19,7 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/cmd/lotus-seed/seed"
 	"github.com/filecoin-project/lotus/genesis"
@@ -93,6 +94,10 @@ var preSealCmd = &cli.Command{
 			Name:  "fake-sectors",
 			Value: false,
 		},
+		&cli.IntFlag{
+			Name:  "network-version",
+			Usage: "specify network version",
+		},
 	},
 	Action: func(c *cli.Context) error {
 		sdir := c.String("sector-dir")
@@ -128,12 +133,17 @@ var preSealCmd = &cli.Command{
 		}
 		sectorSize := abi.SectorSize(sectorSizeInt)
 
-		rp, err := ffiwrapper.SealProofTypeFromSectorSize(sectorSize)
+		nv := build.NewestNetworkVersion
+		if c.IsSet("network-version") {
+			nv = network.Version(c.Uint64("network-version"))
+		}
+
+		spt, err := miner.SealProofTypeFromSectorSize(sectorSize, nv)
 		if err != nil {
 			return err
 		}
 
-		gm, key, err := seed.PreSeal(maddr, rp, abi.SectorNumber(c.Uint64("sector-offset")), c.Int("num-sectors"), sbroot, []byte(c.String("ticket-preimage")), k, c.Bool("fake-sectors"))
+		gm, key, err := seed.PreSeal(maddr, spt, abi.SectorNumber(c.Uint64("sector-offset")), c.Int("num-sectors"), sbroot, []byte(c.String("ticket-preimage")), k, c.Bool("fake-sectors"))
 		if err != nil {
 			return err
 		}

cmd/lotus-seed/seed/seed.go

@@ -19,9 +19,10 @@ import (
 
 	ffi "github.com/filecoin-project/filecoin-ffi"
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-commp-utils/zerocomm"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
-	"github.com/filecoin-project/lotus/extern/sector-storage/zerocomm"
+	"github.com/filecoin-project/specs-storage/storage"
 
 	market2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/market"
 
@@ -42,10 +43,6 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect
 		return nil, nil, err
 	}
 
-	cfg := &ffiwrapper.Config{
-		SealProofType: spt,
-	}
-
 	if err := os.MkdirAll(sbroot, 0775); err != nil { //nolint:gosec
 		return nil, nil, err
 	}
@@ -56,7 +53,7 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect
 		Root: sbroot,
 	}
 
-	sb, err := ffiwrapper.New(sbfs, cfg)
+	sb, err := ffiwrapper.New(sbfs)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -69,16 +66,17 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect
 	var sealedSectors []*genesis.PreSeal
 	for i := 0; i < sectors; i++ {
 		sid := abi.SectorID{Miner: abi.ActorID(mid), Number: next}
+		ref := storage.SectorRef{ID: sid, ProofType: spt}
 		next++
 
 		var preseal *genesis.PreSeal
 		if !fakeSectors {
-			preseal, err = presealSector(sb, sbfs, sid, spt, ssize, preimage)
+			preseal, err = presealSector(sb, sbfs, ref, ssize, preimage)
 			if err != nil {
 				return nil, nil, err
 			}
 		} else {
-			preseal, err = presealSectorFake(sbfs, sid, spt, ssize)
+			preseal, err = presealSectorFake(sbfs, ref, ssize)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -148,7 +146,7 @@ func PreSeal(maddr address.Address, spt abi.RegisteredSealProof, offset abi.Sect
 	return miner, &minerAddr.KeyInfo, nil
 }
 
-func presealSector(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, sid abi.SectorID, spt abi.RegisteredSealProof, ssize abi.SectorSize, preimage []byte) (*genesis.PreSeal, error) {
+func presealSector(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, sid storage.SectorRef, ssize abi.SectorSize, preimage []byte) (*genesis.PreSeal, error) {
 	pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(ssize).Unpadded(), rand.Reader)
 	if err != nil {
 		return nil, err
@@ -182,12 +180,12 @@ func presealSector(sb *ffiwrapper.Sealer, sbfs *basicfs.Provider, sid abi.Sector
 	return &genesis.PreSeal{
 		CommR:     cids.Sealed,
 		CommD:     cids.Unsealed,
-		SectorID:  sid.Number,
-		ProofType: spt,
+		SectorID:  sid.ID.Number,
+		ProofType: sid.ProofType,
 	}, nil
 }
 
-func presealSectorFake(sbfs *basicfs.Provider, sid abi.SectorID, spt abi.RegisteredSealProof, ssize abi.SectorSize) (*genesis.PreSeal, error) {
+func presealSectorFake(sbfs *basicfs.Provider, sid storage.SectorRef, ssize abi.SectorSize) (*genesis.PreSeal, error) {
 	paths, done, err := sbfs.AcquireSector(context.TODO(), sid, 0, storiface.FTSealed|storiface.FTCache, storiface.PathSealing)
 	if err != nil {
 		return nil, xerrors.Errorf("acquire unsealed sector: %w", err)
@@ -198,7 +196,7 @@ func presealSectorFake(sbfs *basicfs.Provider, sid abi.SectorID, spt abi.Registe
 		return nil, xerrors.Errorf("mkdir cache: %w", err)
 	}
 
-	commr, err := ffi.FauxRep(spt, paths.Cache, paths.Sealed)
+	commr, err := ffi.FauxRep(sid.ProofType, paths.Cache, paths.Sealed)
 	if err != nil {
 		return nil, xerrors.Errorf("fauxrep: %w", err)
 	}
@@ -206,13 +204,13 @@ func presealSectorFake(sbfs *basicfs.Provider, sid abi.SectorID, spt abi.Registe
 	return &genesis.PreSeal{
 		CommR:     commr,
 		CommD:     zerocomm.ZeroPieceCommitment(abi.PaddedPieceSize(ssize).Unpadded()),
-		SectorID:  sid.Number,
-		ProofType: spt,
+		SectorID:  sid.ID.Number,
+		ProofType: sid.ProofType,
 	}, nil
 }
 
-func cleanupUnsealed(sbfs *basicfs.Provider, sid abi.SectorID) error {
-	paths, done, err := sbfs.AcquireSector(context.TODO(), sid, storiface.FTUnsealed, storiface.FTNone, storiface.PathSealing)
+func cleanupUnsealed(sbfs *basicfs.Provider, ref storage.SectorRef) error {
+	paths, done, err := sbfs.AcquireSector(context.TODO(), ref, storiface.FTUnsealed, storiface.FTNone, storiface.PathSealing)
 	if err != nil {
 		return err
 	}

cmd/lotus-shed/actor.go

@@ -0,0 +1,740 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/lotus/api"
+
+	miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner"
+
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/lib/tablewriter"
+)
+
+var actorCmd = &cli.Command{
+	Name:  "actor",
+	Usage: "manipulate the miner actor",
+	Subcommands: []*cli.Command{
+		actorWithdrawCmd,
+		actorSetOwnerCmd,
+		actorControl,
+		actorProposeChangeWorker,
+		actorConfirmChangeWorker,
+	},
+}
+
+var actorWithdrawCmd = &cli.Command{
+	Name:      "withdraw",
+	Usage:     "withdraw available balance",
+	ArgsUsage: "[amount (FIL)]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "actor",
+			Usage: "specify the address of miner actor",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		var maddr address.Address
+		if act := cctx.String("actor"); act != "" {
+			var err error
+			maddr, err = address.NewFromString(act)
+			if err != nil {
+				return fmt.Errorf("parsing address %s: %w", act, err)
+			}
+		}
+
+		nodeAPI, acloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer acloser()
+
+		ctx := lcli.ReqContext(cctx)
+
+		if maddr.Empty() {
+			minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
+			if err != nil {
+				return err
+			}
+			defer closer()
+
+			maddr, err = minerAPI.ActorAddress(ctx)
+			if err != nil {
+				return err
+			}
+		}
+
+		mi, err := nodeAPI.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		available, err := nodeAPI.StateMinerAvailableBalance(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		amount := available
+		if cctx.Args().Present() {
+			f, err := types.ParseFIL(cctx.Args().First())
+			if err != nil {
+				return xerrors.Errorf("parsing 'amount' argument: %w", err)
+			}
+
+			amount = abi.TokenAmount(f)
+
+			if amount.GreaterThan(available) {
+				return xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", amount, available)
+			}
+		}
+
+		params, err := actors.SerializeParams(&miner2.WithdrawBalanceParams{
+			AmountRequested: amount, // Default to attempting to withdraw all the extra funds in the miner actor
+		})
+		if err != nil {
+			return err
+		}
+
+		smsg, err := nodeAPI.MpoolPushMessage(ctx, &types.Message{
+			To:     maddr,
+			From:   mi.Owner,
+			Value:  types.NewInt(0),
+			Method: miner.Methods.WithdrawBalance,
+			Params: params,
+		}, &api.MessageSendSpec{MaxFee: abi.TokenAmount(types.MustParseFIL("0.1"))})
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Requested rewards withdrawal in message %s\n", smsg.Cid())
+
+		return nil
+	},
+}
+
+var actorSetOwnerCmd = &cli.Command{
+	Name:      "set-owner",
+	Usage:     "Set owner address (this command should be invoked twice, first with the old owner as the senderAddress, and then with the new owner)",
+	ArgsUsage: "[newOwnerAddress senderAddress]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "actor",
+			Usage: "specify the address of miner actor",
+		},
+		&cli.BoolFlag{
+			Name:  "really-do-it",
+			Usage: "Actually send transaction performing the action",
+			Value: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Bool("really-do-it") {
+			fmt.Println("Pass --really-do-it to actually execute this action")
+			return nil
+		}
+
+		if cctx.NArg() != 2 {
+			return fmt.Errorf("must pass new owner address and sender address")
+		}
+
+		var maddr address.Address
+		if act := cctx.String("actor"); act != "" {
+			var err error
+			maddr, err = address.NewFromString(act)
+			if err != nil {
+				return fmt.Errorf("parsing address %s: %w", act, err)
+			}
+		}
+
+		nodeAPI, acloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer acloser()
+
+		ctx := lcli.ReqContext(cctx)
+
+		na, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		newAddrId, err := nodeAPI.StateLookupID(ctx, na, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		fa, err := address.NewFromString(cctx.Args().Get(1))
+		if err != nil {
+			return err
+		}
+
+		fromAddrId, err := nodeAPI.StateLookupID(ctx, fa, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		if maddr.Empty() {
+			minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
+			if err != nil {
+				return err
+			}
+			defer closer()
+
+			maddr, err = minerAPI.ActorAddress(ctx)
+			if err != nil {
+				return err
+			}
+		}
+
+		mi, err := nodeAPI.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		if fromAddrId != mi.Owner && fromAddrId != newAddrId {
+			return xerrors.New("from address must either be the old owner or the new owner")
+		}
+
+		sp, err := actors.SerializeParams(&newAddrId)
+		if err != nil {
+			return xerrors.Errorf("serializing params: %w", err)
+		}
+
+		smsg, err := nodeAPI.MpoolPushMessage(ctx, &types.Message{
+			From:   fromAddrId,
+			To:     maddr,
+			Method: miner.Methods.ChangeOwnerAddress,
+			Value:  big.Zero(),
+			Params: sp,
+		}, nil)
+		if err != nil {
+			return xerrors.Errorf("mpool push: %w", err)
+		}
+
+		fmt.Println("Message CID:", smsg.Cid())
+
+		// wait for it to get mined into a block
+		wait, err := nodeAPI.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence)
+		if err != nil {
+			return err
+		}
+
+		// check it executed successfully
+		if wait.Receipt.ExitCode != 0 {
+			fmt.Println("owner change failed!")
+			return err
+		}
+
+		fmt.Println("message succeeded!")
+
+		return nil
+	},
+}
+
+var actorControl = &cli.Command{
+	Name:  "control",
+	Usage: "Manage control addresses",
+	Subcommands: []*cli.Command{
+		actorControlList,
+		actorControlSet,
+	},
+}
+
+var actorControlList = &cli.Command{
+	Name:  "list",
+	Usage: "Get currently set control addresses",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "actor",
+			Usage: "specify the address of miner actor",
+		},
+		&cli.BoolFlag{
+			Name: "verbose",
+		},
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		var maddr address.Address
+		if act := cctx.String("actor"); act != "" {
+			var err error
+			maddr, err = address.NewFromString(act)
+			if err != nil {
+				return fmt.Errorf("parsing address %s: %w", act, err)
+			}
+		}
+
+		nodeAPI, acloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer acloser()
+
+		ctx := lcli.ReqContext(cctx)
+
+		if maddr.Empty() {
+			minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
+			if err != nil {
+				return err
+			}
+			defer closer()
+
+			maddr, err = minerAPI.ActorAddress(ctx)
+			if err != nil {
+				return err
+			}
+		}
+
+		mi, err := nodeAPI.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		tw := tablewriter.New(
+			tablewriter.Col("name"),
+			tablewriter.Col("ID"),
+			tablewriter.Col("key"),
+			tablewriter.Col("balance"),
+		)
+
+		printKey := func(name string, a address.Address) {
+			b, err := nodeAPI.WalletBalance(ctx, a)
+			if err != nil {
+				fmt.Printf("%s\t%s: error getting balance: %s\n", name, a, err)
+				return
+			}
+
+			k, err := nodeAPI.StateAccountKey(ctx, a, types.EmptyTSK)
+			if err != nil {
+				fmt.Printf("%s\t%s: error getting account key: %s\n", name, a, err)
+				return
+			}
+
+			kstr := k.String()
+			if !cctx.Bool("verbose") {
+				kstr = kstr[:9] + "..."
+			}
+
+			bstr := types.FIL(b).String()
+			switch {
+			case b.LessThan(types.FromFil(10)):
+				bstr = color.RedString(bstr)
+			case b.LessThan(types.FromFil(50)):
+				bstr = color.YellowString(bstr)
+			default:
+				bstr = color.GreenString(bstr)
+			}
+
+			tw.Write(map[string]interface{}{
+				"name":    name,
+				"ID":      a,
+				"key":     kstr,
+				"balance": bstr,
+			})
+		}
+
+		printKey("owner", mi.Owner)
+		printKey("worker", mi.Worker)
+		for i, ca := range mi.ControlAddresses {
+			printKey(fmt.Sprintf("control-%d", i), ca)
+		}
+
+		return tw.Flush(os.Stdout)
+	},
+}
+
+var actorControlSet = &cli.Command{
+	Name:      "set",
+	Usage:     "Set control address(-es)",
+	ArgsUsage: "[...address]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "actor",
+			Usage: "specify the address of miner actor",
+		},
+		&cli.BoolFlag{
+			Name:  "really-do-it",
+			Usage: "Actually send transaction performing the action",
+			Value: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Bool("really-do-it") {
+			fmt.Println("Pass --really-do-it to actually execute this action")
+			return nil
+		}
+
+		var maddr address.Address
+		if act := cctx.String("actor"); act != "" {
+			var err error
+			maddr, err = address.NewFromString(act)
+			if err != nil {
+				return fmt.Errorf("parsing address %s: %w", act, err)
+			}
+		}
+
+		nodeAPI, acloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer acloser()
+
+		ctx := lcli.ReqContext(cctx)
+
+		if maddr.Empty() {
+			minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
+			if err != nil {
+				return err
+			}
+			defer closer()
+
+			maddr, err = minerAPI.ActorAddress(ctx)
+			if err != nil {
+				return err
+			}
+		}
+
+		mi, err := nodeAPI.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		del := map[address.Address]struct{}{}
+		existing := map[address.Address]struct{}{}
+		for _, controlAddress := range mi.ControlAddresses {
+			ka, err := nodeAPI.StateAccountKey(ctx, controlAddress, types.EmptyTSK)
+			if err != nil {
+				return err
+			}
+
+			del[ka] = struct{}{}
+			existing[ka] = struct{}{}
+		}
+
+		var toSet []address.Address
+
+		for i, as := range cctx.Args().Slice() {
+			a, err := address.NewFromString(as)
+			if err != nil {
+				return xerrors.Errorf("parsing address %d: %w", i, err)
+			}
+
+			ka, err := nodeAPI.StateAccountKey(ctx, a, types.EmptyTSK)
+			if err != nil {
+				return err
+			}
+
+			// make sure the address exists on chain
+			_, err = nodeAPI.StateLookupID(ctx, ka, types.EmptyTSK)
+			if err != nil {
+				return xerrors.Errorf("looking up %s: %w", ka, err)
+			}
+
+			delete(del, ka)
+			toSet = append(toSet, ka)
+		}
+
+		for a := range del {
+			fmt.Println("Remove", a)
+		}
+		for _, a := range toSet {
+			if _, exists := existing[a]; !exists {
+				fmt.Println("Add", a)
+			}
+		}
+
+		cwp := &miner2.ChangeWorkerAddressParams{
+			NewWorker:       mi.Worker,
+			NewControlAddrs: toSet,
+		}
+
+		sp, err := actors.SerializeParams(cwp)
+		if err != nil {
+			return xerrors.Errorf("serializing params: %w", err)
+		}
+
+		smsg, err := nodeAPI.MpoolPushMessage(ctx, &types.Message{
+			From:   mi.Owner,
+			To:     maddr,
+			Method: miner.Methods.ChangeWorkerAddress,
+
+			Value:  big.Zero(),
+			Params: sp,
+		}, nil)
+		if err != nil {
+			return xerrors.Errorf("mpool push: %w", err)
+		}
+
+		fmt.Println("Message CID:", smsg.Cid())
+
+		return nil
+	},
+}
+
+var actorProposeChangeWorker = &cli.Command{
+	Name:      "propose-change-worker",
+	Usage:     "Propose a worker address change",
+	ArgsUsage: "[address]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "actor",
+			Usage: "specify the address of miner actor",
+		},
+		&cli.BoolFlag{
+			Name:  "really-do-it",
+			Usage: "Actually send transaction performing the action",
+			Value: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass address of new worker address")
+		}
+
+		if !cctx.Bool("really-do-it") {
+			fmt.Fprintln(cctx.App.Writer, "Pass --really-do-it to actually execute this action")
+			return nil
+		}
+
+		var maddr address.Address
+		if act := cctx.String("actor"); act != "" {
+			var err error
+			maddr, err = address.NewFromString(act)
+			if err != nil {
+				return fmt.Errorf("parsing address %s: %w", act, err)
+			}
+		}
+
+		nodeAPI, acloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer acloser()
+
+		ctx := lcli.ReqContext(cctx)
+
+		na, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		newAddr, err := nodeAPI.StateLookupID(ctx, na, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		if maddr.Empty() {
+			minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
+			if err != nil {
+				return err
+			}
+			defer closer()
+
+			maddr, err = minerAPI.ActorAddress(ctx)
+			if err != nil {
+				return err
+			}
+		}
+
+		mi, err := nodeAPI.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		if mi.NewWorker.Empty() {
+			if mi.Worker == newAddr {
+				return fmt.Errorf("worker address already set to %s", na)
+			}
+		} else {
+			if mi.NewWorker == newAddr {
+				return fmt.Errorf("change to worker address %s already pending", na)
+			}
+		}
+
+		cwp := &miner2.ChangeWorkerAddressParams{
+			NewWorker:       newAddr,
+			NewControlAddrs: mi.ControlAddresses,
+		}
+
+		sp, err := actors.SerializeParams(cwp)
+		if err != nil {
+			return xerrors.Errorf("serializing params: %w", err)
+		}
+
+		smsg, err := nodeAPI.MpoolPushMessage(ctx, &types.Message{
+			From:   mi.Owner,
+			To:     maddr,
+			Method: miner.Methods.ChangeWorkerAddress,
+			Value:  big.Zero(),
+			Params: sp,
+		}, nil)
+		if err != nil {
+			return xerrors.Errorf("mpool push: %w", err)
+		}
+
+		fmt.Fprintln(cctx.App.Writer, "Propose Message CID:", smsg.Cid())
+
+		// wait for it to get mined into a block
+		wait, err := nodeAPI.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence)
+		if err != nil {
+			return err
+		}
+
+		// check it executed successfully
+		if wait.Receipt.ExitCode != 0 {
+			fmt.Fprintln(cctx.App.Writer, "Propose worker change failed!")
+			return err
+		}
+
+		mi, err = nodeAPI.StateMinerInfo(ctx, maddr, wait.TipSet)
+		if err != nil {
+			return err
+		}
+		if mi.NewWorker != newAddr {
+			return fmt.Errorf("Proposed worker address change not reflected on chain: expected '%s', found '%s'", na, mi.NewWorker)
+		}
+
+		fmt.Fprintf(cctx.App.Writer, "Worker key change to %s successfully proposed.\n", na)
+		fmt.Fprintf(cctx.App.Writer, "Call 'confirm-change-worker' at or after height %d to complete.\n", mi.WorkerChangeEpoch)
+
+		return nil
+	},
+}
+
+var actorConfirmChangeWorker = &cli.Command{
+	Name:      "confirm-change-worker",
+	Usage:     "Confirm a worker address change",
+	ArgsUsage: "[address]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "actor",
+			Usage: "specify the address of miner actor",
+		},
+		&cli.BoolFlag{
+			Name:  "really-do-it",
+			Usage: "Actually send transaction performing the action",
+			Value: false,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass address of new worker address")
+		}
+
+		if !cctx.Bool("really-do-it") {
+			fmt.Fprintln(cctx.App.Writer, "Pass --really-do-it to actually execute this action")
+			return nil
+		}
+
+		var maddr address.Address
+		if act := cctx.String("actor"); act != "" {
+			var err error
+			maddr, err = address.NewFromString(act)
+			if err != nil {
+				return fmt.Errorf("parsing address %s: %w", act, err)
+			}
+		}
+
+		nodeAPI, acloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer acloser()
+
+		ctx := lcli.ReqContext(cctx)
+
+		na, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		newAddr, err := nodeAPI.StateLookupID(ctx, na, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		if maddr.Empty() {
+			minerAPI, closer, err := lcli.GetStorageMinerAPI(cctx)
+			if err != nil {
+				return err
+			}
+			defer closer()
+
+			maddr, err = minerAPI.ActorAddress(ctx)
+			if err != nil {
+				return err
+			}
+		}
+
+		mi, err := nodeAPI.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		if mi.NewWorker.Empty() {
+			return xerrors.Errorf("no worker key change proposed")
+		} else if mi.NewWorker != newAddr {
+			return xerrors.Errorf("worker key %s does not match current worker key proposal %s", newAddr, mi.NewWorker)
+		}
+
+		if head, err := nodeAPI.ChainHead(ctx); err != nil {
+			return xerrors.Errorf("failed to get the chain head: %w", err)
+		} else if head.Height() < mi.WorkerChangeEpoch {
+			return xerrors.Errorf("worker key change cannot be confirmed until %d, current height is %d", mi.WorkerChangeEpoch, head.Height())
+		}
+
+		smsg, err := nodeAPI.MpoolPushMessage(ctx, &types.Message{
+			From:   mi.Owner,
+			To:     maddr,
+			Method: miner.Methods.ConfirmUpdateWorkerKey,
+			Value:  big.Zero(),
+		}, nil)
+		if err != nil {
+			return xerrors.Errorf("mpool push: %w", err)
+		}
+
+		fmt.Fprintln(cctx.App.Writer, "Confirm Message CID:", smsg.Cid())
+
+		// wait for it to get mined into a block
+		wait, err := nodeAPI.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence)
+		if err != nil {
+			return err
+		}
+
+		// check it executed successfully
+		if wait.Receipt.ExitCode != 0 {
+			fmt.Fprintln(cctx.App.Writer, "Worker change failed!")
+			return err
+		}
+
+		mi, err = nodeAPI.StateMinerInfo(ctx, maddr, wait.TipSet)
+		if err != nil {
+			return err
+		}
+		if mi.Worker != newAddr {
+			return fmt.Errorf("Confirmed worker address change not reflected on chain: expected '%s', found '%s'", newAddr, mi.Worker)
+		}
+
+		return nil
+	},
+}

cmd/lotus-shed/balances.go

@@ -2,14 +2,25 @@ package main
 
 import (
 	"context"
+	"encoding/csv"
+	"encoding/json"
 	"fmt"
+	"io"
+	"os"
+	"runtime"
 	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/filecoin-project/lotus/build"
 
 	"github.com/filecoin-project/lotus/chain/gen/genesis"
 
 	_init "github.com/filecoin-project/lotus/chain/actors/builtin/init"
 
 	"github.com/docker/go-units"
+
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/multisig"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
@@ -24,6 +35,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
+
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/state"
@@ -33,7 +45,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/vm"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -58,8 +69,321 @@ var auditsCmd = &cli.Command{
 	Description: "a collection of utilities for auditing the filecoin chain",
 	Subcommands: []*cli.Command{
 		chainBalanceCmd,
+		chainBalanceSanityCheckCmd,
 		chainBalanceStateCmd,
 		chainPledgeCmd,
+		fillBalancesCmd,
+		duplicatedMessagesCmd,
+	},
+}
+
+var duplicatedMessagesCmd = &cli.Command{
+	Name:  "duplicate-messages",
+	Usage: "Check for duplicate messages included in a tipset.",
+	UsageText: `Check for duplicate messages included in a tipset.
+
+Due to Filecoin's expected consensus, a tipset may include the same message multiple times in
+different blocks. The message will only be executed once.
+
+This command will find such duplicate messages and print them to standard out as newline-delimited
+JSON. Status messages in the form of "H: $HEIGHT ($PROGRESS%)" will be printed to standard error for
+every day of chain processed.
+`,
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Name:        "parallel",
+			Usage:       "the number of parallel threads for block processing",
+			DefaultText: "half the number of cores",
+		},
+		&cli.IntFlag{
+			Name:        "start",
+			Usage:       "the first epoch to check",
+			DefaultText: "genesis",
+		},
+		&cli.IntFlag{
+			Name:        "end",
+			Usage:       "the last epoch to check",
+			DefaultText: "the current head",
+		},
+		&cli.IntSliceFlag{
+			Name:        "method",
+			Usage:       "filter results by method number",
+			DefaultText: "all methods",
+		},
+		&cli.StringSliceFlag{
+			Name:        "include-to",
+			Usage:       "include only messages to the given address (does not perform address resolution)",
+			DefaultText: "all recipients",
+		},
+		&cli.StringSliceFlag{
+			Name:        "include-from",
+			Usage:       "include only messages from the given address (does not perform address resolution)",
+			DefaultText: "all senders",
+		},
+		&cli.StringSliceFlag{
+			Name:  "exclude-to",
+			Usage: "exclude messages to the given address (does not perform address resolution)",
+		},
+		&cli.StringSliceFlag{
+			Name:  "exclude-from",
+			Usage: "exclude messages from the given address (does not perform address resolution)",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		var head *types.TipSet
+		if cctx.IsSet("end") {
+			epoch := abi.ChainEpoch(cctx.Int("end"))
+			head, err = api.ChainGetTipSetByHeight(ctx, epoch, types.EmptyTSK)
+		} else {
+			head, err = api.ChainHead(ctx)
+		}
+		if err != nil {
+			return err
+		}
+
+		var printLk sync.Mutex
+
+		threads := runtime.NumCPU() / 2
+		if cctx.IsSet("parallel") {
+			threads = cctx.Int("int")
+			if threads <= 0 {
+				return fmt.Errorf("parallelism needs to be at least 1")
+			}
+		} else if threads == 0 {
+			threads = 1 // if we have one core, but who are we kidding...
+		}
+
+		throttle := make(chan struct{}, threads)
+
+		methods := map[abi.MethodNum]bool{}
+		for _, m := range cctx.IntSlice("method") {
+			if m < 0 {
+				return fmt.Errorf("expected method numbers to be non-negative")
+			}
+			methods[abi.MethodNum(m)] = true
+		}
+
+		addressSet := func(flag string) (map[address.Address]bool, error) {
+			if !cctx.IsSet(flag) {
+				return nil, nil
+			}
+			addrs := cctx.StringSlice(flag)
+			set := make(map[address.Address]bool, len(addrs))
+			for _, addrStr := range addrs {
+				addr, err := address.NewFromString(addrStr)
+				if err != nil {
+					return nil, fmt.Errorf("failed to parse address %s: %w", addrStr, err)
+				}
+				set[addr] = true
+			}
+			return set, nil
+		}
+
+		onlyFrom, err := addressSet("include-from")
+		if err != nil {
+			return err
+		}
+		onlyTo, err := addressSet("include-to")
+		if err != nil {
+			return err
+		}
+		excludeFrom, err := addressSet("exclude-from")
+		if err != nil {
+			return err
+		}
+		excludeTo, err := addressSet("exclude-to")
+		if err != nil {
+			return err
+		}
+
+		target := abi.ChainEpoch(cctx.Int("start"))
+		if target < 0 || target > head.Height() {
+			return fmt.Errorf("start height must be greater than 0 and less than the end height")
+		}
+		totalEpochs := head.Height() - target
+
+		for target <= head.Height() {
+			select {
+			case throttle <- struct{}{}:
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+
+			go func(ts *types.TipSet) {
+				defer func() {
+					<-throttle
+				}()
+
+				type addrNonce struct {
+					s address.Address
+					n uint64
+				}
+				anonce := func(m *types.Message) addrNonce {
+					return addrNonce{
+						s: m.From,
+						n: m.Nonce,
+					}
+				}
+
+				msgs := map[addrNonce]map[cid.Cid]*types.Message{}
+
+				processMessage := func(c cid.Cid, m *types.Message) {
+					// Filter
+					if len(methods) > 0 && !methods[m.Method] {
+						return
+					}
+					if len(onlyFrom) > 0 && !onlyFrom[m.From] {
+						return
+					}
+					if len(onlyTo) > 0 && !onlyTo[m.To] {
+						return
+					}
+					if excludeFrom[m.From] || excludeTo[m.To] {
+						return
+					}
+
+					// Record
+					msgSet, ok := msgs[anonce(m)]
+					if !ok {
+						msgSet = make(map[cid.Cid]*types.Message, 1)
+						msgs[anonce(m)] = msgSet
+					}
+					msgSet[c] = m
+				}
+
+				encoder := json.NewEncoder(os.Stdout)
+
+				for _, bh := range ts.Blocks() {
+					bms, err := api.ChainGetBlockMessages(ctx, bh.Cid())
+					if err != nil {
+						fmt.Fprintln(os.Stderr, "ERROR: ", err)
+						return
+					}
+
+					for i, m := range bms.BlsMessages {
+						processMessage(bms.Cids[i], m)
+					}
+
+					for i, m := range bms.SecpkMessages {
+						processMessage(bms.Cids[len(bms.BlsMessages)+i], &m.Message)
+					}
+				}
+				for _, ms := range msgs {
+					if len(ms) == 1 {
+						continue
+					}
+					type Msg struct {
+						Cid    string
+						Value  string
+						Method uint64
+					}
+					grouped := map[string][]Msg{}
+					for c, m := range ms {
+						addr := m.To.String()
+						grouped[addr] = append(grouped[addr], Msg{
+							Cid:    c.String(),
+							Value:  types.FIL(m.Value).String(),
+							Method: uint64(m.Method),
+						})
+					}
+					printLk.Lock()
+					err := encoder.Encode(grouped)
+					if err != nil {
+						fmt.Fprintln(os.Stderr, "ERROR: ", err)
+					}
+					printLk.Unlock()
+				}
+			}(head)
+
+			if head.Parents().IsEmpty() {
+				break
+			}
+
+			head, err = api.ChainGetTipSet(ctx, head.Parents())
+			if err != nil {
+				return err
+			}
+
+			if head.Height()%2880 == 0 {
+				printLk.Lock()
+				fmt.Fprintf(os.Stderr, "H: %s (%d%%)\n", head.Height(), (100*(head.Height()-target))/totalEpochs)
+				printLk.Unlock()
+			}
+		}
+
+		for i := 0; i < threads; i++ {
+			select {
+			case throttle <- struct{}{}:
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+
+		}
+
+		printLk.Lock()
+		fmt.Fprintf(os.Stderr, "H: %s (100%%)\n", head.Height())
+		printLk.Unlock()
+
+		return nil
+	},
+}
+
+var chainBalanceSanityCheckCmd = &cli.Command{
+	Name:        "chain-balance-sanity",
+	Description: "Confirms that the total balance of every actor in state is still 2 billion",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "tipset",
+			Usage: "specify tipset to start from",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		ts, err := lcli.LoadTipSet(ctx, cctx, api)
+		if err != nil {
+			return err
+		}
+
+		tsk := ts.Key()
+		actors, err := api.StateListActors(ctx, tsk)
+		if err != nil {
+			return err
+		}
+
+		bal := big.Zero()
+		for _, addr := range actors {
+			act, err := api.StateGetActor(ctx, addr, tsk)
+			if err != nil {
+				return err
+			}
+
+			bal = big.Add(bal, act.Balance)
+		}
+
+		attoBase := big.Mul(big.NewInt(int64(build.FilBase)), big.NewInt(int64(build.FilecoinPrecision)))
+
+		if big.Cmp(attoBase, bal) != 0 {
+			return xerrors.Errorf("sanity check failed (expected %s, actual %s)", attoBase, bal)
+		}
+
+		fmt.Println("sanity check successful")
+
+		return nil
 	},
 }
 
@@ -168,19 +492,26 @@ var chainBalanceStateCmd = &cli.Command{
 
 		defer lkrepo.Close() //nolint:errcheck
 
-		ds, err := lkrepo.Datastore("/chain")
+		bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
+		if err != nil {
+			return fmt.Errorf("failed to open blockstore: %w", err)
+		}
+
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
+
+		mds, err := lkrepo.Datastore(context.Background(), "/metadata")
 		if err != nil {
 			return err
 		}
 
-		mds, err := lkrepo.Datastore("/metadata")
-		if err != nil {
-			return err
-		}
-
-		bs := blockstore.NewBlockstore(ds)
-
-		cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
+		cs := store.NewChainStore(bs, bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
+		defer cs.Close() //nolint:errcheck
 
 		cst := cbor.NewCborStore(bs)
 		store := adt.WrapStore(ctx, cst)
@@ -382,19 +713,26 @@ var chainPledgeCmd = &cli.Command{
 
 		defer lkrepo.Close() //nolint:errcheck
 
-		ds, err := lkrepo.Datastore("/chain")
+		bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
+		if err != nil {
+			return xerrors.Errorf("failed to open blockstore: %w", err)
+		}
+
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
+
+		mds, err := lkrepo.Datastore(context.Background(), "/metadata")
 		if err != nil {
 			return err
 		}
 
-		mds, err := lkrepo.Datastore("/metadata")
-		if err != nil {
-			return err
-		}
-
-		bs := blockstore.NewBlockstore(ds)
-
-		cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
+		cs := store.NewChainStore(bs, bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
+		defer cs.Close() //nolint:errcheck
 
 		cst := cbor.NewCborStore(bs)
 		store := adt.WrapStore(ctx, cst)
@@ -471,3 +809,119 @@ var chainPledgeCmd = &cli.Command{
 		return nil
 	},
 }
+
+const dateFmt = "1/02/06"
+
+func parseCsv(inp string) ([]time.Time, []address.Address, error) {
+	fi, err := os.Open(inp)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	r := csv.NewReader(fi)
+	recs, err := r.ReadAll()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var addrs []address.Address
+	for _, rec := range recs[1:] {
+		a, err := address.NewFromString(rec[0])
+		if err != nil {
+			return nil, nil, err
+		}
+		addrs = append(addrs, a)
+	}
+
+	var dates []time.Time
+	for _, d := range recs[0][1:] {
+		if len(d) == 0 {
+			continue
+		}
+		p := strings.Split(d, " ")
+		t, err := time.Parse(dateFmt, p[len(p)-1])
+		if err != nil {
+			return nil, nil, err
+		}
+
+		dates = append(dates, t)
+	}
+
+	return dates, addrs, nil
+}
+
+func heightForDate(d time.Time, ts *types.TipSet) abi.ChainEpoch {
+	secs := d.Unix()
+	gents := ts.Blocks()[0].Timestamp
+	gents -= uint64(30 * ts.Height())
+	return abi.ChainEpoch((secs - int64(gents)) / 30)
+}
+
+var fillBalancesCmd = &cli.Command{
+	Name:        "fill-balances",
+	Description: "fill out balances for addresses on dates in given spreadsheet",
+	Flags:       []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		dates, addrs, err := parseCsv(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		ts, err := api.ChainHead(ctx)
+		if err != nil {
+			return err
+		}
+
+		var tipsets []*types.TipSet
+		for _, d := range dates {
+			h := heightForDate(d, ts)
+			hts, err := api.ChainGetTipSetByHeight(ctx, h, ts.Key())
+			if err != nil {
+				return err
+			}
+			tipsets = append(tipsets, hts)
+		}
+
+		var balances [][]abi.TokenAmount
+		for _, a := range addrs {
+			var b []abi.TokenAmount
+			for _, hts := range tipsets {
+				act, err := api.StateGetActor(ctx, a, hts.Key())
+				if err != nil {
+					if !strings.Contains(err.Error(), "actor not found") {
+						return fmt.Errorf("error for %s at %s: %w", a, hts.Key(), err)
+					}
+					b = append(b, types.NewInt(0))
+					continue
+				}
+				b = append(b, act.Balance)
+			}
+			balances = append(balances, b)
+		}
+
+		var datestrs []string
+		for _, d := range dates {
+			datestrs = append(datestrs, "Balance at "+d.Format(dateFmt))
+		}
+
+		w := csv.NewWriter(os.Stdout)
+		w.Write(append([]string{"Wallet Address"}, datestrs...)) // nolint:errcheck
+		for i := 0; i < len(addrs); i++ {
+			row := []string{addrs[i].String()}
+			for _, b := range balances[i] {
+				row = append(row, types.FIL(b).String())
+			}
+			w.Write(row) // nolint:errcheck
+		}
+		w.Flush()
+		return nil
+	},
+}

cmd/lotus-shed/base64.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/go-address"
+
+	"github.com/urfave/cli/v2"
+)
+
+var base64Cmd = &cli.Command{
+	Name:        "base64",
+	Description: "multiformats base64",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "decodeAddr",
+			Value: false,
+			Usage: "Decode a base64 addr",
+		},
+		&cli.BoolFlag{
+			Name:  "decodeBig",
+			Value: false,
+			Usage: "Decode a base64 big",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		var input io.Reader
+
+		if cctx.Args().Len() == 0 {
+			input = os.Stdin
+		} else {
+			input = strings.NewReader(cctx.Args().First())
+		}
+
+		bytes, err := ioutil.ReadAll(input)
+		if err != nil {
+			return nil
+		}
+
+		decoded, err := base64.RawStdEncoding.DecodeString(strings.TrimSpace(string(bytes)))
+		if err != nil {
+			return err
+		}
+
+		if cctx.Bool("decodeAddr") {
+			addr, err := address.NewFromBytes(decoded)
+			if err != nil {
+				return err
+			}
+
+			fmt.Println(addr)
+
+			return nil
+		}
+
+		if cctx.Bool("decodeBig") {
+			var val abi.TokenAmount
+			err = val.UnmarshalBinary(decoded)
+			if err != nil {
+				return err
+			}
+
+			fmt.Println(val)
+		}
+
+		return nil
+	},
+}

cmd/lotus-shed/bitfield.go

@@ -17,6 +17,7 @@ import (
 
 var bitFieldCmd = &cli.Command{
 	Name:        "bitfield",
+	Usage:       "Bitfield analyze tool",
 	Description: "analyze bitfields",
 	Flags: []cli.Flag{
 		&cli.StringFlag{
@@ -26,53 +27,24 @@ var bitFieldCmd = &cli.Command{
 		},
 	},
 	Subcommands: []*cli.Command{
+		bitFieldEncodeCmd,
+		bitFieldDecodeCmd,
 		bitFieldRunsCmd,
 		bitFieldStatCmd,
-		bitFieldDecodeCmd,
+		bitFieldMergeCmd,
 		bitFieldIntersectCmd,
-		bitFieldEncodeCmd,
 		bitFieldSubCmd,
 	},
 }
 
 var bitFieldRunsCmd = &cli.Command{
 	Name:        "runs",
+	Usage:       "Bitfield bit runs",
 	Description: "print bit runs in a bitfield",
-	Flags: []cli.Flag{
-		&cli.StringFlag{
-			Name:  "enc",
-			Value: "base64",
-			Usage: "specify input encoding to parse",
-		},
-	},
 	Action: func(cctx *cli.Context) error {
-		var val string
-		if cctx.Args().Present() {
-			val = cctx.Args().Get(0)
-		} else {
-			b, err := ioutil.ReadAll(os.Stdin)
-			if err != nil {
-				return err
-			}
-			val = string(b)
-		}
-
-		var dec []byte
-		switch cctx.String("enc") {
-		case "base64":
-			d, err := base64.StdEncoding.DecodeString(val)
-			if err != nil {
-				return fmt.Errorf("decoding base64 value: %w", err)
-			}
-			dec = d
-		case "hex":
-			d, err := hex.DecodeString(val)
-			if err != nil {
-				return fmt.Errorf("decoding hex value: %w", err)
-			}
-			dec = d
-		default:
-			return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc"))
+		dec, err := decodeToByte(cctx, 0)
+		if err != nil {
+			return err
 		}
 
 		rle, err := rlepluslazy.FromBuf(dec)
@@ -98,7 +70,7 @@ var bitFieldRunsCmd = &cli.Command{
 				s = "FALSE"
 			}
 
-			fmt.Printf("@%d %s * %d\n", idx, s, r.Len)
+			fmt.Printf("@%08d %s * %d\n", idx, s, r.Len)
 
 			idx += r.Len
 		}
@@ -109,43 +81,14 @@ var bitFieldRunsCmd = &cli.Command{
 
 var bitFieldStatCmd = &cli.Command{
 	Name:        "stat",
+	Usage:       "Bitfield stats",
 	Description: "print bitfield stats",
-	Flags: []cli.Flag{
-		&cli.StringFlag{
-			Name:  "enc",
-			Value: "base64",
-			Usage: "specify input encoding to parse",
-		},
-	},
 	Action: func(cctx *cli.Context) error {
-		var val string
-		if cctx.Args().Present() {
-			val = cctx.Args().Get(0)
-		} else {
-			b, err := ioutil.ReadAll(os.Stdin)
-			if err != nil {
-				return err
-			}
-			val = string(b)
-		}
-
-		var dec []byte
-		switch cctx.String("enc") {
-		case "base64":
-			d, err := base64.StdEncoding.DecodeString(val)
-			if err != nil {
-				return fmt.Errorf("decoding base64 value: %w", err)
-			}
-			dec = d
-		case "hex":
-			d, err := hex.DecodeString(val)
-			if err != nil {
-				return fmt.Errorf("decoding hex value: %w", err)
-			}
-			dec = d
-		default:
-			return fmt.Errorf("unrecognized encoding: %s", cctx.String("enc"))
+		dec, err := decodeToByte(cctx, 0)
+		if err != nil {
+			return err
 		}
+		fmt.Printf("Raw length: %d bits (%d bytes)\n", len(dec)*8, len(dec))
 
 		rle, err := rlepluslazy.FromBuf(dec)
 		if err != nil {
@@ -157,10 +100,7 @@ var bitFieldStatCmd = &cli.Command{
 			return xerrors.Errorf("getting run iterator: %w", err)
 		}
 
-		fmt.Printf("Raw length: %d bits (%d bytes)\n", len(dec)*8, len(dec))
-
 		var ones, zeros, oneRuns, zeroRuns, invalid uint64
-
 		for rit.HasNext() {
 			r, err := rit.NextRun()
 			if err != nil {
@@ -195,14 +135,8 @@ var bitFieldStatCmd = &cli.Command{
 
 var bitFieldDecodeCmd = &cli.Command{
 	Name:        "decode",
+	Usage:       "Bitfield to decimal number",
 	Description: "decode bitfield and print all numbers in it",
-	Flags: []cli.Flag{
-		&cli.StringFlag{
-			Name:  "enc",
-			Value: "base64",
-			Usage: "specify input encoding to parse",
-		},
-	},
 	Action: func(cctx *cli.Context) error {
 		rle, err := decode(cctx, 0)
 		if err != nil {
@@ -219,43 +153,61 @@ var bitFieldDecodeCmd = &cli.Command{
 	},
 }
 
-var bitFieldIntersectCmd = &cli.Command{
-	Name:        "intersect",
-	Description: "intersect 2 bitfields and print the resulting bitfield as base64",
-	Flags: []cli.Flag{
-		&cli.StringFlag{
-			Name:  "enc",
-			Value: "base64",
-			Usage: "specify input encoding to parse",
-		},
-	},
+var bitFieldMergeCmd = &cli.Command{
+	Name:        "merge",
+	Usage:       "Merge 2 bitfields",
+	Description: "Merge 2 bitfields and print the resulting bitfield",
 	Action: func(cctx *cli.Context) error {
+		a, err := decode(cctx, 0)
+		if err != nil {
+			return err
+		}
+
 		b, err := decode(cctx, 1)
 		if err != nil {
 			return err
 		}
 
+		o, err := bitfield.MergeBitFields(a, b)
+		if err != nil {
+			return xerrors.Errorf("merge: %w", err)
+		}
+
+		str, err := encode(cctx, o)
+		if err != nil {
+			return err
+		}
+		fmt.Println(str)
+
+		return nil
+	},
+}
+
+var bitFieldIntersectCmd = &cli.Command{
+	Name:        "intersect",
+	Usage:       "Intersect 2 bitfields",
+	Description: "intersect 2 bitfields and print the resulting bitfield",
+	Action: func(cctx *cli.Context) error {
 		a, err := decode(cctx, 0)
 		if err != nil {
 			return err
 		}
 
+		b, err := decode(cctx, 1)
+		if err != nil {
+			return err
+		}
+
 		o, err := bitfield.IntersectBitField(a, b)
 		if err != nil {
 			return xerrors.Errorf("intersect: %w", err)
 		}
 
-		s, err := o.RunIterator()
+		str, err := encode(cctx, o)
 		if err != nil {
 			return err
 		}
-
-		bytes, err := rlepluslazy.EncodeRuns(s, []byte{})
-		if err != nil {
-			return err
-		}
-
-		fmt.Println(base64.StdEncoding.EncodeToString(bytes))
+		fmt.Println(str)
 
 		return nil
 	},
@@ -263,41 +215,29 @@ var bitFieldIntersectCmd = &cli.Command{
 
 var bitFieldSubCmd = &cli.Command{
 	Name:        "sub",
-	Description: "subtract 2 bitfields and print the resulting bitfield as base64",
-	Flags: []cli.Flag{
-		&cli.StringFlag{
-			Name:  "enc",
-			Value: "base64",
-			Usage: "specify input encoding to parse",
-		},
-	},
+	Usage:       "Subtract 2 bitfields",
+	Description: "subtract 2 bitfields and print the resulting bitfield",
 	Action: func(cctx *cli.Context) error {
-		b, err := decode(cctx, 1)
+		a, err := decode(cctx, 0)
 		if err != nil {
 			return err
 		}
 
-		a, err := decode(cctx, 0)
+		b, err := decode(cctx, 1)
 		if err != nil {
 			return err
 		}
 
 		o, err := bitfield.SubtractBitField(a, b)
 		if err != nil {
-			return xerrors.Errorf("intersect: %w", err)
+			return xerrors.Errorf("subtract: %w", err)
 		}
 
-		s, err := o.RunIterator()
+		str, err := encode(cctx, o)
 		if err != nil {
 			return err
 		}
-
-		bytes, err := rlepluslazy.EncodeRuns(s, []byte{})
-		if err != nil {
-			return err
-		}
-
-		fmt.Println(base64.StdEncoding.EncodeToString(bytes))
+		fmt.Println(str)
 
 		return nil
 	},
@@ -305,15 +245,9 @@ var bitFieldSubCmd = &cli.Command{
 
 var bitFieldEncodeCmd = &cli.Command{
 	Name:        "encode",
+	Usage:       "Decimal number to bitfield",
 	Description: "encode a series of decimal numbers into a bitfield",
 	ArgsUsage:   "[infile]",
-	Flags: []cli.Flag{
-		&cli.StringFlag{
-			Name:  "enc",
-			Value: "base64",
-			Usage: "specify input encoding to parse",
-		},
-	},
 	Action: func(cctx *cli.Context) error {
 		f, err := os.Open(cctx.Args().First())
 		if err != nil {
@@ -331,38 +265,64 @@ var bitFieldEncodeCmd = &cli.Command{
 			out.Set(i)
 		}
 
-		s, err := out.RunIterator()
+		str, err := encode(cctx, out)
 		if err != nil {
 			return err
 		}
-
-		bytes, err := rlepluslazy.EncodeRuns(s, []byte{})
-		if err != nil {
-			return err
-		}
-
-		fmt.Println(base64.StdEncoding.EncodeToString(bytes))
+		fmt.Println(str)
 
 		return nil
 	},
 }
 
-func decode(cctx *cli.Context, a int) (bitfield.BitField, error) {
+func encode(cctx *cli.Context, field bitfield.BitField) (string, error) {
+	s, err := field.RunIterator()
+	if err != nil {
+		return "", err
+	}
+
+	bytes, err := rlepluslazy.EncodeRuns(s, []byte{})
+	if err != nil {
+		return "", err
+	}
+
+	var str string
+	switch cctx.String("enc") {
+	case "base64":
+		str = base64.StdEncoding.EncodeToString(bytes)
+	case "hex":
+		str = hex.EncodeToString(bytes)
+	default:
+		return "", fmt.Errorf("unrecognized encoding: %s", cctx.String("enc"))
+	}
+
+	return str, nil
+
+}
+func decode(cctx *cli.Context, i int) (bitfield.BitField, error) {
+	b, err := decodeToByte(cctx, i)
+	if err != nil {
+		return bitfield.BitField{}, err
+	}
+	return bitfield.NewFromBytes(b)
+}
+
+func decodeToByte(cctx *cli.Context, i int) ([]byte, error) {
 	var val string
 	if cctx.Args().Present() {
-		if a >= cctx.NArg() {
-			return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a)
+		if i >= cctx.NArg() {
+			return nil, xerrors.Errorf("need more than %d args", i)
 		}
-		val = cctx.Args().Get(a)
+		val = cctx.Args().Get(i)
 	} else {
-		if a > 0 {
-			return bitfield.BitField{}, xerrors.Errorf("need more than %d args", a)
+		if i > 0 {
+			return nil, xerrors.Errorf("need more than %d args", i)
 		}
-		b, err := ioutil.ReadAll(os.Stdin)
+		r, err := ioutil.ReadAll(os.Stdin)
 		if err != nil {
-			return bitfield.BitField{}, err
+			return nil, err
 		}
-		val = string(b)
+		val = string(r)
 	}
 
 	var dec []byte
@@ -370,18 +330,18 @@ func decode(cctx *cli.Context, a int) (bitfield.BitField, error) {
 	case "base64":
 		d, err := base64.StdEncoding.DecodeString(val)
 		if err != nil {
-			return bitfield.BitField{}, fmt.Errorf("decoding base64 value: %w", err)
+			return nil, fmt.Errorf("decoding base64 value: %w", err)
 		}
 		dec = d
 	case "hex":
 		d, err := hex.DecodeString(val)
 		if err != nil {
-			return bitfield.BitField{}, fmt.Errorf("decoding hex value: %w", err)
+			return nil, fmt.Errorf("decoding hex value: %w", err)
 		}
 		dec = d
 	default:
-		return bitfield.BitField{}, fmt.Errorf("unrecognized encoding: %s", cctx.String("enc"))
+		return nil, fmt.Errorf("unrecognized encoding: %s", cctx.String("enc"))
 	}
 
-	return bitfield.NewFromBytes(dec)
+	return dec, nil
 }

cmd/lotus-shed/blockmsgid.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+
+	blake2b "github.com/minio/blake2b-simd"
+	"github.com/urfave/cli/v2"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var blockmsgidCmd = &cli.Command{
+	Name:      "blockmsgid",
+	Usage:     "Print a block's pubsub message ID",
+	ArgsUsage: "<blockCid> ...",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		for _, arg := range cctx.Args().Slice() {
+			blkcid, err := cid.Decode(arg)
+			if err != nil {
+				return fmt.Errorf("error decoding block cid: %w", err)
+			}
+
+			blkhdr, err := api.ChainGetBlock(ctx, blkcid)
+			if err != nil {
+				return fmt.Errorf("error retrieving block header: %w", err)
+			}
+
+			blkmsgs, err := api.ChainGetBlockMessages(ctx, blkcid)
+			if err != nil {
+				return fmt.Errorf("error retrieving block messages: %w", err)
+			}
+
+			blkmsg := &types.BlockMsg{
+				Header: blkhdr,
+			}
+
+			for _, m := range blkmsgs.BlsMessages {
+				blkmsg.BlsMessages = append(blkmsg.BlsMessages, m.Cid())
+			}
+
+			for _, m := range blkmsgs.SecpkMessages {
+				blkmsg.SecpkMessages = append(blkmsg.SecpkMessages, m.Cid())
+			}
+
+			bytes, err := blkmsg.Serialize()
+			if err != nil {
+				return fmt.Errorf("error serializing BlockMsg: %w", err)
+			}
+
+			msgId := blake2b.Sum256(bytes)
+			msgId64 := base64.StdEncoding.EncodeToString(msgId[:])
+
+			fmt.Println(msgId64)
+		}
+
+		return nil
+	},
+}

cmd/lotus-shed/cid.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var cidCmd = &cli.Command{
+	Name:  "cid",
+	Usage: "Cid command",
+	Subcommands: cli.Commands{
+		cidIdCmd,
+	},
+}
+
+var cidIdCmd = &cli.Command{
+	Name:      "id",
+	Usage:     "Create identity CID from hex or base64 data",
+	ArgsUsage: "[data]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "encoding",
+			Value: "base64",
+			Usage: "specify input encoding to parse",
+		},
+		&cli.StringFlag{
+			Name:  "codec",
+			Value: "id",
+			Usage: "multicodec-packed content types: abi or id",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must specify data")
+		}
+
+		var dec []byte
+		switch cctx.String("encoding") {
+		case "base64":
+			data, err := base64.StdEncoding.DecodeString(cctx.Args().First())
+			if err != nil {
+				return xerrors.Errorf("decoding base64 value: %w", err)
+			}
+			dec = data
+		case "hex":
+			data, err := hex.DecodeString(cctx.Args().First())
+			if err != nil {
+				return xerrors.Errorf("decoding hex value: %w", err)
+			}
+			dec = data
+		default:
+			return xerrors.Errorf("unrecognized encoding: %s", cctx.String("encoding"))
+		}
+
+		switch cctx.String("codec") {
+		case "abi":
+			aCid, err := abi.CidBuilder.Sum(dec)
+			if err != nil {
+				return xerrors.Errorf("cidBuilder abi: %w", err)
+			}
+			fmt.Println(aCid)
+		case "id":
+			builder := cid.V1Builder{Codec: cid.Raw, MhType: mh.IDENTITY}
+			rCid, err := builder.Sum(dec)
+			if err != nil {
+				return xerrors.Errorf("cidBuilder raw: %w", err)
+			}
+			fmt.Println(rCid)
+		default:
+			return xerrors.Errorf("unrecognized codec: %s", cctx.String("codec"))
+		}
+
+		return nil
+	},
+}

cmd/lotus-shed/commp.go

@@ -1,27 +1,55 @@
 package main
 
 import (
+	"encoding/base64"
 	"encoding/hex"
 	"fmt"
 
 	commcid "github.com/filecoin-project/go-fil-commcid"
 	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
 )
 
 var commpToCidCmd = &cli.Command{
 	Name:        "commp-to-cid",
+	Usage:       "Convert commP to Cid",
 	Description: "Convert a raw commP to a piece-Cid",
+	ArgsUsage:   "[data]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "encoding",
+			Value: "base64",
+			Usage: "specify input encoding to parse",
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		if !cctx.Args().Present() {
 			return fmt.Errorf("must specify commP to convert")
 		}
 
-		dec, err := hex.DecodeString(cctx.Args().First())
-		if err != nil {
-			return fmt.Errorf("failed to decode input as hex string: %w", err)
+		var dec []byte
+		switch cctx.String("encoding") {
+		case "base64":
+			data, err := base64.StdEncoding.DecodeString(cctx.Args().First())
+			if err != nil {
+				return xerrors.Errorf("decoding base64 value: %w", err)
+			}
+			dec = data
+		case "hex":
+			data, err := hex.DecodeString(cctx.Args().First())
+			if err != nil {
+				return xerrors.Errorf("decoding hex value: %w", err)
+			}
+			dec = data
+		default:
+			return xerrors.Errorf("unrecognized encoding: %s", cctx.String("encoding"))
 		}
 
-		fmt.Println(commcid.PieceCommitmentV1ToCID(dec))
+		cid, err := commcid.PieceCommitmentV1ToCID(dec)
+		if err != nil {
+			return err
+		}
+		fmt.Println(cid)
 		return nil
 	},
 }

cmd/lotus-shed/consensus.go

@@ -36,7 +36,7 @@ type consensusItem struct {
 	targetTipset  *types.TipSet
 	headTipset    *types.TipSet
 	peerID        peer.ID
-	version       api.Version
+	version       api.APIVersion
 	api           api.FullNode
 }
 
@@ -113,12 +113,12 @@ var consensusCheckCmd = &cli.Command{
 				return err
 			}
 			ainfo := cliutil.APIInfo{Addr: apima.String()}
-			addr, err := ainfo.DialArgs()
+			addr, err := ainfo.DialArgs("v1")
 			if err != nil {
 				return err
 			}
 
-			api, closer, err := client.NewFullNodeRPC(cctx.Context, addr, nil)
+			api, closer, err := client.NewFullNodeRPCV1(cctx.Context, addr, nil)
 			if err != nil {
 				return err
 			}

cmd/lotus-shed/cron-count.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/build"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var cronWcCmd = &cli.Command{
+	Name:        "cron-wc",
+	Description: "cron stats",
+	Subcommands: []*cli.Command{
+		minerDeadlineCronCountCmd,
+	},
+}
+
+var minerDeadlineCronCountCmd = &cli.Command{
+	Name:        "deadline",
+	Description: "list all addresses of miners with active deadline crons",
+	Action: func(c *cli.Context) error {
+		return countDeadlineCrons(c)
+	},
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "tipset",
+			Usage: "specify tipset state to search on (pass comma separated array of cids)",
+		},
+	},
+}
+
+func findDeadlineCrons(c *cli.Context) (map[address.Address]struct{}, error) {
+	api, acloser, err := lcli.GetFullNodeAPI(c)
+	if err != nil {
+		return nil, err
+	}
+	defer acloser()
+	ctx := lcli.ReqContext(c)
+
+	ts, err := lcli.LoadTipSet(ctx, c, api)
+	if err != nil {
+		return nil, err
+	}
+	if ts == nil {
+		ts, err = api.ChainHead(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	mAddrs, err := api.StateListMiners(ctx, ts.Key())
+	if err != nil {
+		return nil, err
+	}
+	activeMiners := make(map[address.Address]struct{})
+	for _, mAddr := range mAddrs {
+		// All miners have active cron before v4.
+		// v4 upgrade epoch is last epoch running v3 epoch and api.StateReadState reads
+		// parent state, so v4 state isn't read until upgrade epoch + 2
+		if ts.Height() <= build.UpgradeTurboHeight+1 {
+			activeMiners[mAddr] = struct{}{}
+			continue
+		}
+		st, err := api.StateReadState(ctx, mAddr, ts.Key())
+		if err != nil {
+			return nil, err
+		}
+		minerState, ok := st.State.(map[string]interface{})
+		if !ok {
+			return nil, xerrors.Errorf("internal error: failed to cast miner state to expected map type")
+		}
+
+		activeDlineIface, ok := minerState["DeadlineCronActive"]
+		if !ok {
+			return nil, xerrors.Errorf("miner %s had no deadline state, is this a v3 state root?", mAddr)
+		}
+		active := activeDlineIface.(bool)
+		if active {
+			activeMiners[mAddr] = struct{}{}
+		}
+	}
+
+	return activeMiners, nil
+}
+
+func countDeadlineCrons(c *cli.Context) error {
+	activeMiners, err := findDeadlineCrons(c)
+	if err != nil {
+		return err
+	}
+	for addr := range activeMiners {
+		fmt.Printf("%s\n", addr)
+	}
+
+	return nil
+}

cmd/lotus-shed/datastore.go

@@ -1,17 +1,23 @@
 package main
 
 import (
+	"bufio"
+	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 
+	"github.com/dgraph-io/badger/v2"
 	"github.com/docker/go-units"
 	"github.com/ipfs/go-datastore"
 	dsq "github.com/ipfs/go-datastore/query"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
+	"github.com/mitchellh/go-homedir"
 	"github.com/polydawn/refmt/cbor"
 	"github.com/urfave/cli/v2"
+	"go.uber.org/multierr"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/lib/backupds"
@@ -25,6 +31,7 @@ var datastoreCmd = &cli.Command{
 		datastoreBackupCmd,
 		datastoreListCmd,
 		datastoreGetCmd,
+		datastoreRewriteCmd,
 	},
 }
 
@@ -69,7 +76,7 @@ var datastoreListCmd = &cli.Command{
 		}
 		defer lr.Close() //nolint:errcheck
 
-		ds, err := lr.Datastore(datastore.NewKey(cctx.Args().First()).String())
+		ds, err := lr.Datastore(context.Background(), datastore.NewKey(cctx.Args().First()).String())
 		if err != nil {
 			return err
 		}
@@ -114,7 +121,7 @@ var datastoreGetCmd = &cli.Command{
 	},
 	ArgsUsage: "[namespace key]",
 	Action: func(cctx *cli.Context) error {
-		logging.SetLogLevel("badger", "ERROR") // nolint:errchec
+		logging.SetLogLevel("badger", "ERROR") // nolint:errcheck
 
 		r, err := repo.NewFS(cctx.String("repo"))
 		if err != nil {
@@ -135,7 +142,7 @@ var datastoreGetCmd = &cli.Command{
 		}
 		defer lr.Close() //nolint:errcheck
 
-		ds, err := lr.Datastore(datastore.NewKey(cctx.Args().First()).String())
+		ds, err := lr.Datastore(context.Background(), datastore.NewKey(cctx.Args().First()).String())
 		if err != nil {
 			return err
 		}
@@ -173,8 +180,11 @@ var datastoreBackupStatCmd = &cli.Command{
 		}
 		defer f.Close() // nolint:errcheck
 
-		var keys, kbytes, vbytes uint64
-		err = backupds.ReadBackup(f, func(key datastore.Key, value []byte) error {
+		var keys, logs, kbytes, vbytes uint64
+		clean, err := backupds.ReadBackup(f, func(key datastore.Key, value []byte, log bool) error {
+			if log {
+				logs++
+			}
 			keys++
 			kbytes += uint64(len(key.String()))
 			vbytes += uint64(len(value))
@@ -184,7 +194,9 @@ var datastoreBackupStatCmd = &cli.Command{
 			return err
 		}
 
+		fmt.Println("Truncated:   ", !clean)
 		fmt.Println("Keys:        ", keys)
+		fmt.Println("Log values:  ", log)
 		fmt.Println("Key bytes:   ", units.BytesSize(float64(kbytes)))
 		fmt.Println("Value bytes: ", units.BytesSize(float64(vbytes)))
 
@@ -218,7 +230,7 @@ var datastoreBackupListCmd = &cli.Command{
 		defer f.Close() // nolint:errcheck
 
 		printKv := kvPrinter(cctx.Bool("top-level"), cctx.String("get-enc"))
-		err = backupds.ReadBackup(f, func(key datastore.Key, value []byte) error {
+		_, err = backupds.ReadBackup(f, func(key datastore.Key, value []byte, _ bool) error {
 			return printKv(key.String(), value)
 		})
 		if err != nil {
@@ -288,3 +300,76 @@ func printVal(enc string, val []byte) error {
 
 	return nil
 }
+
+var datastoreRewriteCmd = &cli.Command{
+	Name:        "rewrite",
+	Description: "rewrites badger datastore to compact it and possibly change params",
+	ArgsUsage:   "source destination",
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 2 {
+			return xerrors.Errorf("expected 2 arguments, got %d", cctx.NArg())
+		}
+		fromPath, err := homedir.Expand(cctx.Args().Get(0))
+		if err != nil {
+			return xerrors.Errorf("cannot get fromPath: %w", err)
+		}
+		toPath, err := homedir.Expand(cctx.Args().Get(1))
+		if err != nil {
+			return xerrors.Errorf("cannot get toPath: %w", err)
+		}
+
+		var (
+			from *badger.DB
+			to   *badger.DB
+		)
+
+		// open the destination (to) store.
+		opts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, toPath, false)
+		if err != nil {
+			return xerrors.Errorf("failed to get badger options: %w", err)
+		}
+		opts.SyncWrites = false
+		if to, err = badger.Open(opts.Options); err != nil {
+			return xerrors.Errorf("opening 'to' badger store: %w", err)
+		}
+
+		// open the source (from) store.
+		opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, fromPath, true)
+		if err != nil {
+			return xerrors.Errorf("failed to get badger options: %w", err)
+		}
+		if from, err = badger.Open(opts.Options); err != nil {
+			return xerrors.Errorf("opening 'from' datastore: %w", err)
+		}
+
+		pr, pw := io.Pipe()
+		errCh := make(chan error)
+		go func() {
+			bw := bufio.NewWriterSize(pw, 64<<20)
+			_, err := from.Backup(bw, 0)
+			_ = bw.Flush()
+			_ = pw.CloseWithError(err)
+			errCh <- err
+		}()
+		go func() {
+			err := to.Load(pr, 256)
+			errCh <- err
+		}()
+
+		err = <-errCh
+		if err != nil {
+			select {
+			case nerr := <-errCh:
+				err = multierr.Append(err, nerr)
+			default:
+			}
+			return err
+		}
+
+		err = <-errCh
+		if err != nil {
+			return err
+		}
+		return multierr.Append(from.Close(), to.Close())
+	},
+}

cmd/lotus-shed/dealtracker.go

@@ -1,325 +0,0 @@
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"net"
-	"net/http"
-	"sync"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/lotus/api"
-	lcli "github.com/filecoin-project/lotus/cli"
-	"github.com/ipfs/go-cid"
-	"github.com/urfave/cli/v2"
-)
-
-type dealStatsServer struct {
-	api api.FullNode
-}
-
-// Requested by @jbenet
-// How many epochs back to look at for dealstats
-var epochLookback = abi.ChainEpoch(10)
-
-// these lists grow continuously with the network
-// TODO: need to switch this to an LRU of sorts, to ensure refreshes
-var knownFiltered = new(sync.Map)
-var resolvedWallets = new(sync.Map)
-
-func init() {
-	for _, a := range []string{
-		"t0100", // client for genesis miner
-		"t0101", // client for genesis miner
-		"t0102", // client for genesis miner
-		"t0112", // client for genesis miner
-		"t0113", // client for genesis miner
-		"t0114", // client for genesis miner
-		"t1nslxql4pck5pq7hddlzym3orxlx35wkepzjkm3i", // SR1 dealbot wallet
-		"t1stghxhdp2w53dym2nz2jtbpk6ccd4l2lxgmezlq", // SR1 dealbot wallet
-		"t1mcr5xkgv4jdl3rnz77outn6xbmygb55vdejgbfi", // SR1 dealbot wallet
-		"t1qiqdbbmrdalbntnuapriirduvxu5ltsc5mhy7si", // SR1 dealbot wallet
-	} {
-		a, err := address.NewFromString(a)
-		if err != nil {
-			panic(err)
-		}
-		knownFiltered.Store(a, true)
-	}
-}
-
-type dealCountResp struct {
-	Epoch    int64  `json:"epoch"`
-	Endpoint string `json:"endpoint"`
-	Payload  int64  `json:"payload"`
-}
-
-func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *http.Request) {
-
-	epoch, deals := dss.filteredDealList()
-	if epoch == 0 {
-		w.WriteHeader(500)
-		return
-	}
-
-	if err := json.NewEncoder(w).Encode(&dealCountResp{
-		Endpoint: "COUNT_DEALS",
-		Payload:  int64(len(deals)),
-		Epoch:    epoch,
-	}); err != nil {
-		log.Warnf("failed to write back deal count response: %s", err)
-		return
-	}
-}
-
-type dealAverageResp struct {
-	Epoch    int64  `json:"epoch"`
-	Endpoint string `json:"endpoint"`
-	Payload  int64  `json:"payload"`
-}
-
-func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, r *http.Request) {
-
-	epoch, deals := dss.filteredDealList()
-	if epoch == 0 {
-		w.WriteHeader(500)
-		return
-	}
-
-	var totalBytes int64
-	for _, d := range deals {
-		totalBytes += int64(d.deal.Proposal.PieceSize.Unpadded())
-	}
-
-	if err := json.NewEncoder(w).Encode(&dealAverageResp{
-		Endpoint: "AVERAGE_DEAL_SIZE",
-		Payload:  totalBytes / int64(len(deals)),
-		Epoch:    epoch,
-	}); err != nil {
-		log.Warnf("failed to write back deal average response: %s", err)
-		return
-	}
-}
-
-type dealTotalResp struct {
-	Epoch    int64  `json:"epoch"`
-	Endpoint string `json:"endpoint"`
-	Payload  int64  `json:"payload"`
-}
-
-func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r *http.Request) {
-	epoch, deals := dss.filteredDealList()
-	if epoch == 0 {
-		w.WriteHeader(500)
-		return
-	}
-
-	var totalBytes int64
-	for _, d := range deals {
-		totalBytes += int64(d.deal.Proposal.PieceSize.Unpadded())
-	}
-
-	if err := json.NewEncoder(w).Encode(&dealTotalResp{
-		Endpoint: "DEAL_BYTES",
-		Payload:  totalBytes,
-		Epoch:    epoch,
-	}); err != nil {
-		log.Warnf("failed to write back deal average response: %s", err)
-		return
-	}
-
-}
-
-type clientStatsOutput struct {
-	Epoch    int64          `json:"epoch"`
-	Endpoint string         `json:"endpoint"`
-	Payload  []*clientStats `json:"payload"`
-}
-
-type clientStats struct {
-	Client    address.Address `json:"client"`
-	DataSize  int64           `json:"data_size"`
-	NumCids   int             `json:"num_cids"`
-	NumDeals  int             `json:"num_deals"`
-	NumMiners int             `json:"num_miners"`
-
-	cids      map[cid.Cid]bool
-	providers map[address.Address]bool
-}
-
-func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *http.Request) {
-	epoch, deals := dss.filteredDealList()
-	if epoch == 0 {
-		w.WriteHeader(500)
-		return
-	}
-
-	stats := make(map[address.Address]*clientStats)
-
-	for _, d := range deals {
-
-		st, ok := stats[d.deal.Proposal.Client]
-		if !ok {
-			st = &clientStats{
-				Client:    d.resolvedWallet,
-				cids:      make(map[cid.Cid]bool),
-				providers: make(map[address.Address]bool),
-			}
-			stats[d.deal.Proposal.Client] = st
-		}
-
-		st.DataSize += int64(d.deal.Proposal.PieceSize.Unpadded())
-		st.cids[d.deal.Proposal.PieceCID] = true
-		st.providers[d.deal.Proposal.Provider] = true
-		st.NumDeals++
-	}
-
-	out := clientStatsOutput{
-		Epoch:    epoch,
-		Endpoint: "CLIENT_DEAL_STATS",
-		Payload:  make([]*clientStats, 0, len(stats)),
-	}
-	for _, cs := range stats {
-		cs.NumCids = len(cs.cids)
-		cs.NumMiners = len(cs.providers)
-		out.Payload = append(out.Payload, cs)
-	}
-
-	if err := json.NewEncoder(w).Encode(out); err != nil {
-		log.Warnf("failed to write back client stats response: %s", err)
-		return
-	}
-}
-
-type dealInfo struct {
-	deal           api.MarketDeal
-	resolvedWallet address.Address
-}
-
-// filteredDealList returns the current epoch and a list of filtered deals
-// on error returns an epoch of 0
-func (dss *dealStatsServer) filteredDealList() (int64, map[string]dealInfo) {
-	ctx := context.Background()
-
-	head, err := dss.api.ChainHead(ctx)
-	if err != nil {
-		log.Warnf("failed to get chain head: %s", err)
-		return 0, nil
-	}
-
-	head, err = dss.api.ChainGetTipSetByHeight(ctx, head.Height()-epochLookback, head.Key())
-	if err != nil {
-		log.Warnf("failed to walk back %s epochs: %s", epochLookback, err)
-		return 0, nil
-	}
-
-	// Disabled as per @pooja's request
-	//
-	// // Exclude any address associated with a miner
-	// miners, err := dss.api.StateListMiners(ctx, head.Key())
-	// if err != nil {
-	// 	log.Warnf("failed to get miner list: %s", err)
-	// 	return 0, nil
-	// }
-	// for _, m := range miners {
-	// 	info, err := dss.api.StateMinerInfo(ctx, m, head.Key())
-	// 	if err != nil {
-	// 		log.Warnf("failed to get info for known miner '%s': %s", m, err)
-	// 		continue
-	// 	}
-
-	// 	knownFiltered.Store(info.Owner, true)
-	// 	knownFiltered.Store(info.Worker, true)
-	// 	for _, a := range info.ControlAddresses {
-	// 		knownFiltered.Store(a, true)
-	// 	}
-	// }
-
-	deals, err := dss.api.StateMarketDeals(ctx, head.Key())
-	if err != nil {
-		log.Warnf("failed to get market deals: %s", err)
-		return 0, nil
-	}
-
-	ret := make(map[string]dealInfo, len(deals))
-	for dealKey, d := range deals {
-
-		// Counting no-longer-active deals as per Pooja's request
-		// // https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85
-		// if d.State.SectorStartEpoch < 0 {
-		// 	continue
-		// }
-
-		if _, isFiltered := knownFiltered.Load(d.Proposal.Client); isFiltered {
-			continue
-		}
-
-		if _, wasSeen := resolvedWallets.Load(d.Proposal.Client); !wasSeen {
-			w, err := dss.api.StateAccountKey(ctx, d.Proposal.Client, head.Key())
-			if err != nil {
-				log.Warnf("failed to resolve id '%s' to wallet address: %s", d.Proposal.Client, err)
-				continue
-			} else {
-				resolvedWallets.Store(d.Proposal.Client, w)
-			}
-		}
-
-		w, _ := resolvedWallets.Load(d.Proposal.Client)
-		if _, isFiltered := knownFiltered.Load(w); isFiltered {
-			continue
-		}
-
-		ret[dealKey] = dealInfo{
-			deal:           d,
-			resolvedWallet: w.(address.Address),
-		}
-	}
-
-	return int64(head.Height()), ret
-}
-
-var serveDealStatsCmd = &cli.Command{
-	Name:  "serve-deal-stats",
-	Flags: []cli.Flag{},
-	Action: func(cctx *cli.Context) error {
-		api, closer, err := lcli.GetFullNodeAPI(cctx)
-		if err != nil {
-			return err
-		}
-
-		defer closer()
-		ctx := lcli.ReqContext(cctx)
-
-		_ = ctx
-
-		dss := &dealStatsServer{api}
-
-		mux := &http.ServeMux{}
-		mux.HandleFunc("/api/storagedeal/count", dss.handleStorageDealCount)
-		mux.HandleFunc("/api/storagedeal/averagesize", dss.handleStorageDealAverageSize)
-		mux.HandleFunc("/api/storagedeal/totalreal", dss.handleStorageDealTotalReal)
-		mux.HandleFunc("/api/storagedeal/clientstats", dss.handleStorageClientStats)
-
-		s := &http.Server{
-			Addr:    ":7272",
-			Handler: mux,
-		}
-
-		go func() {
-			<-ctx.Done()
-			if err := s.Shutdown(context.TODO()); err != nil {
-				log.Error(err)
-			}
-		}()
-
-		list, err := net.Listen("tcp", ":7272") // nolint
-		if err != nil {
-			panic(err)
-		}
-
-		log.Warnf("deal-stat server listening on %s\n== NOTE: QUERIES ARE EXPENSIVE - YOU MUST FRONT-CACHE THIS SERVICE\n", list.Addr().String())
-
-		return s.Serve(list)
-	},
-}

cmd/lotus-shed/election.go

@@ -0,0 +1,227 @@
+package main
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"math/rand"
+
+	"github.com/filecoin-project/lotus/api/v0api"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/chain/gen"
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+	builtin2 "github.com/filecoin-project/specs-actors/v2/actors/builtin"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var electionCmd = &cli.Command{
+	Name:  "election",
+	Usage: "Commands related to leader election",
+	Subcommands: []*cli.Command{
+		electionRunDummy,
+		electionEstimate,
+		electionBacktest,
+	},
+}
+
+var electionRunDummy = &cli.Command{
+	Name:  "run-dummy",
+	Usage: "Runs dummy elections with given power",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "network-power",
+			Usage: "network storage power",
+		},
+		&cli.StringFlag{
+			Name:  "miner-power",
+			Usage: "miner storage power",
+		},
+		&cli.Uint64Flag{
+			Name:  "seed",
+			Usage: "rand number",
+			Value: 0,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+		minerPow, err := types.BigFromString(cctx.String("miner-power"))
+		if err != nil {
+			return xerrors.Errorf("decoding miner-power: %w", err)
+		}
+		networkPow, err := types.BigFromString(cctx.String("network-power"))
+		if err != nil {
+			return xerrors.Errorf("decoding network-power: %w", err)
+		}
+
+		ep := &types.ElectionProof{}
+		ep.VRFProof = make([]byte, 32)
+		seed := cctx.Uint64("seed")
+		if seed == 0 {
+			seed = rand.Uint64()
+		}
+		binary.BigEndian.PutUint64(ep.VRFProof, seed)
+
+		i := uint64(0)
+		for {
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+			binary.BigEndian.PutUint64(ep.VRFProof[8:], i)
+			j := ep.ComputeWinCount(minerPow, networkPow)
+			_, err := fmt.Printf("%t, %d\n", j != 0, j)
+			if err != nil {
+				return err
+			}
+			i++
+		}
+	},
+}
+
+var electionEstimate = &cli.Command{
+	Name:  "estimate",
+	Usage: "Estimate elections with given power",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "network-power",
+			Usage: "network storage power",
+		},
+		&cli.StringFlag{
+			Name:  "miner-power",
+			Usage: "miner storage power",
+		},
+		&cli.Uint64Flag{
+			Name:  "seed",
+			Usage: "rand number",
+			Value: 0,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		minerPow, err := types.BigFromString(cctx.String("miner-power"))
+		if err != nil {
+			return xerrors.Errorf("decoding miner-power: %w", err)
+		}
+		networkPow, err := types.BigFromString(cctx.String("network-power"))
+		if err != nil {
+			return xerrors.Errorf("decoding network-power: %w", err)
+		}
+
+		ep := &types.ElectionProof{}
+		ep.VRFProof = make([]byte, 32)
+		seed := cctx.Uint64("seed")
+		if seed == 0 {
+			seed = rand.Uint64()
+		}
+		binary.BigEndian.PutUint64(ep.VRFProof, seed)
+
+		winYear := int64(0)
+		for i := 0; i < builtin2.EpochsInYear; i++ {
+			binary.BigEndian.PutUint64(ep.VRFProof[8:], uint64(i))
+			j := ep.ComputeWinCount(minerPow, networkPow)
+			winYear += j
+		}
+		winHour := winYear * builtin2.EpochsInHour / builtin2.EpochsInYear
+		winDay := winYear * builtin2.EpochsInDay / builtin2.EpochsInYear
+		winMonth := winYear * builtin2.EpochsInDay * 30 / builtin2.EpochsInYear
+		fmt.Println("winInHour, winInDay, winInMonth, winInYear")
+		fmt.Printf("%d, %d, %d, %d\n", winHour, winDay, winMonth, winYear)
+		return nil
+	},
+}
+
+var electionBacktest = &cli.Command{
+	Name:      "backtest",
+	Usage:     "Backtest elections with given miner",
+	ArgsUsage: "[minerAddress]",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:  "height",
+			Usage: "blockchain head height",
+		},
+		&cli.IntFlag{
+			Name:  "count",
+			Usage: "number of won elections to look for",
+			Value: 120,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return xerrors.Errorf("GetFullNodeAPI: %w", err)
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		var head *types.TipSet
+		if cctx.IsSet("height") {
+			head, err = api.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(cctx.Uint64("height")), types.EmptyTSK)
+			if err != nil {
+				return xerrors.Errorf("ChainGetTipSetByHeight: %w", err)
+			}
+		} else {
+			head, err = api.ChainHead(ctx)
+			if err != nil {
+				return xerrors.Errorf("ChainHead: %w", err)
+			}
+		}
+
+		miner, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return xerrors.Errorf("miner address: %w", err)
+		}
+
+		count := cctx.Int("count")
+		if count < 1 {
+			return xerrors.Errorf("count: %d", count)
+		}
+
+		fmt.Println("height, winCount")
+		roundEnd := head.Height() + abi.ChainEpoch(1)
+		for i := 0; i < count; {
+			for round := head.Height() + abi.ChainEpoch(1); round <= roundEnd; round++ {
+				i++
+				win, err := backTestWinner(ctx, miner, round, head, api)
+				if err == nil && win != nil {
+					fmt.Printf("%d, %d\n", round, win.WinCount)
+				}
+			}
+
+			roundEnd = head.Height()
+			head, err = api.ChainGetTipSet(ctx, head.Parents())
+			if err != nil {
+				break
+			}
+		}
+		return nil
+	},
+}
+
+func backTestWinner(ctx context.Context, miner address.Address, round abi.ChainEpoch, ts *types.TipSet, api v0api.FullNode) (*types.ElectionProof, error) {
+	mbi, err := api.MinerGetBaseInfo(ctx, miner, round, ts.Key())
+	if err != nil {
+		return nil, xerrors.Errorf("failed to get mining base info: %w", err)
+	}
+	if mbi == nil {
+		return nil, nil
+	}
+	if !mbi.EligibleForMining {
+		return nil, nil
+	}
+
+	brand := mbi.PrevBeaconEntry
+	bvals := mbi.BeaconEntries
+	if len(bvals) > 0 {
+		brand = bvals[len(bvals)-1]
+	}
+
+	winner, err := gen.IsRoundWinner(ctx, ts, round, miner, brand, mbi, api)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to check if we win next round: %w", err)
+	}
+
+	return winner, nil
+}

cmd/lotus-shed/export-car.go

@@ -0,0 +1,103 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-merkledag"
+	"github.com/ipld/go-car"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+func carWalkFunc(nd format.Node) (out []*format.Link, err error) {
+	for _, link := range nd.Links() {
+		if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
+			continue
+		}
+		out = append(out, link)
+	}
+	return out, nil
+}
+
+var exportCarCmd = &cli.Command{
+	Name:        "export-car",
+	Description: "Export a car from repo (requires node to be offline)",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "repo",
+			Value: "~/.lotus",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 2 {
+			return lcli.ShowHelp(cctx, fmt.Errorf("must specify file name and object"))
+		}
+
+		outfile := cctx.Args().First()
+		var roots []cid.Cid
+		for _, arg := range cctx.Args().Tail() {
+			c, err := cid.Decode(arg)
+			if err != nil {
+				return err
+			}
+			roots = append(roots, c)
+		}
+
+		ctx := lcli.ReqContext(cctx)
+
+		r, err := repo.NewFS(cctx.String("repo"))
+		if err != nil {
+			return xerrors.Errorf("opening fs repo: %w", err)
+		}
+
+		exists, err := r.Exists()
+		if err != nil {
+			return err
+		}
+		if !exists {
+			return xerrors.Errorf("lotus repo doesn't exist")
+		}
+
+		lr, err := r.Lock(repo.FullNode)
+		if err != nil {
+			return err
+		}
+		defer lr.Close() //nolint:errcheck
+
+		fi, err := os.Create(outfile)
+		if err != nil {
+			return xerrors.Errorf("opening the output file: %w", err)
+		}
+
+		defer fi.Close() //nolint:errcheck
+
+		bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
+		if err != nil {
+			return fmt.Errorf("failed to open blockstore: %w", err)
+		}
+
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
+
+		dag := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
+		err = car.WriteCarWithWalker(ctx, dag, roots, fi, carWalkFunc)
+		if err != nil {
+			return err
+		}
+		return nil
+	},
+}

cmd/lotus-shed/export.go

@@ -3,16 +3,17 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
+
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -71,19 +72,27 @@ var exportChainCmd = &cli.Command{
 
 		defer fi.Close() //nolint:errcheck
 
-		ds, err := lr.Datastore("/chain")
+		bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
+		if err != nil {
+			return fmt.Errorf("failed to open blockstore: %w", err)
+		}
+
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
+
+		mds, err := lr.Datastore(context.Background(), "/metadata")
 		if err != nil {
 			return err
 		}
 
-		mds, err := lr.Datastore("/metadata")
-		if err != nil {
-			return err
-		}
+		cs := store.NewChainStore(bs, bs, mds, nil, nil)
+		defer cs.Close() //nolint:errcheck
 
-		bs := blockstore.NewBlockstore(ds)
-
-		cs := store.NewChainStore(bs, mds, nil, nil)
 		if err := cs.Load(); err != nil {
 			return err
 		}

cmd/lotus-shed/frozen-miners.go

@@ -35,12 +35,6 @@ var frozenMinersCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
-		if ts == nil {
-			ts, err = api.ChainHead(ctx)
-			if err != nil {
-				return err
-			}
-		}
 
 		queryEpoch := ts.Height()
 

cmd/lotus-shed/genesis-verify.go

@@ -17,6 +17,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/big"
 
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/account"
@@ -26,7 +27,6 @@ import (
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 )
 
 type addrInfo struct {
@@ -50,9 +50,10 @@ var genesisVerifyCmd = &cli.Command{
 		if !cctx.Args().Present() {
 			return fmt.Errorf("must pass genesis car file")
 		}
-		bs := blockstore.NewBlockstore(datastore.NewMapDatastore())
+		bs := blockstore.FromDatastore(datastore.NewMapDatastore())
 
-		cs := store.NewChainStore(bs, datastore.NewMapDatastore(), nil, nil)
+		cs := store.NewChainStore(bs, bs, datastore.NewMapDatastore(), nil, nil)
+		defer cs.Close() //nolint:errcheck
 
 		cf := cctx.Args().Get(0)
 		f, err := os.Open(cf)

cmd/lotus-shed/import-car.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"encoding/hex"
 	"fmt"
 	"io"
@@ -12,7 +13,6 @@ import (
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -25,6 +25,8 @@ var importCarCmd = &cli.Command{
 			return xerrors.Errorf("opening fs repo: %w", err)
 		}
 
+		ctx := context.TODO()
+
 		exists, err := r.Exists()
 		if err != nil {
 			return err
@@ -45,12 +47,18 @@ var importCarCmd = &cli.Command{
 			return xerrors.Errorf("opening the car file: %w", err)
 		}
 
-		ds, err := lr.Datastore("/chain")
+		bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
 		if err != nil {
 			return err
 		}
 
-		bs := blockstore.NewBlockstore(ds)
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
 
 		cr, err := car.NewCarReader(f)
 		if err != nil {
@@ -65,7 +73,7 @@ var importCarCmd = &cli.Command{
 					return err
 				}
 				fmt.Println()
-				return ds.Close()
+				return nil
 			default:
 				if err := f.Close(); err != nil {
 					return err
@@ -94,6 +102,8 @@ var importObjectCmd = &cli.Command{
 			return xerrors.Errorf("opening fs repo: %w", err)
 		}
 
+		ctx := context.TODO()
+
 		exists, err := r.Exists()
 		if err != nil {
 			return err
@@ -108,12 +118,18 @@ var importObjectCmd = &cli.Command{
 		}
 		defer lr.Close() //nolint:errcheck
 
-		ds, err := lr.Datastore("/chain")
+		bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to open blockstore: %w", err)
 		}
 
-		bs := blockstore.NewBlockstore(ds)
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
 
 		c, err := cid.Decode(cctx.Args().Get(0))
 		if err != nil {

cmd/lotus-shed/jwt.go

@@ -15,7 +15,8 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/filecoin-project/go-jsonrpc/auth"
-	"github.com/filecoin-project/lotus/api/apistruct"
+
+	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/modules"
 )
@@ -98,19 +99,19 @@ var jwtTokenCmd = &cli.Command{
 		perms := []auth.Permission{}
 
 		if cctx.Bool("read") {
-			perms = append(perms, apistruct.PermRead)
+			perms = append(perms, api.PermRead)
 		}
 
 		if cctx.Bool("write") {
-			perms = append(perms, apistruct.PermWrite)
+			perms = append(perms, api.PermWrite)
 		}
 
 		if cctx.Bool("sign") {
-			perms = append(perms, apistruct.PermSign)
+			perms = append(perms, api.PermSign)
 		}
 
 		if cctx.Bool("admin") {
-			perms = append(perms, apistruct.PermAdmin)
+			perms = append(perms, api.PermAdmin)
 		}
 
 		p := modules.JwtPayload{
@@ -152,7 +153,7 @@ var jwtNewCmd = &cli.Command{
 		}
 
 		p := modules.JwtPayload{
-			Allow: apistruct.AllPermissions,
+			Allow: api.AllPermissions,
 		}
 
 		token, err := jwt.Sign(&p, jwt.NewHS256(keyInfo.PrivateKey))
@@ -168,7 +169,7 @@ var jwtNewCmd = &cli.Command{
 
 		defer func() {
 			if err := file.Close(); err != nil {
-				log.Warnf("failed to close output file: %w", err)
+				log.Warnf("failed to close output file: %v", err)
 			}
 		}()
 

cmd/lotus-shed/keyinfo.go

@@ -427,7 +427,7 @@ var keyinfoNewCmd = &cli.Command{
 
 		defer func() {
 			if err := file.Close(); err != nil {
-				log.Warnf("failed to close output file: %w", err)
+				log.Warnf("failed to close output file: %v", err)
 			}
 		}()
 

cmd/lotus-shed/ledger.go

@@ -6,12 +6,14 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/filecoin-project/lotus/api/v0api"
+
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/urfave/cli/v2"
 	ledgerfil "github.com/whyrusleeping/ledger-filecoin-go"
 
-	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/types"
 	ledgerwallet "github.com/filecoin-project/lotus/chain/wallet/ledger"
 	lcli "github.com/filecoin-project/lotus/cli"
@@ -25,6 +27,7 @@ var ledgerCmd = &cli.Command{
 		ledgerListAddressesCmd,
 		ledgerKeyInfoCmd,
 		ledgerSignTestCmd,
+		ledgerShowCmd,
 	},
 }
 
@@ -40,7 +43,7 @@ var ledgerListAddressesCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		var api api.FullNode
+		var api v0api.FullNode
 		if cctx.Bool("print-balances") {
 			a, closer, err := lcli.GetFullNodeAPI(cctx)
 			if err != nil {
@@ -57,6 +60,7 @@ var ledgerListAddressesCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
+		defer fl.Close() // nolint
 
 		end := 20
 		for i := 0; i < end; i++ {
@@ -166,6 +170,7 @@ var ledgerKeyInfoCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
+		defer fl.Close() // nolint
 
 		p, err := parseHDPath(cctx.Args().First())
 		if err != nil {
@@ -242,13 +247,46 @@ var ledgerSignTestCmd = &cli.Command{
 		if err != nil {
 			return err
 		}
+		fmt.Printf("Message: %x\n", b.RawData())
 
 		sig, err := fl.SignSECP256K1(p, b.RawData())
 		if err != nil {
 			return err
 		}
 
-		fmt.Println(sig.SignatureBytes())
+		sigBytes := append([]byte{byte(crypto.SigTypeSecp256k1)}, sig.SignatureBytes()...)
+
+		fmt.Printf("Signature: %x\n", sigBytes)
+
+		return nil
+	},
+}
+
+var ledgerShowCmd = &cli.Command{
+	Name:      "show",
+	ArgsUsage: "[hd path]",
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return cli.ShowCommandHelp(cctx, cctx.Command.Name)
+		}
+
+		fl, err := ledgerfil.FindLedgerFilecoinApp()
+		if err != nil {
+			return err
+		}
+		defer fl.Close() // nolint
+
+		p, err := parseHDPath(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		_, _, a, err := fl.ShowAddressPubKeySECP256K1(p)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(a)
 
 		return nil
 	},

cmd/lotus-shed/main.go

@@ -16,9 +16,11 @@ func main() {
 	logging.SetLogLevel("*", "INFO")
 
 	local := []*cli.Command{
+		base64Cmd,
 		base32Cmd,
 		base16Cmd,
 		bitFieldCmd,
+		cronWcCmd,
 		frozenMinersCmd,
 		keyinfoCmd,
 		jwtCmd,
@@ -30,22 +32,34 @@ func main() {
 		importObjectCmd,
 		commpToCidCmd,
 		fetchParamCmd,
+		postFindCmd,
 		proofsCmd,
 		verifRegCmd,
+		marketCmd,
 		miscCmd,
 		mpoolCmd,
 		genesisVerifyCmd,
 		mathCmd,
+		minerCmd,
 		mpoolStatsCmd,
 		exportChainCmd,
+		exportCarCmd,
 		consensusCmd,
-		serveDealStatsCmd,
+		storageStatsCmd,
 		syncCmd,
 		stateTreePruneCmd,
 		datastoreCmd,
 		ledgerCmd,
 		sectorsCmd,
 		msgCmd,
+		electionCmd,
+		rpcCmd,
+		cidCmd,
+		blockmsgidCmd,
+		signaturesCmd,
+		actorCmd,
+		minerTypesCmd,
+		minerMultisigsCmd,
 	}
 
 	app := &cli.App{

cmd/lotus-shed/market.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"fmt"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var marketCmd = &cli.Command{
+	Name:  "market",
+	Usage: "Interact with the market actor",
+	Flags: []cli.Flag{},
+	Subcommands: []*cli.Command{
+		marketDealFeesCmd,
+	},
+}
+
+var marketDealFeesCmd = &cli.Command{
+	Name:  "get-deal-fees",
+	Usage: "View the storage fees associated with a particular deal or storage provider",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "provider",
+			Usage: "provider whose outstanding fees you'd like to calculate",
+		},
+		&cli.IntFlag{
+			Name:  "dealId",
+			Usage: "deal whose outstanding fees you'd like to calculate",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		ts, err := lcli.LoadTipSet(ctx, cctx, api)
+		if err != nil {
+			return err
+		}
+
+		ht := ts.Height()
+
+		if cctx.IsSet("provider") {
+			p, err := address.NewFromString(cctx.String("provider"))
+			if err != nil {
+				return fmt.Errorf("failed to parse provider: %w", err)
+			}
+
+			deals, err := api.StateMarketDeals(ctx, ts.Key())
+			if err != nil {
+				return err
+			}
+
+			ef := big.Zero()
+			pf := big.Zero()
+			count := 0
+
+			for _, deal := range deals {
+				if deal.Proposal.Provider == p {
+					e, p := deal.Proposal.GetDealFees(ht)
+					ef = big.Add(ef, e)
+					pf = big.Add(pf, p)
+					count++
+				}
+			}
+
+			fmt.Println("Total deals: ", count)
+			fmt.Println("Total earned fees: ", ef)
+			fmt.Println("Total pending fees: ", pf)
+			fmt.Println("Total fees: ", big.Add(ef, pf))
+
+			return nil
+		}
+
+		if dealid := cctx.Int("dealId"); dealid != 0 {
+			deal, err := api.StateMarketStorageDeal(ctx, abi.DealID(dealid), ts.Key())
+			if err != nil {
+				return err
+			}
+
+			ef, pf := deal.Proposal.GetDealFees(ht)
+
+			fmt.Println("Earned fees: ", ef)
+			fmt.Println("Pending fees: ", pf)
+			fmt.Println("Total fees: ", big.Add(ef, pf))
+
+			return nil
+		}
+
+		return xerrors.New("must provide either --provider or --dealId flag")
+	},
+}

cmd/lotus-shed/math.go

@@ -8,8 +8,10 @@ import (
 	"strings"
 
 	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/chain/types"
+	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
 )
 
 var mathCmd = &cli.Command{
@@ -17,6 +19,7 @@ var mathCmd = &cli.Command{
 	Usage: "utility commands around doing math on a list of numbers",
 	Subcommands: []*cli.Command{
 		mathSumCmd,
+		mathAggFeesCmd,
 	},
 }
 
@@ -101,3 +104,30 @@ var mathSumCmd = &cli.Command{
 		return nil
 	},
 }
+
+var mathAggFeesCmd = &cli.Command{
+	Name: "agg-fees",
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Name:     "size",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:     "base-fee",
+			Usage:    "baseFee aFIL",
+			Required: true,
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		as := cctx.Int("size")
+
+		bf, err := types.BigFromString(cctx.String("base-fee"))
+		if err != nil {
+			return xerrors.Errorf("parsing basefee: %w", err)
+		}
+
+		fmt.Println(types.FIL(miner5.AggregateNetworkFee(as, bf)))
+
+		return nil
+	},
+}

cmd/lotus-shed/mempool-stats.go

@@ -8,7 +8,7 @@ import (
 
 	"contrib.go.opencensus.io/exporter/prometheus"
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/urfave/cli/v2"
 	"go.opencensus.io/stats"
 	"go.opencensus.io/stats/view"

cmd/lotus-shed/miner-multisig.go

@@ -0,0 +1,388 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
+
+	msig5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/multisig"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var minerMultisigsCmd = &cli.Command{
+	Name:        "miner-multisig",
+	Description: "a collection of utilities for using multisigs as owner addresses of miners",
+	Subcommands: []*cli.Command{
+		mmProposeWithdrawBalance,
+		mmApproveWithdrawBalance,
+		mmProposeChangeOwner,
+		mmApproveChangeOwner,
+	},
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:     "from",
+			Usage:    "specify address to send message from",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:     "multisig",
+			Usage:    "specify multisig that will receive the message",
+			Required: true,
+		},
+		&cli.StringFlag{
+			Name:     "miner",
+			Usage:    "specify miner being acted upon",
+			Required: true,
+		},
+	},
+}
+
+var mmProposeWithdrawBalance = &cli.Command{
+	Name:      "propose-withdraw",
+	Usage:     "Propose to withdraw FIL from the miner",
+	ArgsUsage: "[amount]",
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass amount to withdraw")
+		}
+
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		multisigAddr, sender, minerAddr, err := getInputs(cctx)
+		if err != nil {
+			return err
+		}
+
+		val, err := types.ParseFIL(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		sp, err := actors.SerializeParams(&miner5.WithdrawBalanceParams{
+			AmountRequested: abi.TokenAmount(val),
+		})
+		if err != nil {
+			return err
+		}
+
+		pcid, err := api.MsigPropose(ctx, multisigAddr, minerAddr, big.Zero(), sender, uint64(miner.Methods.WithdrawBalance), sp)
+		if err != nil {
+			return xerrors.Errorf("proposing message: %w", err)
+		}
+
+		fmt.Fprintln(cctx.App.Writer, "Propose Message CID:", pcid)
+
+		// wait for it to get mined into a block
+		wait, err := api.StateWaitMsg(ctx, pcid, build.MessageConfidence)
+		if err != nil {
+			return err
+		}
+
+		// check it executed successfully
+		if wait.Receipt.ExitCode != 0 {
+			fmt.Fprintln(cctx.App.Writer, "Propose owner change tx failed!")
+			return err
+		}
+
+		var retval msig5.ProposeReturn
+		if err := retval.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil {
+			return fmt.Errorf("failed to unmarshal propose return value: %w", err)
+		}
+
+		fmt.Printf("Transaction ID: %d\n", retval.TxnID)
+		if retval.Applied {
+			fmt.Printf("Transaction was executed during propose\n")
+			fmt.Printf("Exit Code: %d\n", retval.Code)
+			fmt.Printf("Return Value: %x\n", retval.Ret)
+		}
+
+		return nil
+	},
+}
+
+var mmApproveWithdrawBalance = &cli.Command{
+	Name:      "approve-withdraw",
+	Usage:     "Approve to withdraw FIL from the miner",
+	ArgsUsage: "[amount txnId proposer]",
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 3 {
+			return fmt.Errorf("must pass amount, txn Id, and proposer address")
+		}
+
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		multisigAddr, sender, minerAddr, err := getInputs(cctx)
+		if err != nil {
+			return err
+		}
+
+		val, err := types.ParseFIL(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		sp, err := actors.SerializeParams(&miner5.WithdrawBalanceParams{
+			AmountRequested: abi.TokenAmount(val),
+		})
+		if err != nil {
+			return err
+		}
+
+		txid, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64)
+		if err != nil {
+			return err
+		}
+
+		proposer, err := address.NewFromString(cctx.Args().Get(2))
+		if err != nil {
+			return err
+		}
+
+		acid, err := api.MsigApproveTxnHash(ctx, multisigAddr, txid, proposer, minerAddr, big.Zero(), sender, uint64(miner.Methods.WithdrawBalance), sp)
+		if err != nil {
+			return xerrors.Errorf("approving message: %w", err)
+		}
+
+		fmt.Fprintln(cctx.App.Writer, "Approve Message CID:", acid)
+
+		// wait for it to get mined into a block
+		wait, err := api.StateWaitMsg(ctx, acid, build.MessageConfidence)
+		if err != nil {
+			return err
+		}
+
+		// check it executed successfully
+		if wait.Receipt.ExitCode != 0 {
+			fmt.Fprintln(cctx.App.Writer, "Approve owner change tx failed!")
+			return err
+		}
+
+		var retval msig5.ApproveReturn
+		if err := retval.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil {
+			return fmt.Errorf("failed to unmarshal approve return value: %w", err)
+		}
+
+		if retval.Applied {
+			fmt.Printf("Transaction was executed with the approve\n")
+			fmt.Printf("Exit Code: %d\n", retval.Code)
+			fmt.Printf("Return Value: %x\n", retval.Ret)
+		} else {
+			fmt.Println("Transaction was approved, but not executed")
+		}
+		return nil
+	},
+}
+
+var mmProposeChangeOwner = &cli.Command{
+	Name:      "propose-change-owner",
+	Usage:     "Propose an owner address change",
+	ArgsUsage: "[newOwner]",
+	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass new owner address")
+		}
+
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		multisigAddr, sender, minerAddr, err := getInputs(cctx)
+		if err != nil {
+			return err
+		}
+
+		na, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		newAddr, err := api.StateLookupID(ctx, na, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		mi, err := api.StateMinerInfo(ctx, minerAddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		if mi.Owner == newAddr {
+			return fmt.Errorf("owner address already set to %s", na)
+		}
+
+		sp, err := actors.SerializeParams(&newAddr)
+		if err != nil {
+			return xerrors.Errorf("serializing params: %w", err)
+		}
+
+		pcid, err := api.MsigPropose(ctx, multisigAddr, minerAddr, big.Zero(), sender, uint64(miner.Methods.ChangeOwnerAddress), sp)
+		if err != nil {
+			return xerrors.Errorf("proposing message: %w", err)
+		}
+
+		fmt.Fprintln(cctx.App.Writer, "Propose Message CID:", pcid)
+
+		// wait for it to get mined into a block
+		wait, err := api.StateWaitMsg(ctx, pcid, build.MessageConfidence)
+		if err != nil {
+			return err
+		}
+
+		// check it executed successfully
+		if wait.Receipt.ExitCode != 0 {
+			fmt.Fprintln(cctx.App.Writer, "Propose owner change tx failed!")
+			return err
+		}
+
+		var retval msig5.ProposeReturn
+		if err := retval.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil {
+			return fmt.Errorf("failed to unmarshal propose return value: %w", err)
+		}
+
+		fmt.Printf("Transaction ID: %d\n", retval.TxnID)
+		if retval.Applied {
+			fmt.Printf("Transaction was executed during propose\n")
+			fmt.Printf("Exit Code: %d\n", retval.Code)
+			fmt.Printf("Return Value: %x\n", retval.Ret)
+		}
+		return nil
+	},
+}
+
+var mmApproveChangeOwner = &cli.Command{
+	Name:      "approve-change-owner",
+	Usage:     "Approve an owner address change",
+	ArgsUsage: "[newOwner txnId proposer]",
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 3 {
+			return fmt.Errorf("must pass new owner address, txn Id, and proposer address")
+		}
+
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		multisigAddr, sender, minerAddr, err := getInputs(cctx)
+		if err != nil {
+			return err
+		}
+
+		na, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		newAddr, err := api.StateLookupID(ctx, na, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		txid, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64)
+		if err != nil {
+			return err
+		}
+
+		proposer, err := address.NewFromString(cctx.Args().Get(2))
+		if err != nil {
+			return err
+		}
+
+		mi, err := api.StateMinerInfo(ctx, minerAddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		if mi.Owner == newAddr {
+			return fmt.Errorf("owner address already set to %s", na)
+		}
+
+		sp, err := actors.SerializeParams(&newAddr)
+		if err != nil {
+			return xerrors.Errorf("serializing params: %w", err)
+		}
+
+		acid, err := api.MsigApproveTxnHash(ctx, multisigAddr, txid, proposer, minerAddr, big.Zero(), sender, uint64(miner.Methods.ChangeOwnerAddress), sp)
+		if err != nil {
+			return xerrors.Errorf("approving message: %w", err)
+		}
+
+		fmt.Fprintln(cctx.App.Writer, "Approve Message CID:", acid)
+
+		// wait for it to get mined into a block
+		wait, err := api.StateWaitMsg(ctx, acid, build.MessageConfidence)
+		if err != nil {
+			return err
+		}
+
+		// check it executed successfully
+		if wait.Receipt.ExitCode != 0 {
+			fmt.Fprintln(cctx.App.Writer, "Approve owner change tx failed!")
+			return err
+		}
+
+		var retval msig5.ApproveReturn
+		if err := retval.UnmarshalCBOR(bytes.NewReader(wait.Receipt.Return)); err != nil {
+			return fmt.Errorf("failed to unmarshal approve return value: %w", err)
+		}
+
+		if retval.Applied {
+			fmt.Printf("Transaction was executed with the approve\n")
+			fmt.Printf("Exit Code: %d\n", retval.Code)
+			fmt.Printf("Return Value: %x\n", retval.Ret)
+		} else {
+			fmt.Println("Transaction was approved, but not executed")
+		}
+		return nil
+	},
+}
+
+func getInputs(cctx *cli.Context) (address.Address, address.Address, address.Address, error) {
+	multisigAddr, err := address.NewFromString(cctx.String("multisig"))
+	if err != nil {
+		return address.Undef, address.Undef, address.Undef, err
+	}
+
+	sender, err := address.NewFromString(cctx.String("from"))
+	if err != nil {
+		return address.Undef, address.Undef, address.Undef, err
+	}
+
+	minerAddr, err := address.NewFromString(cctx.String("miner"))
+	if err != nil {
+		return address.Undef, address.Undef, address.Undef, err
+	}
+
+	return multisigAddr, sender, minerAddr, nil
+}

cmd/lotus-shed/miner-types.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/big"
+
+	big2 "github.com/filecoin-project/go-state-types/big"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
+	"github.com/filecoin-project/lotus/chain/state"
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/lotus/node/repo"
+	builtin4 "github.com/filecoin-project/specs-actors/v4/actors/builtin"
+	"github.com/filecoin-project/specs-actors/v4/actors/util/adt"
+	"github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var minerTypesCmd = &cli.Command{
+	Name:  "miner-types",
+	Usage: "Scrape state to report on how many miners of each WindowPoStProofType exist", Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "repo",
+			Value: "~/.lotus",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := context.TODO()
+
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass state root")
+		}
+
+		sroot, err := cid.Decode(cctx.Args().First())
+		if err != nil {
+			return fmt.Errorf("failed to parse input: %w", err)
+		}
+
+		fsrepo, err := repo.NewFS(cctx.String("repo"))
+		if err != nil {
+			return err
+		}
+
+		lkrepo, err := fsrepo.Lock(repo.FullNode)
+		if err != nil {
+			return err
+		}
+
+		defer lkrepo.Close() //nolint:errcheck
+
+		bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
+		if err != nil {
+			return fmt.Errorf("failed to open blockstore: %w", err)
+		}
+
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
+
+		mds, err := lkrepo.Datastore(context.Background(), "/metadata")
+		if err != nil {
+			return err
+		}
+
+		cs := store.NewChainStore(bs, bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
+		defer cs.Close() //nolint:errcheck
+
+		cst := cbor.NewCborStore(bs)
+		store := adt.WrapStore(ctx, cst)
+
+		tree, err := state.LoadStateTree(cst, sroot)
+		if err != nil {
+			return err
+		}
+
+		typeMap := make(map[abi.RegisteredPoStProof]int64)
+		pa, err := tree.GetActor(power.Address)
+		if err != nil {
+			return err
+		}
+
+		ps, err := power.Load(store, pa)
+		if err != nil {
+			return err
+		}
+
+		dc := 0
+		dz := power.Claim{
+			RawBytePower:    abi.NewStoragePower(0),
+			QualityAdjPower: abi.NewStoragePower(0),
+		}
+
+		err = tree.ForEach(func(addr address.Address, act *types.Actor) error {
+			if act.Code == builtin4.StorageMinerActorCodeID {
+				ms, err := miner.Load(store, act)
+				if err != nil {
+					return err
+				}
+
+				mi, err := ms.Info()
+				if err != nil {
+					return err
+				}
+
+				if mi.WindowPoStProofType == abi.RegisteredPoStProof_StackedDrgWindow64GiBV1 {
+					mp, f, err := ps.MinerPower(addr)
+					if err != nil {
+						return err
+					}
+
+					if f && mp.RawBytePower.Cmp(big.NewInt(10<<40)) >= 0 && mp.RawBytePower.Cmp(big.NewInt(20<<40)) < 0 {
+						dc = dc + 1
+						dz.RawBytePower = big2.Add(dz.RawBytePower, mp.RawBytePower)
+						dz.QualityAdjPower = big2.Add(dz.QualityAdjPower, mp.QualityAdjPower)
+					}
+				}
+
+				c, f := typeMap[mi.WindowPoStProofType]
+				if !f {
+					typeMap[mi.WindowPoStProofType] = 1
+				} else {
+					typeMap[mi.WindowPoStProofType] = c + 1
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			return xerrors.Errorf("failed to loop over actors: %w", err)
+		}
+
+		for k, v := range typeMap {
+			fmt.Println("Type:", k, " Count: ", v)
+		}
+
+		fmt.Println("Mismatched power (raw, QA): ", dz.RawBytePower, " ", dz.QualityAdjPower)
+		fmt.Println("Mismatched 64 GiB miner count: ", dc)
+
+		return nil
+	},
+}

cmd/lotus-shed/miner.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mitchellh/go-homedir"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var minerCmd = &cli.Command{
+	Name:  "miner",
+	Usage: "miner-related utilities",
+	Subcommands: []*cli.Command{
+		minerUnpackInfoCmd,
+	},
+}
+
+var minerUnpackInfoCmd = &cli.Command{
+	Name:      "unpack-info",
+	Usage:     "unpack miner info all dump",
+	ArgsUsage: "[allinfo.txt] [dir]",
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 2 {
+			return xerrors.Errorf("expected 2 args")
+		}
+
+		src, err := homedir.Expand(cctx.Args().Get(0))
+		if err != nil {
+			return xerrors.Errorf("expand src: %w", err)
+		}
+
+		f, err := os.Open(src)
+		if err != nil {
+			return xerrors.Errorf("open file: %w", err)
+		}
+		defer f.Close() // nolint
+
+		dest, err := homedir.Expand(cctx.Args().Get(1))
+		if err != nil {
+			return xerrors.Errorf("expand dest: %w", err)
+		}
+
+		var outf *os.File
+
+		r := bufio.NewReader(f)
+		for {
+			l, _, err := r.ReadLine()
+			if err == io.EOF {
+				if outf != nil {
+					return outf.Close()
+				}
+			}
+			if err != nil {
+				return xerrors.Errorf("read line: %w", err)
+			}
+			sl := string(l)
+
+			if strings.HasPrefix(sl, "#") {
+				if strings.Contains(sl, "..") {
+					return xerrors.Errorf("bad name %s", sl)
+				}
+
+				if strings.HasPrefix(sl, "#: ") {
+					if outf != nil {
+						if err := outf.Close(); err != nil {
+							return xerrors.Errorf("close out file: %w", err)
+						}
+					}
+					p := filepath.Join(dest, sl[len("#: "):])
+					if err := os.MkdirAll(filepath.Dir(p), 0775); err != nil {
+						return xerrors.Errorf("mkdir: %w", err)
+					}
+					outf, err = os.Create(p)
+					if err != nil {
+						return xerrors.Errorf("create out file: %w", err)
+					}
+					continue
+				}
+
+				if strings.HasPrefix(sl, "##: ") {
+					if outf != nil {
+						if err := outf.Close(); err != nil {
+							return xerrors.Errorf("close out file: %w", err)
+						}
+					}
+					p := filepath.Join(dest, "Per Sector Infos", sl[len("##: "):])
+					if err := os.MkdirAll(filepath.Dir(p), 0775); err != nil {
+						return xerrors.Errorf("mkdir: %w", err)
+					}
+					outf, err = os.Create(p)
+					if err != nil {
+						return xerrors.Errorf("create out file: %w", err)
+					}
+					continue
+				}
+			}
+
+			if outf != nil {
+				if _, err := outf.Write(l); err != nil {
+					return xerrors.Errorf("write line: %w", err)
+				}
+				if _, err := outf.Write([]byte("\n")); err != nil {
+					return xerrors.Errorf("write line end: %w", err)
+				}
+			}
+		}
+	},
+}

cmd/lotus-shed/mpool.go

@@ -15,6 +15,7 @@ var mpoolCmd = &cli.Command{
 	Flags: []cli.Flag{},
 	Subcommands: []*cli.Command{
 		minerSelectMsgsCmd,
+		mpoolClear,
 	},
 }
 
@@ -66,3 +67,36 @@ var minerSelectMsgsCmd = &cli.Command{
 		return nil
 	},
 }
+
+var mpoolClear = &cli.Command{
+	Name:  "clear",
+	Usage: "Clear all pending messages from the mpool (USE WITH CARE)",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "local",
+			Usage: "also clear local messages",
+		},
+		&cli.BoolFlag{
+			Name:  "really-do-it",
+			Usage: "must be specified for the action to take effect",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		really := cctx.Bool("really-do-it")
+		if !really {
+			//nolint:golint
+			return fmt.Errorf("--really-do-it must be specified for this action to have an effect; you have been warned")
+		}
+
+		local := cctx.Bool("local")
+
+		ctx := lcli.ReqContext(cctx)
+		return api.MpoolClear(ctx, local)
+	},
+}

cmd/lotus-shed/params.go

@@ -25,7 +25,7 @@ var fetchParamCmd = &cli.Command{
 			return err
 		}
 		sectorSize := uint64(sectorSizeInt)
-		err = paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), sectorSize)
+		err = paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), build.SrsJSON(), sectorSize)
 		if err != nil {
 			return xerrors.Errorf("fetching proof parameters: %w", err)
 		}

cmd/lotus-shed/postfind.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	lapi "github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/specs-actors/v2/actors/builtin"
+	"github.com/urfave/cli/v2"
+)
+
+var postFindCmd = &cli.Command{
+	Name:        "post-find",
+	Description: "return addresses of all miners who have over zero power and have posted in the last day",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "tipset",
+			Usage: "specify tipset state to search on",
+		},
+		&cli.BoolFlag{
+			Name:  "verbose",
+			Usage: "get more frequent print updates",
+		},
+		&cli.BoolFlag{
+			Name:  "withpower",
+			Usage: "only print addrs of miners with more than zero power",
+		},
+		&cli.IntFlag{
+			Name:  "lookback",
+			Usage: "number of past epochs to search for post",
+			Value: 2880, //default 1 day
+		},
+	},
+	Action: func(c *cli.Context) error {
+		api, acloser, err := lcli.GetFullNodeAPI(c)
+		if err != nil {
+			return err
+		}
+		defer acloser()
+		ctx := lcli.ReqContext(c)
+		verbose := c.Bool("verbose")
+		withpower := c.Bool("withpower")
+
+		startTs, err := lcli.LoadTipSet(ctx, c, api)
+		if err != nil {
+			return err
+		}
+		stopEpoch := startTs.Height() - abi.ChainEpoch(c.Int("lookback"))
+		if verbose {
+			fmt.Printf("Collecting messages between %d and %d\n", startTs.Height(), stopEpoch)
+		}
+		// Get all messages over the last day
+		ts := startTs
+		msgs := make([]*types.Message, 0)
+		for ts.Height() > stopEpoch {
+			// Get messages on ts parent
+			next, err := api.ChainGetParentMessages(ctx, ts.Cids()[0])
+			if err != nil {
+				return err
+			}
+			msgs = append(msgs, messagesFromAPIMessages(next)...)
+
+			// Next ts
+			ts, err = api.ChainGetTipSet(ctx, ts.Parents())
+			if err != nil {
+				return err
+			}
+			if verbose && int64(ts.Height())%100 == 0 {
+				fmt.Printf("Collected messages back to height %d\n", ts.Height())
+			}
+		}
+		fmt.Printf("Loaded messages to height %d\n", ts.Height())
+
+		mAddrs, err := api.StateListMiners(ctx, startTs.Key())
+		if err != nil {
+			return err
+		}
+
+		minersToCheck := make(map[address.Address]struct{})
+		for _, mAddr := range mAddrs {
+			// if they have no power ignore. This filters out 14k inactive miners
+			// so we can do 100x fewer expensive message queries
+			if withpower {
+				power, err := api.StateMinerPower(ctx, mAddr, startTs.Key())
+				if err != nil {
+					return err
+				}
+				if power.MinerPower.RawBytePower.GreaterThan(big.Zero()) {
+					minersToCheck[mAddr] = struct{}{}
+				}
+			} else {
+				minersToCheck[mAddr] = struct{}{}
+			}
+		}
+		fmt.Printf("Loaded %d miners to check\n", len(minersToCheck))
+
+		postedMiners := make(map[address.Address]struct{})
+		for _, msg := range msgs {
+			_, shouldCheck := minersToCheck[msg.To]
+			_, seenBefore := postedMiners[msg.To]
+
+			if shouldCheck && !seenBefore {
+				if msg.Method == builtin.MethodsMiner.SubmitWindowedPoSt {
+					fmt.Printf("%s\n", msg.To)
+					postedMiners[msg.To] = struct{}{}
+				}
+			}
+		}
+		return nil
+	},
+}
+
+func messagesFromAPIMessages(apiMessages []lapi.Message) []*types.Message {
+	messages := make([]*types.Message, len(apiMessages))
+	for i, apiMessage := range apiMessages {
+		messages[i] = apiMessage.Message
+	}
+	return messages
+}

cmd/lotus-shed/pruning.go

@@ -3,20 +3,19 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/ipfs/bbloom"
+	"github.com/ipfs/go-cid"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
-	"github.com/filecoin-project/lotus/lib/blockstore"
 	"github.com/filecoin-project/lotus/node/repo"
-	"github.com/ipfs/bbloom"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/query"
-	dshelp "github.com/ipfs/go-ipfs-ds-help"
-	"github.com/urfave/cli/v2"
-	"golang.org/x/xerrors"
 )
 
 type cidSet interface {
@@ -132,37 +131,47 @@ var stateTreePruneCmd = &cli.Command{
 
 		defer lkrepo.Close() //nolint:errcheck
 
-		ds, err := lkrepo.Datastore("/chain")
+		bs, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to open blockstore: %w", err)
 		}
 
-		defer ds.Close() //nolint:errcheck
+		defer func() {
+			if c, ok := bs.(io.Closer); ok {
+				if err := c.Close(); err != nil {
+					log.Warnf("failed to close blockstore: %s", err)
+				}
+			}
+		}()
 
-		mds, err := lkrepo.Datastore("/metadata")
+		// After migrating to native blockstores, this has been made
+		// database-specific.
+		badgbs, ok := bs.(*badgerbs.Blockstore)
+		if !ok {
+			return fmt.Errorf("only badger blockstores are supported")
+		}
+
+		mds, err := lkrepo.Datastore(context.Background(), "/metadata")
 		if err != nil {
 			return err
 		}
 		defer mds.Close() //nolint:errcheck
 
+		const DiscardRatio = 0.2
 		if cctx.Bool("only-ds-gc") {
-			gcds, ok := ds.(datastore.GCDatastore)
-			if ok {
-				fmt.Println("running datastore gc....")
-				for i := 0; i < cctx.Int("gc-count"); i++ {
-					if err := gcds.CollectGarbage(); err != nil {
-						return xerrors.Errorf("datastore GC failed: %w", err)
-					}
+			fmt.Println("running datastore gc....")
+			for i := 0; i < cctx.Int("gc-count"); i++ {
+				if err := badgbs.DB.RunValueLogGC(DiscardRatio); err != nil {
+					return xerrors.Errorf("datastore GC failed: %w", err)
 				}
-				fmt.Println("gc complete!")
-				return nil
 			}
-			return fmt.Errorf("datastore doesnt support gc")
+			fmt.Println("gc complete!")
+			return nil
 		}
 
-		bs := blockstore.NewBlockstore(ds)
+		cs := store.NewChainStore(bs, bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
+		defer cs.Close() //nolint:errcheck
 
-		cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier), nil)
 		if err := cs.Load(); err != nil {
 			return fmt.Errorf("loading chainstore: %w", err)
 		}
@@ -182,7 +191,7 @@ var stateTreePruneCmd = &cli.Command{
 
 		rrLb := abi.ChainEpoch(cctx.Int64("keep-from-lookback"))
 
-		if err := cs.WalkSnapshot(ctx, ts, rrLb, true, func(c cid.Cid) error {
+		if err := cs.WalkSnapshot(ctx, ts, rrLb, true, true, func(c cid.Cid) error {
 			if goodSet.Len()%20 == 0 {
 				fmt.Printf("\renumerating keep set: %d             ", goodSet.Len())
 			}
@@ -199,63 +208,30 @@ var stateTreePruneCmd = &cli.Command{
 			return nil
 		}
 
-		var b datastore.Batch
-		var batchCount int
+		b := badgbs.DB.NewWriteBatch()
+		defer b.Cancel()
+
 		markForRemoval := func(c cid.Cid) error {
-			if b == nil {
-				nb, err := ds.Batch()
-				if err != nil {
-					return fmt.Errorf("opening batch: %w", err)
-				}
-
-				b = nb
-			}
-			batchCount++
-
-			if err := b.Delete(dshelp.MultihashToDsKey(c.Hash())); err != nil {
-				return err
-			}
-
-			if batchCount > 100 {
-				if err := b.Commit(); err != nil {
-					return xerrors.Errorf("failed to commit batch deletes: %w", err)
-				}
-				b = nil
-				batchCount = 0
-			}
-			return nil
+			return b.Delete(badgbs.StorageKey(nil, c))
 		}
 
-		res, err := ds.Query(query.Query{KeysOnly: true})
+		keys, err := bs.AllKeysChan(context.Background())
 		if err != nil {
-			return xerrors.Errorf("failed to query datastore: %w", err)
+			return xerrors.Errorf("failed to query blockstore: %w", err)
 		}
 
 		dupTo := cctx.Int("delete-up-to")
 
 		var deleteCount int
 		var goodHits int
-		for {
-			v, ok := res.NextSync()
-			if !ok {
-				break
-			}
-
-			bk, err := dshelp.BinaryFromDsKey(datastore.RawKey(v.Key[len("/blocks"):]))
-			if err != nil {
-				return xerrors.Errorf("failed to parse key: %w", err)
-			}
-
-			if goodSet.HasRaw(bk) {
+		for k := range keys {
+			if goodSet.HasRaw(k.Bytes()) {
 				goodHits++
 				continue
 			}
 
-			nc := cid.NewCidV1(cid.Raw, bk)
-
-			deleteCount++
-			if err := markForRemoval(nc); err != nil {
-				return fmt.Errorf("failed to remove cid %s: %w", nc, err)
+			if err := markForRemoval(k); err != nil {
+				return fmt.Errorf("failed to remove cid %s: %w", k, err)
 			}
 
 			if deleteCount%20 == 0 {
@@ -267,22 +243,17 @@ var stateTreePruneCmd = &cli.Command{
 			}
 		}
 
-		if b != nil {
-			if err := b.Commit(); err != nil {
-				return xerrors.Errorf("failed to commit final batch delete: %w", err)
-			}
+		if err := b.Flush(); err != nil {
+			return xerrors.Errorf("failed to flush final batch delete: %w", err)
 		}
 
-		gcds, ok := ds.(datastore.GCDatastore)
-		if ok {
-			fmt.Println("running datastore gc....")
-			for i := 0; i < cctx.Int("gc-count"); i++ {
-				if err := gcds.CollectGarbage(); err != nil {
-					return xerrors.Errorf("datastore GC failed: %w", err)
-				}
+		fmt.Println("running datastore gc....")
+		for i := 0; i < cctx.Int("gc-count"); i++ {
+			if err := badgbs.DB.RunValueLogGC(DiscardRatio); err != nil {
+				return xerrors.Errorf("datastore GC failed: %w", err)
 			}
-			fmt.Println("gc complete!")
 		}
+		fmt.Println("gc complete!")
 
 		return nil
 	},

cmd/lotus-shed/rpc.go

@@ -0,0 +1,172 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"text/scanner"
+
+	"github.com/chzyer/readline"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+var rpcCmd = &cli.Command{
+	Name:  "rpc",
+	Usage: "Interactive JsonPRC shell",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name: "miner",
+		},
+		&cli.StringFlag{
+			Name:  "version",
+			Value: "v0",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		rt := repo.FullNode
+		if cctx.Bool("miner") {
+			rt = repo.StorageMiner
+		}
+
+		addr, headers, err := lcli.GetRawAPI(cctx, rt, cctx.String("version"))
+		if err != nil {
+			return err
+		}
+
+		u, err := url.Parse(addr)
+		if err != nil {
+			return xerrors.Errorf("parsing api URL: %w", err)
+		}
+
+		switch u.Scheme {
+		case "ws":
+			u.Scheme = "http"
+		case "wss":
+			u.Scheme = "https"
+		}
+
+		addr = u.String()
+
+		ctx := lcli.ReqContext(cctx)
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		afmt := lcli.NewAppFmt(cctx.App)
+
+		cs := readline.NewCancelableStdin(afmt.Stdin)
+		go func() {
+			<-ctx.Done()
+			cs.Close() // nolint:errcheck
+		}()
+
+		send := func(method, params string) error {
+			jreq, err := json.Marshal(struct {
+				Jsonrpc string          `json:"jsonrpc"`
+				ID      int             `json:"id"`
+				Method  string          `json:"method"`
+				Params  json.RawMessage `json:"params"`
+			}{
+				Jsonrpc: "2.0",
+				Method:  "Filecoin." + method,
+				Params:  json.RawMessage(params),
+				ID:      0,
+			})
+			if err != nil {
+				return err
+			}
+
+			req, err := http.NewRequest("POST", addr, bytes.NewReader(jreq))
+			if err != nil {
+				return err
+			}
+			req.Header = headers
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				return err
+			}
+
+			rb, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				return err
+			}
+
+			fmt.Println(string(rb))
+
+			if err := resp.Body.Close(); err != nil {
+				return err
+			}
+
+			return nil
+		}
+
+		if cctx.Args().Present() {
+			if cctx.Args().Len() > 2 {
+				return xerrors.Errorf("expected 1 or 2 arguments: method [params]")
+			}
+
+			params := cctx.Args().Get(1)
+			if params == "" {
+				// TODO: try to be smart and use zero-values for method
+				params = "[]"
+			}
+
+			return send(cctx.Args().Get(0), params)
+		}
+
+		cctx.App.Metadata["repoType"] = repo.FullNode
+		if err := lcli.VersionCmd.Action(cctx); err != nil {
+			return err
+		}
+		fmt.Println("Usage: > Method [Param1, Param2, ...]")
+
+		rl, err := readline.NewEx(&readline.Config{
+			Stdin:             cs,
+			HistoryFile:       "/tmp/lotusrpc.tmp",
+			Prompt:            "> ",
+			EOFPrompt:         "exit",
+			HistorySearchFold: true,
+
+			// TODO: Some basic auto completion
+		})
+		if err != nil {
+			return err
+		}
+
+		for {
+			line, err := rl.Readline()
+			if err == readline.ErrInterrupt {
+				if len(line) == 0 {
+					break
+				} else {
+					continue
+				}
+			} else if err == io.EOF {
+				break
+			}
+
+			var s scanner.Scanner
+			s.Init(strings.NewReader(line))
+			s.Scan()
+			method := s.TokenText()
+
+			s.Scan()
+			params := line[s.Position.Offset:]
+
+			if err := send(method, params); err != nil {
+				_, _ = fmt.Fprintf(os.Stderr, "%v", err)
+			}
+		}
+
+		return nil
+	},
+}

cmd/lotus-shed/sectors.go

@@ -6,6 +6,7 @@ import (
 
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
@@ -25,6 +26,7 @@ var sectorsCmd = &cli.Command{
 	Flags: []cli.Flag{},
 	Subcommands: []*cli.Command{
 		terminateSectorCmd,
+		terminateSectorPenaltyEstimationCmd,
 	},
 }
 
@@ -33,6 +35,10 @@ var terminateSectorCmd = &cli.Command{
 	Usage:     "Forcefully terminate a sector (WARNING: This means losing power and pay a one-time termination penalty(including collateral) for the terminated sector)",
 	ArgsUsage: "[sectorNum1 sectorNum2 ...]",
 	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "actor",
+			Usage: "specify the address of miner actor",
+		},
 		&cli.BoolFlag{
 			Name:  "really-do-it",
 			Usage: "pass this flag if you know what you are doing",
@@ -43,6 +49,15 @@ var terminateSectorCmd = &cli.Command{
 			return fmt.Errorf("at least one sector must be specified")
 		}
 
+		var maddr address.Address
+		if act := cctx.String("actor"); act != "" {
+			var err error
+			maddr, err = address.NewFromString(act)
+			if err != nil {
+				return fmt.Errorf("parsing address %s: %w", act, err)
+			}
+		}
+
 		if !cctx.Bool("really-do-it") {
 			return fmt.Errorf("this is a command for advanced users, only use it if you are sure of what you are doing")
 		}
@@ -53,17 +68,19 @@ var terminateSectorCmd = &cli.Command{
 		}
 		defer closer()
 
-		api, acloser, err := lcli.GetStorageMinerAPI(cctx)
-		if err != nil {
-			return err
-		}
-		defer acloser()
-
 		ctx := lcli.ReqContext(cctx)
 
-		maddr, err := api.ActorAddress(ctx)
-		if err != nil {
-			return err
+		if maddr.Empty() {
+			api, acloser, err := lcli.GetStorageMinerAPI(cctx)
+			if err != nil {
+				return err
+			}
+			defer acloser()
+
+			maddr, err = api.ActorAddress(ctx)
+			if err != nil {
+				return err
+			}
 		}
 
 		mi, err := nodeApi.StateMinerInfo(ctx, maddr, types.EmptyTSK)
@@ -131,3 +148,118 @@ var terminateSectorCmd = &cli.Command{
 		return nil
 	},
 }
+
+func findPenaltyInInternalExecutions(prefix string, trace []types.ExecutionTrace) {
+	for _, im := range trace {
+		if im.Msg.To.String() == "f099" /*Burn actor*/ {
+			fmt.Printf("Estimated termination penalty: %s attoFIL\n", im.Msg.Value)
+			return
+		}
+		findPenaltyInInternalExecutions(prefix+"\t", im.Subcalls)
+	}
+}
+
+var terminateSectorPenaltyEstimationCmd = &cli.Command{
+	Name:      "termination-estimate",
+	Usage:     "Estimate the termination penalty",
+	ArgsUsage: "[sectorNum1 sectorNum2 ...]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "actor",
+			Usage: "specify the address of miner actor",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() < 1 {
+			return fmt.Errorf("at least one sector must be specified")
+		}
+
+		var maddr address.Address
+		if act := cctx.String("actor"); act != "" {
+			var err error
+			maddr, err = address.NewFromString(act)
+			if err != nil {
+				return fmt.Errorf("parsing address %s: %w", act, err)
+			}
+		}
+
+		nodeApi, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		if maddr.Empty() {
+			api, acloser, err := lcli.GetStorageMinerAPI(cctx)
+			if err != nil {
+				return err
+			}
+			defer acloser()
+
+			maddr, err = api.ActorAddress(ctx)
+			if err != nil {
+				return err
+			}
+		}
+
+		mi, err := nodeApi.StateMinerInfo(ctx, maddr, types.EmptyTSK)
+		if err != nil {
+			return err
+		}
+
+		terminationDeclarationParams := []miner2.TerminationDeclaration{}
+
+		for _, sn := range cctx.Args().Slice() {
+			sectorNum, err := strconv.ParseUint(sn, 10, 64)
+			if err != nil {
+				return fmt.Errorf("could not parse sector number: %w", err)
+			}
+
+			sectorbit := bitfield.New()
+			sectorbit.Set(sectorNum)
+
+			loca, err := nodeApi.StateSectorPartition(ctx, maddr, abi.SectorNumber(sectorNum), types.EmptyTSK)
+			if err != nil {
+				return fmt.Errorf("get state sector partition %s", err)
+			}
+
+			para := miner2.TerminationDeclaration{
+				Deadline:  loca.Deadline,
+				Partition: loca.Partition,
+				Sectors:   sectorbit,
+			}
+
+			terminationDeclarationParams = append(terminationDeclarationParams, para)
+		}
+
+		terminateSectorParams := &miner2.TerminateSectorsParams{
+			Terminations: terminationDeclarationParams,
+		}
+
+		sp, err := actors.SerializeParams(terminateSectorParams)
+		if err != nil {
+			return xerrors.Errorf("serializing params: %w", err)
+		}
+
+		msg := &types.Message{
+			From:   mi.Owner,
+			To:     maddr,
+			Method: miner.Methods.TerminateSectors,
+
+			Value:  big.Zero(),
+			Params: sp,
+		}
+
+		//TODO: 4667 add an option to give a more precise estimation with pending termination penalty excluded
+
+		invocResult, err := nodeApi.StateCall(ctx, msg, types.EmptyTSK)
+		if err != nil {
+			return xerrors.Errorf("fail to state call: %w", err)
+		}
+
+		findPenaltyInInternalExecutions("\t", invocResult.ExecutionTrace.Subcalls)
+		return nil
+	},
+}

cmd/lotus-shed/signatures.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"encoding/hex"
+	"fmt"
+	"strconv"
+
+	ffi "github.com/filecoin-project/filecoin-ffi"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/lotus/lib/sigs"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var signaturesCmd = &cli.Command{
+	Name:  "signatures",
+	Usage: "tools involving signatures",
+	Subcommands: []*cli.Command{
+		sigsVerifyVoteCmd,
+		sigsVerifyBlsMsgsCmd,
+	},
+}
+
+var sigsVerifyBlsMsgsCmd = &cli.Command{
+	Name:        "verify-bls",
+	Description: "given a block, verifies the bls signature of the messages in the block",
+	Usage:       "<blockCid>",
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() != 1 {
+			return xerrors.Errorf("usage: <blockCid>")
+		}
+
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		bc, err := cid.Decode(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		b, err := api.ChainGetBlock(ctx, bc)
+		if err != nil {
+			return err
+		}
+
+		ms, err := api.ChainGetBlockMessages(ctx, bc)
+		if err != nil {
+			return err
+		}
+
+		var sigCids []cid.Cid // this is what we get for people not wanting the marshalcbor method on the cid type
+		var pubks [][]byte
+
+		for _, m := range ms.BlsMessages {
+			sigCids = append(sigCids, m.Cid())
+
+			if m.From.Protocol() != address.BLS {
+				return xerrors.Errorf("address must be BLS address")
+			}
+
+			pubks = append(pubks, m.From.Payload())
+		}
+
+		msgsS := make([]ffi.Message, len(sigCids))
+		pubksS := make([]ffi.PublicKey, len(sigCids))
+		for i := 0; i < len(sigCids); i++ {
+			msgsS[i] = sigCids[i].Bytes()
+			copy(pubksS[i][:], pubks[i][:ffi.PublicKeyBytes])
+		}
+
+		sigS := new(ffi.Signature)
+		copy(sigS[:], b.BLSAggregate.Data[:ffi.SignatureBytes])
+
+		if len(sigCids) == 0 {
+			return nil
+		}
+
+		valid := ffi.HashVerify(sigS, msgsS, pubksS)
+		if !valid {
+			return xerrors.New("bls aggregate signature failed to verify")
+		}
+
+		fmt.Println("BLS siggys valid!")
+		return nil
+	},
+}
+
+var sigsVerifyVoteCmd = &cli.Command{
+	Name:        "verify-vote",
+	Description: "can be used to verify signed votes being submitted for FILPolls",
+	Usage:       "<FIPnumber> <signingAddress> <signature>",
+	Action: func(cctx *cli.Context) error {
+
+		if cctx.Args().Len() != 3 {
+			return xerrors.Errorf("usage: verify-vote <FIPnumber> <signingAddress> <signature>")
+		}
+
+		fip, err := strconv.ParseInt(cctx.Args().First(), 10, 64)
+		if err != nil {
+			return xerrors.Errorf("couldn't parse FIP number: %w", err)
+		}
+
+		addr, err := address.NewFromString(cctx.Args().Get(1))
+		if err != nil {
+			return xerrors.Errorf("couldn't parse signing address: %w", err)
+		}
+
+		sigBytes, err := hex.DecodeString(cctx.Args().Get(2))
+		if err != nil {
+			return xerrors.Errorf("couldn't parse sig: %w", err)
+		}
+
+		var sig crypto.Signature
+		if err := sig.UnmarshalBinary(sigBytes); err != nil {
+			return xerrors.Errorf("couldn't unmarshal sig: %w", err)
+		}
+
+		switch fip {
+		case 14:
+			approve := []byte("7 - Approve")
+
+			if sigs.Verify(&sig, addr, approve) == nil {
+				fmt.Println("valid vote for approving FIP-0014")
+				return nil
+			}
+
+			reject := []byte("7 - Reject")
+			if sigs.Verify(&sig, addr, reject) == nil {
+				fmt.Println("valid vote for rejecting FIP-0014")
+				return nil
+			}
+
+			return xerrors.Errorf("invalid vote for FIP-0014!")
+		default:
+			return xerrors.Errorf("unrecognized FIP number")
+		}
+	},
+}

cmd/lotus-shed/stateroot-stats.go

@@ -56,13 +56,6 @@ var staterootDiffsCmd = &cli.Command{
 			return err
 		}
 
-		if ts == nil {
-			ts, err = api.ChainHead(ctx)
-			if err != nil {
-				return err
-			}
-		}
-
 		fn := func(ts *types.TipSet) (cid.Cid, []cid.Cid) {
 			blk := ts.Blocks()[0]
 			strt := blk.ParentStateRoot
@@ -134,13 +127,6 @@ var staterootStatCmd = &cli.Command{
 			return err
 		}
 
-		if ts == nil {
-			ts, err = api.ChainHead(ctx)
-			if err != nil {
-				return err
-			}
-		}
-
 		var addrs []address.Address
 
 		for _, inp := range cctx.Args().Slice() {

cmd/lotus-shed/storage-stats.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"encoding/json"
+	corebig "math/big"
+	"os"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	filbig "github.com/filecoin-project/go-state-types/big"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/ipfs/go-cid"
+	"github.com/urfave/cli/v2"
+)
+
+// How many epochs back to look at for dealstats
+var defaultEpochLookback = abi.ChainEpoch(10)
+
+type networkTotalsOutput struct {
+	Epoch    int64         `json:"epoch"`
+	Endpoint string        `json:"endpoint"`
+	Payload  networkTotals `json:"payload"`
+}
+
+type networkTotals struct {
+	QaNetworkPower       filbig.Int `json:"total_qa_power"`
+	RawNetworkPower      filbig.Int `json:"total_raw_capacity"`
+	CapacityCarryingData float64    `json:"capacity_fraction_carrying_data"`
+	UniqueCids           int        `json:"total_unique_cids"`
+	UniqueProviders      int        `json:"total_unique_providers"`
+	UniqueClients        int        `json:"total_unique_clients"`
+	TotalDeals           int        `json:"total_num_deals"`
+	TotalBytes           int64      `json:"total_stored_data_size"`
+	FilplusTotalDeals    int        `json:"filplus_total_num_deals"`
+	FilplusTotalBytes    int64      `json:"filplus_total_stored_data_size"`
+
+	seenClient   map[address.Address]bool
+	seenProvider map[address.Address]bool
+	seenPieceCid map[cid.Cid]bool
+}
+
+var storageStatsCmd = &cli.Command{
+	Name:  "storage-stats",
+	Usage: "Translates current lotus state into a json summary suitable for driving https://storage.filecoin.io/",
+	Flags: []cli.Flag{
+		&cli.Int64Flag{
+			Name: "height",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		api, apiCloser, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer apiCloser()
+
+		head, err := api.ChainHead(ctx)
+		if err != nil {
+			return err
+		}
+
+		requestedHeight := cctx.Int64("height")
+		if requestedHeight > 0 {
+			head, err = api.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(requestedHeight), head.Key())
+		} else {
+			head, err = api.ChainGetTipSetByHeight(ctx, head.Height()-defaultEpochLookback, head.Key())
+		}
+		if err != nil {
+			return err
+		}
+
+		power, err := api.StateMinerPower(ctx, address.Address{}, head.Key())
+		if err != nil {
+			return err
+		}
+
+		netTotals := networkTotals{
+			QaNetworkPower:  power.TotalPower.QualityAdjPower,
+			RawNetworkPower: power.TotalPower.RawBytePower,
+			seenClient:      make(map[address.Address]bool),
+			seenProvider:    make(map[address.Address]bool),
+			seenPieceCid:    make(map[cid.Cid]bool),
+		}
+
+		deals, err := api.StateMarketDeals(ctx, head.Key())
+		if err != nil {
+			return err
+		}
+
+		for _, dealInfo := range deals {
+
+			// Only count deals that have properly started, not past/future ones
+			// https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85
+			// Bail on 0 as well in case SectorStartEpoch is uninitialized due to some bug
+			if dealInfo.State.SectorStartEpoch <= 0 ||
+				dealInfo.State.SectorStartEpoch > head.Height() {
+				continue
+			}
+
+			netTotals.seenClient[dealInfo.Proposal.Client] = true
+			netTotals.TotalBytes += int64(dealInfo.Proposal.PieceSize)
+			netTotals.seenProvider[dealInfo.Proposal.Provider] = true
+			netTotals.seenPieceCid[dealInfo.Proposal.PieceCID] = true
+			netTotals.TotalDeals++
+
+			if dealInfo.Proposal.VerifiedDeal {
+				netTotals.FilplusTotalDeals++
+				netTotals.FilplusTotalBytes += int64(dealInfo.Proposal.PieceSize)
+			}
+		}
+
+		netTotals.UniqueCids = len(netTotals.seenPieceCid)
+		netTotals.UniqueClients = len(netTotals.seenClient)
+		netTotals.UniqueProviders = len(netTotals.seenProvider)
+
+		netTotals.CapacityCarryingData, _ = new(corebig.Rat).SetFrac(
+			corebig.NewInt(netTotals.TotalBytes),
+			netTotals.RawNetworkPower.Int,
+		).Float64()
+
+		return json.NewEncoder(os.Stdout).Encode(
+			networkTotalsOutput{
+				Epoch:    int64(head.Height()),
+				Endpoint: "NETWORK_WIDE_TOTALS",
+				Payload:  netTotals,
+			},
+		)
+	},
+}

cmd/lotus-shed/sync.go

@@ -172,12 +172,13 @@ var syncScrapePowerCmd = &cli.Command{
 			return err
 		}
 
-		qpercI := types.BigDiv(types.BigMul(totalWonPower.QualityAdjPower, types.NewInt(1000000)), totalPower.TotalPower.QualityAdjPower)
-
 		fmt.Println("Number of winning miners: ", len(miners))
 		fmt.Println("QAdjPower of winning miners: ", totalWonPower.QualityAdjPower)
 		fmt.Println("QAdjPower of all miners: ", totalPower.TotalPower.QualityAdjPower)
-		fmt.Println("Percentage of winning QAdjPower: ", float64(qpercI.Int64())/10000)
+		fmt.Println("Percentage of winning QAdjPower: ", types.BigDivFloat(
+			types.BigMul(totalWonPower.QualityAdjPower, big.NewInt(100)),
+			totalPower.TotalPower.QualityAdjPower,
+		))
 
 		return nil
 	},

cmd/lotus-shed/verifreg.go

@@ -13,7 +13,7 @@ import (
 
 	verifreg2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg"
 
-	"github.com/filecoin-project/lotus/api/apibstore"
+	"github.com/filecoin-project/lotus/blockstore"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/adt"
@@ -67,11 +67,13 @@ var verifRegAddVerifierCmd = &cli.Command{
 			return err
 		}
 
-		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		srv, err := lcli.GetFullNodeServices(cctx)
 		if err != nil {
 			return err
 		}
-		defer closer()
+		defer srv.Close() //nolint:errcheck
+
+		api := srv.FullNodeAPI()
 		ctx := lcli.ReqContext(cctx)
 
 		vrk, err := api.StateVerifiedRegistryRootKey(ctx, types.EmptyTSK)
@@ -79,14 +81,21 @@ var verifRegAddVerifierCmd = &cli.Command{
 			return err
 		}
 
-		smsg, err := api.MsigPropose(ctx, vrk, verifreg.Address, big.Zero(), sender, uint64(verifreg.Methods.AddVerifier), params)
+		proto, err := api.MsigPropose(ctx, vrk, verifreg.Address, big.Zero(), sender, uint64(verifreg.Methods.AddVerifier), params)
 		if err != nil {
 			return err
 		}
 
-		fmt.Printf("message sent, now waiting on cid: %s\n", smsg)
+		sm, _, err := srv.PublishMessage(ctx, proto, false)
+		if err != nil {
+			return err
+		}
 
-		mwait, err := api.StateWaitMsg(ctx, smsg, build.MessageConfidence)
+		msgCid := sm.Cid()
+
+		fmt.Printf("message sent, now waiting on cid: %s\n", msgCid)
+
+		mwait, err := api.StateWaitMsg(ctx, msgCid, uint64(cctx.Int("confidence")), build.Finality, true)
 		if err != nil {
 			return err
 		}
@@ -102,8 +111,9 @@ var verifRegAddVerifierCmd = &cli.Command{
 }
 
 var verifRegVerifyClientCmd = &cli.Command{
-	Name:  "verify-client",
-	Usage: "make a given account a verified client",
+	Name:   "verify-client",
+	Usage:  "make a given account a verified client",
+	Hidden: true,
 	Flags: []cli.Flag{
 		&cli.StringFlag{
 			Name:  "from",
@@ -111,6 +121,7 @@ var verifRegVerifyClientCmd = &cli.Command{
 		},
 	},
 	Action: func(cctx *cli.Context) error {
+		fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
 		froms := cctx.String("from")
 		if froms == "" {
 			return fmt.Errorf("must specify from address with --from")
@@ -175,9 +186,11 @@ var verifRegVerifyClientCmd = &cli.Command{
 }
 
 var verifRegListVerifiersCmd = &cli.Command{
-	Name:  "list-verifiers",
-	Usage: "list all verifiers",
+	Name:   "list-verifiers",
+	Usage:  "list all verifiers",
+	Hidden: true,
 	Action: func(cctx *cli.Context) error {
+		fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
 		api, closer, err := lcli.GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -190,7 +203,7 @@ var verifRegListVerifiersCmd = &cli.Command{
 			return err
 		}
 
-		apibs := apibstore.NewAPIBlockstore(api)
+		apibs := blockstore.NewAPIBlockstore(api)
 		store := adt.WrapStore(ctx, cbor.NewCborStore(apibs))
 
 		st, err := verifreg.Load(store, act)
@@ -205,9 +218,11 @@ var verifRegListVerifiersCmd = &cli.Command{
 }
 
 var verifRegListClientsCmd = &cli.Command{
-	Name:  "list-clients",
-	Usage: "list all verified clients",
+	Name:   "list-clients",
+	Usage:  "list all verified clients",
+	Hidden: true,
 	Action: func(cctx *cli.Context) error {
+		fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
 		api, closer, err := lcli.GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -220,7 +235,7 @@ var verifRegListClientsCmd = &cli.Command{
 			return err
 		}
 
-		apibs := apibstore.NewAPIBlockstore(api)
+		apibs := blockstore.NewAPIBlockstore(api)
 		store := adt.WrapStore(ctx, cbor.NewCborStore(apibs))
 
 		st, err := verifreg.Load(store, act)
@@ -235,9 +250,11 @@ var verifRegListClientsCmd = &cli.Command{
 }
 
 var verifRegCheckClientCmd = &cli.Command{
-	Name:  "check-client",
-	Usage: "check verified client remaining bytes",
+	Name:   "check-client",
+	Usage:  "check verified client remaining bytes",
+	Hidden: true,
 	Action: func(cctx *cli.Context) error {
+		fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
 		if !cctx.Args().Present() {
 			return fmt.Errorf("must specify client address to check")
 		}
@@ -269,9 +286,11 @@ var verifRegCheckClientCmd = &cli.Command{
 }
 
 var verifRegCheckVerifierCmd = &cli.Command{
-	Name:  "check-verifier",
-	Usage: "check verifiers remaining bytes",
+	Name:   "check-verifier",
+	Usage:  "check verifiers remaining bytes",
+	Hidden: true,
 	Action: func(cctx *cli.Context) error {
+		fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
 		if !cctx.Args().Present() {
 			return fmt.Errorf("must specify verifier address to check")
 		}
@@ -303,7 +322,7 @@ var verifRegCheckVerifierCmd = &cli.Command{
 			return err
 		}
 
-		apibs := apibstore.NewAPIBlockstore(api)
+		apibs := blockstore.NewAPIBlockstore(api)
 		store := adt.WrapStore(ctx, cbor.NewCborStore(apibs))
 
 		st, err := verifreg.Load(store, act)

cmd/lotus-sim/copy.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+)
+
+var copySimCommand = &cli.Command{
+	Name:      "copy",
+	ArgsUsage: "<new-name>",
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+		if cctx.NArg() != 1 {
+			return fmt.Errorf("expected 1 argument")
+		}
+		name := cctx.Args().First()
+		return node.CopySim(cctx.Context, cctx.String("simulation"), name)
+	},
+}

cmd/lotus-sim/create.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var createSimCommand = &cli.Command{
+	Name:      "create",
+	ArgsUsage: "[tipset]",
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		var ts *types.TipSet
+		switch cctx.NArg() {
+		case 0:
+			if err := node.Chainstore.Load(); err != nil {
+				return err
+			}
+			ts = node.Chainstore.GetHeaviestTipSet()
+		case 1:
+			cids, err := lcli.ParseTipSetString(cctx.Args().Get(1))
+			if err != nil {
+				return err
+			}
+			tsk := types.NewTipSetKey(cids...)
+			ts, err = node.Chainstore.LoadTipSet(tsk)
+			if err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("expected 0 or 1 arguments")
+		}
+		_, err = node.CreateSim(cctx.Context, cctx.String("simulation"), ts)
+		return err
+	},
+}

cmd/lotus-sim/delete.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+var deleteSimCommand = &cli.Command{
+	Name: "delete",
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		return node.DeleteSim(cctx.Context, cctx.String("simulation"))
+	},
+}

cmd/lotus-sim/info.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"text/tabwriter"
+	"time"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/go-state-types/big"
+
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation"
+)
+
+func getTotalPower(ctx context.Context, sm *stmgr.StateManager, ts *types.TipSet) (power.Claim, error) {
+	actor, err := sm.LoadActor(ctx, power.Address, ts)
+	if err != nil {
+		return power.Claim{}, err
+	}
+	state, err := power.Load(sm.ChainStore().ActorStore(ctx), actor)
+	if err != nil {
+		return power.Claim{}, err
+	}
+	return state.TotalPower()
+}
+
+func printInfo(ctx context.Context, sim *simulation.Simulation, out io.Writer) error {
+	head := sim.GetHead()
+	start := sim.GetStart()
+
+	powerNow, err := getTotalPower(ctx, sim.StateManager, head)
+	if err != nil {
+		return err
+	}
+	powerLookbackEpoch := head.Height() - builtin.EpochsInDay*2
+	if powerLookbackEpoch < start.Height() {
+		powerLookbackEpoch = start.Height()
+	}
+	lookbackTs, err := sim.Node.Chainstore.GetTipsetByHeight(ctx, powerLookbackEpoch, head, false)
+	if err != nil {
+		return err
+	}
+	powerLookback, err := getTotalPower(ctx, sim.StateManager, lookbackTs)
+	if err != nil {
+		return err
+	}
+	// growth rate in size/day
+	growthRate := big.Div(
+		big.Mul(big.Sub(powerNow.RawBytePower, powerLookback.RawBytePower),
+			big.NewInt(builtin.EpochsInDay)),
+		big.NewInt(int64(head.Height()-lookbackTs.Height())),
+	)
+
+	tw := tabwriter.NewWriter(out, 8, 8, 1, ' ', 0)
+
+	headEpoch := head.Height()
+	firstEpoch := start.Height() + 1
+
+	headTime := time.Unix(int64(head.MinTimestamp()), 0)
+	startTime := time.Unix(int64(start.MinTimestamp()), 0)
+	duration := headTime.Sub(startTime)
+
+	fmt.Fprintf(tw, "Name:\t%s\n", sim.Name())
+	fmt.Fprintf(tw, "Head:\t%s\n", head)
+	fmt.Fprintf(tw, "Start Epoch:\t%d\n", firstEpoch)
+	fmt.Fprintf(tw, "End Epoch:\t%d\n", headEpoch)
+	fmt.Fprintf(tw, "Length:\t%d\n", headEpoch-firstEpoch)
+	fmt.Fprintf(tw, "Start Date:\t%s\n", startTime)
+	fmt.Fprintf(tw, "End Date:\t%s\n", headTime)
+	fmt.Fprintf(tw, "Duration:\t%.2f day(s)\n", duration.Hours()/24)
+	fmt.Fprintf(tw, "Capacity:\t%s\n", types.SizeStr(powerNow.RawBytePower))
+	fmt.Fprintf(tw, "Daily Capacity Growth:\t%s/day\n", types.SizeStr(growthRate))
+	fmt.Fprintf(tw, "Network Version:\t%d\n", sim.GetNetworkVersion())
+	return tw.Flush()
+}
+
+var infoSimCommand = &cli.Command{
+	Name:        "info",
+	Description: "Output information about the simulation.",
+	Subcommands: []*cli.Command{
+		infoCommitGasSimCommand,
+		infoMessageSizeSimCommand,
+		infoWindowPostBandwidthSimCommand,
+		infoCapacityGrowthSimCommand,
+		infoStateGrowthSimCommand,
+	},
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
+		if err != nil {
+			return err
+		}
+		return printInfo(cctx.Context, sim, cctx.App.Writer)
+	},
+}

cmd/lotus-sim/info_capacity.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/go-state-types/big"
+
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+var infoCapacityGrowthSimCommand = &cli.Command{
+	Name:        "capacity-growth",
+	Description: "List daily capacity growth over the course of the simulation starting at the end.",
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
+		if err != nil {
+			return err
+		}
+
+		firstEpoch := sim.GetStart().Height()
+		ts := sim.GetHead()
+		lastPower, err := getTotalPower(cctx.Context, sim.StateManager, ts)
+		if err != nil {
+			return err
+		}
+		lastHeight := ts.Height()
+
+		for ts.Height() > firstEpoch && cctx.Err() == nil {
+			ts, err = sim.Node.Chainstore.LoadTipSet(ts.Parents())
+			if err != nil {
+				return err
+			}
+			newEpoch := ts.Height()
+			if newEpoch != firstEpoch && newEpoch+builtin.EpochsInDay > lastHeight {
+				continue
+			}
+
+			newPower, err := getTotalPower(cctx.Context, sim.StateManager, ts)
+			if err != nil {
+				return err
+			}
+
+			growthRate := big.Div(
+				big.Mul(big.Sub(lastPower.RawBytePower, newPower.RawBytePower),
+					big.NewInt(builtin.EpochsInDay)),
+				big.NewInt(int64(lastHeight-newEpoch)),
+			)
+			lastPower = newPower
+			lastHeight = newEpoch
+			fmt.Fprintf(cctx.App.Writer, "%s/day\n", types.SizeStr(growthRate))
+		}
+		return cctx.Err()
+	},
+}

cmd/lotus-sim/info_commit.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"syscall"
+
+	"github.com/streadway/quantile"
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/go-state-types/exitcode"
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation"
+	"github.com/filecoin-project/lotus/lib/stati"
+)
+
+var infoCommitGasSimCommand = &cli.Command{
+	Name:        "commit-gas",
+	Description: "Output information about the gas for commits",
+	Flags: []cli.Flag{
+		&cli.Int64Flag{
+			Name:  "lookback",
+			Value: 0,
+		},
+	},
+	Action: func(cctx *cli.Context) (err error) {
+		log := func(f string, i ...interface{}) {
+			fmt.Fprintf(os.Stderr, f, i...)
+		}
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		go profileOnSignal(cctx, syscall.SIGUSR2)
+
+		sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
+		if err != nil {
+			return err
+		}
+
+		var gasAgg, proofsAgg uint64
+		var gasAggMax, proofsAggMax uint64
+		var gasSingle, proofsSingle uint64
+
+		qpoints := []struct{ q, tol float64 }{
+			{0.01, 0.0005},
+			{0.05, 0.001},
+			{0.20, 0.01},
+			{0.25, 0.01},
+			{0.30, 0.01},
+			{0.40, 0.01},
+			{0.45, 0.01},
+			{0.50, 0.01},
+			{0.60, 0.01},
+			{0.80, 0.01},
+			{0.95, 0.001},
+			{0.99, 0.0005},
+		}
+		estims := make([]quantile.Estimate, len(qpoints))
+		for i, p := range qpoints {
+			estims[i] = quantile.Known(p.q, p.tol)
+		}
+		qua := quantile.New(estims...)
+		hist, err := stati.NewHistogram([]float64{
+			1, 3, 5, 7, 15, 30, 50, 100, 200, 400, 600, 700, 819})
+		if err != nil {
+			return err
+		}
+
+		err = sim.Walk(cctx.Context, cctx.Int64("lookback"), func(
+			sm *stmgr.StateManager, ts *types.TipSet, stCid cid.Cid,
+			messages []*simulation.AppliedMessage,
+		) error {
+			for _, m := range messages {
+				if m.ExitCode != exitcode.Ok {
+					continue
+				}
+				if m.Method == miner.Methods.ProveCommitAggregate {
+					param := miner.ProveCommitAggregateParams{}
+					err := param.UnmarshalCBOR(bytes.NewReader(m.Params))
+					if err != nil {
+						log("failed to decode params: %+v", err)
+						return nil
+					}
+					c, err := param.SectorNumbers.Count()
+					if err != nil {
+						log("failed to count sectors")
+						return nil
+					}
+					gasAgg += uint64(m.GasUsed)
+					proofsAgg += c
+					if c == 819 {
+						gasAggMax += uint64(m.GasUsed)
+						proofsAggMax += c
+					}
+					for i := uint64(0); i < c; i++ {
+						qua.Add(float64(c))
+					}
+					hist.Observe(float64(c))
+				}
+
+				if m.Method == miner.Methods.ProveCommitSector {
+					gasSingle += uint64(m.GasUsed)
+					proofsSingle++
+					qua.Add(1)
+					hist.Observe(1)
+				}
+			}
+
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+		idealGassUsed := float64(gasAggMax) / float64(proofsAggMax) * float64(proofsAgg+proofsSingle)
+
+		fmt.Printf("Gas usage efficiency in comparison to all 819: %f%%\n", 100*idealGassUsed/float64(gasAgg+gasSingle))
+
+		fmt.Printf("Proofs in singles: %d\n", proofsSingle)
+		fmt.Printf("Proofs in Aggs: %d\n", proofsAgg)
+		fmt.Printf("Proofs in Aggs(819): %d\n", proofsAggMax)
+
+		fmt.Println()
+		fmt.Println("Quantiles of proofs in given aggregate size:")
+		for _, p := range qpoints {
+			fmt.Printf("%.0f%%\t%.0f\n", p.q*100, qua.Get(p.q))
+		}
+		fmt.Println()
+		fmt.Println("Histogram of messages:")
+		fmt.Printf("Total\t%d\n", hist.Total())
+		for i, b := range hist.Buckets[1:] {
+			fmt.Printf("%.0f\t%d\n", b, hist.Get(i))
+		}
+
+		return nil
+	},
+}

cmd/lotus-sim/info_message.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation"
+	"github.com/filecoin-project/lotus/lib/stati"
+	"github.com/ipfs/go-cid"
+	"github.com/streadway/quantile"
+	"github.com/urfave/cli/v2"
+)
+
+var infoMessageSizeSimCommand = &cli.Command{
+	Name:        "message-size",
+	Description: "Output information about message size distribution",
+	Flags: []cli.Flag{
+		&cli.Int64Flag{
+			Name:  "lookback",
+			Value: 0,
+		},
+	},
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		go profileOnSignal(cctx, syscall.SIGUSR2)
+
+		sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
+		if err != nil {
+			return err
+		}
+
+		qpoints := []struct{ q, tol float64 }{
+			{0.30, 0.01},
+			{0.40, 0.01},
+			{0.60, 0.01},
+			{0.70, 0.01},
+			{0.80, 0.01},
+			{0.85, 0.01},
+			{0.90, 0.01},
+			{0.95, 0.001},
+			{0.99, 0.0005},
+			{0.999, 0.0001},
+		}
+		estims := make([]quantile.Estimate, len(qpoints))
+		for i, p := range qpoints {
+			estims[i] = quantile.Known(p.q, p.tol)
+		}
+		qua := quantile.New(estims...)
+		hist, err := stati.NewHistogram([]float64{
+			1 << 8, 1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15, 1 << 16,
+		})
+		if err != nil {
+			return err
+		}
+
+		err = sim.Walk(cctx.Context, cctx.Int64("lookback"), func(
+			sm *stmgr.StateManager, ts *types.TipSet, stCid cid.Cid,
+			messages []*simulation.AppliedMessage,
+		) error {
+			for _, m := range messages {
+				msgSize := float64(m.ChainLength())
+				qua.Add(msgSize)
+				hist.Observe(msgSize)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+		fmt.Println("Quantiles of message sizes:")
+		for _, p := range qpoints {
+			fmt.Printf("%.1f%%\t%.0f\n", p.q*100, qua.Get(p.q))
+		}
+		fmt.Println()
+		fmt.Println("Histogram of message sizes:")
+		fmt.Printf("Total\t%d\n", hist.Total())
+		for i, b := range hist.Buckets[1:] {
+			fmt.Printf("%.0f\t%d\t%.1f%%\n", b, hist.Get(i), 100*hist.GetRatio(i))
+		}
+
+		return nil
+	},
+}

cmd/lotus-sim/info_state.go

@@ -0,0 +1,141 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math"
+	"sync"
+	"sync/atomic"
+
+	"github.com/ipfs/go-cid"
+	"github.com/urfave/cli/v2"
+	cbg "github.com/whyrusleeping/cbor-gen"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+var infoStateGrowthSimCommand = &cli.Command{
+	Name:        "state-size",
+	Description: "List daily state size over the course of the simulation starting at the end.",
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
+		if err != nil {
+			return err
+		}
+
+		// NOTE: This code is entirely read-bound.
+		store := node.Chainstore.StateBlockstore()
+		stateSize := func(ctx context.Context, c cid.Cid) (uint64, error) {
+			seen := cid.NewSet()
+			sema := make(chan struct{}, 40)
+			var lock sync.Mutex
+			var recSize func(cid.Cid) (uint64, error)
+			recSize = func(c cid.Cid) (uint64, error) {
+				// Not a part of the chain state.
+				if err := ctx.Err(); err != nil {
+					return 0, err
+				}
+
+				lock.Lock()
+				visit := seen.Visit(c)
+				lock.Unlock()
+				// Already seen?
+				if !visit {
+					return 0, nil
+				}
+
+				var links []cid.Cid
+				var totalSize uint64
+				if err := store.View(c, func(data []byte) error {
+					totalSize += uint64(len(data))
+					return cbg.ScanForLinks(bytes.NewReader(data), func(c cid.Cid) {
+						if c.Prefix().Codec != cid.DagCBOR {
+							return
+						}
+
+						links = append(links, c)
+					})
+				}); err != nil {
+					return 0, err
+				}
+
+				var wg sync.WaitGroup
+				errCh := make(chan error, 1)
+				cb := func(c cid.Cid) {
+					size, err := recSize(c)
+					if err != nil {
+						select {
+						case errCh <- err:
+						default:
+						}
+						return
+					}
+					atomic.AddUint64(&totalSize, size)
+				}
+				asyncCb := func(c cid.Cid) {
+					wg.Add(1)
+					go func() {
+						defer wg.Done()
+						defer func() { <-sema }()
+						cb(c)
+					}()
+				}
+				for _, link := range links {
+					select {
+					case sema <- struct{}{}:
+						asyncCb(link)
+					default:
+						cb(link)
+					}
+
+				}
+				wg.Wait()
+
+				select {
+				case err := <-errCh:
+					return 0, err
+				default:
+				}
+
+				return totalSize, nil
+			}
+			return recSize(c)
+		}
+
+		firstEpoch := sim.GetStart().Height()
+		ts := sim.GetHead()
+		lastHeight := abi.ChainEpoch(math.MaxInt64)
+		for ts.Height() > firstEpoch && cctx.Err() == nil {
+			if ts.Height()+builtin.EpochsInDay <= lastHeight {
+				lastHeight = ts.Height()
+
+				parentStateSize, err := stateSize(cctx.Context, ts.ParentState())
+				if err != nil {
+					return err
+				}
+
+				fmt.Fprintf(cctx.App.Writer, "%d: %s\n", ts.Height(), types.SizeStr(types.NewInt(parentStateSize)))
+			}
+
+			ts, err = sim.Node.Chainstore.LoadTipSet(ts.Parents())
+			if err != nil {
+				return err
+			}
+		}
+		return cctx.Err()
+	},
+}

cmd/lotus-sim/info_wdpost.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-cid"
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/go-state-types/exitcode"
+
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation"
+)
+
+var infoWindowPostBandwidthSimCommand = &cli.Command{
+	Name:        "post-bandwidth",
+	Description: "List average chain bandwidth used by window posts for each day of the simulation.",
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
+		if err != nil {
+			return err
+		}
+
+		var postGas, totalGas int64
+		printStats := func() {
+			fmt.Fprintf(cctx.App.Writer, "%.4f%%\n", float64(100*postGas)/float64(totalGas))
+		}
+		idx := 0
+		err = sim.Walk(cctx.Context, 0, func(
+			sm *stmgr.StateManager, ts *types.TipSet, stCid cid.Cid,
+			messages []*simulation.AppliedMessage,
+		) error {
+			for _, m := range messages {
+				totalGas += m.GasUsed
+				if m.ExitCode != exitcode.Ok {
+					continue
+				}
+				if m.Method == miner.Methods.SubmitWindowedPoSt {
+					postGas += m.GasUsed
+				}
+			}
+			idx++
+			idx %= builtin.EpochsInDay
+			if idx == 0 {
+				printStats()
+				postGas = 0
+				totalGas = 0
+			}
+			return nil
+		})
+		if idx > 0 {
+			printStats()
+		}
+		return err
+	},
+}

cmd/lotus-sim/list.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"fmt"
+	"text/tabwriter"
+
+	"github.com/urfave/cli/v2"
+)
+
+var listSimCommand = &cli.Command{
+	Name: "list",
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		list, err := node.ListSims(cctx.Context)
+		if err != nil {
+			return err
+		}
+		tw := tabwriter.NewWriter(cctx.App.Writer, 8, 8, 0, ' ', 0)
+		for _, name := range list {
+			sim, err := node.LoadSim(cctx.Context, name)
+			if err != nil {
+				return err
+			}
+			head := sim.GetHead()
+			fmt.Fprintf(tw, "%s\t%s\t%s\n", name, head.Height(), head.Key())
+		}
+		return tw.Flush()
+	},
+}

cmd/lotus-sim/main.go

@@ -0,0 +1,63 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/urfave/cli/v2"
+
+	logging "github.com/ipfs/go-log/v2"
+)
+
+var root []*cli.Command = []*cli.Command{
+	createSimCommand,
+	deleteSimCommand,
+	copySimCommand,
+	renameSimCommand,
+	listSimCommand,
+
+	runSimCommand,
+	infoSimCommand,
+	upgradeCommand,
+}
+
+func main() {
+	if _, set := os.LookupEnv("GOLOG_LOG_LEVEL"); !set {
+		_ = logging.SetLogLevel("simulation", "DEBUG")
+		_ = logging.SetLogLevel("simulation-mock", "DEBUG")
+	}
+	app := &cli.App{
+		Name:      "lotus-sim",
+		Usage:     "A tool to simulate a network.",
+		Commands:  root,
+		Writer:    os.Stdout,
+		ErrWriter: os.Stderr,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "repo",
+				EnvVars: []string{"LOTUS_PATH"},
+				Hidden:  true,
+				Value:   "~/.lotus",
+			},
+			&cli.StringFlag{
+				Name:    "simulation",
+				Aliases: []string{"sim"},
+				EnvVars: []string{"LOTUS_SIMULATION"},
+				Value:   "default",
+			},
+		},
+	}
+
+	ctx, cancel := signal.NotifyContext(context.Background(),
+		syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP)
+	defer cancel()
+
+	if err := app.RunContext(ctx, os.Args); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %s\n", err)
+		os.Exit(1)
+		return
+	}
+}

cmd/lotus-sim/profile.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime/pprof"
+	"time"
+
+	"github.com/urfave/cli/v2"
+)
+
+func takeProfiles(ctx context.Context) (fname string, _err error) {
+	dir, err := os.MkdirTemp(".", ".profiles-temp*")
+	if err != nil {
+		return "", err
+	}
+
+	if err := writeProfiles(ctx, dir); err != nil {
+		_ = os.RemoveAll(dir)
+		return "", err
+	}
+
+	fname = fmt.Sprintf("pprof-simulation-%s", time.Now().Format(time.RFC3339))
+	if err := os.Rename(dir, fname); err != nil {
+		_ = os.RemoveAll(dir)
+		return "", err
+	}
+	return fname, nil
+}
+
+func writeProfiles(ctx context.Context, dir string) error {
+	for _, profile := range pprof.Profiles() {
+		file, err := os.Create(filepath.Join(dir, profile.Name()+".pprof.gz"))
+		if err != nil {
+			return err
+		}
+		if err := profile.WriteTo(file, 0); err != nil {
+			_ = file.Close()
+			return err
+		}
+		if err := file.Close(); err != nil {
+			return err
+		}
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+	}
+
+	file, err := os.Create(filepath.Join(dir, "cpu.pprof.gz"))
+	if err != nil {
+		return err
+	}
+
+	if err := pprof.StartCPUProfile(file); err != nil {
+		_ = file.Close()
+		return err
+	}
+	select {
+	case <-time.After(30 * time.Second):
+	case <-ctx.Done():
+	}
+	pprof.StopCPUProfile()
+	err = file.Close()
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	return err
+}
+
+func profileOnSignal(cctx *cli.Context, signals ...os.Signal) {
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, signals...)
+	defer signal.Stop(ch)
+
+	for {
+		select {
+		case <-ch:
+			fname, err := takeProfiles(cctx.Context)
+			switch err {
+			case context.Canceled:
+				return
+			case nil:
+				fmt.Fprintf(cctx.App.ErrWriter, "Wrote profile to %q\n", fname)
+			default:
+				fmt.Fprintf(cctx.App.ErrWriter, "ERROR: failed to write profile: %s\n", err)
+			}
+		case <-cctx.Done():
+			return
+		}
+	}
+}

cmd/lotus-sim/rename.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+)
+
+var renameSimCommand = &cli.Command{
+	Name:      "rename",
+	ArgsUsage: "<new-name>",
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		if cctx.NArg() != 1 {
+			return fmt.Errorf("expected 1 argument")
+		}
+		name := cctx.Args().First()
+		return node.RenameSim(cctx.Context, cctx.String("simulation"), name)
+	},
+}

cmd/lotus-sim/run.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/urfave/cli/v2"
+)
+
+var runSimCommand = &cli.Command{
+	Name: "run",
+	Description: `Run the simulation.
+
+Signals:
+- SIGUSR1: Print information about the current simulation (equivalent to 'lotus-sim info').
+- SIGUSR2: Write pprof profiles to ./pprof-simulation-$DATE/`,
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Name:  "epochs",
+			Usage: "Advance the given number of epochs then stop.",
+		},
+	},
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		go profileOnSignal(cctx, syscall.SIGUSR2)
+
+		sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
+		if err != nil {
+			return err
+		}
+		targetEpochs := cctx.Int("epochs")
+
+		ch := make(chan os.Signal, 1)
+		signal.Notify(ch, syscall.SIGUSR1)
+		defer signal.Stop(ch)
+
+		for i := 0; targetEpochs == 0 || i < targetEpochs; i++ {
+			ts, err := sim.Step(cctx.Context)
+			if err != nil {
+				return err
+			}
+
+			fmt.Fprintf(cctx.App.Writer, "advanced to %d %s\n", ts.Height(), ts.Key())
+
+			// Print
+			select {
+			case <-ch:
+				fmt.Fprintln(cctx.App.Writer, "---------------------")
+				if err := printInfo(cctx.Context, sim, cctx.App.Writer); err != nil {
+					fmt.Fprintf(cctx.App.ErrWriter, "ERROR: failed to print info: %s\n", err)
+				}
+				fmt.Fprintln(cctx.App.Writer, "---------------------")
+			case <-cctx.Context.Done():
+				return cctx.Err()
+			default:
+			}
+		}
+		fmt.Fprintln(cctx.App.Writer, "simulation done")
+		return err
+	},
+}

cmd/lotus-sim/simulation/block.go

@@ -0,0 +1,93 @@
+package simulation
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"time"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+const beaconPrefix = "mockbeacon:"
+
+// nextBeaconEntries returns a fake beacon entries for the next block.
+func (sim *Simulation) nextBeaconEntries() []types.BeaconEntry {
+	parentBeacons := sim.head.Blocks()[0].BeaconEntries
+	lastBeacon := parentBeacons[len(parentBeacons)-1]
+	beaconRound := lastBeacon.Round + 1
+
+	buf := make([]byte, len(beaconPrefix)+8)
+	copy(buf, beaconPrefix)
+	binary.BigEndian.PutUint64(buf[len(beaconPrefix):], beaconRound)
+	beaconRand := sha256.Sum256(buf)
+	return []types.BeaconEntry{{
+		Round: beaconRound,
+		Data:  beaconRand[:],
+	}}
+}
+
+// nextTicket returns a fake ticket for the next block.
+func (sim *Simulation) nextTicket() *types.Ticket {
+	newProof := sha256.Sum256(sim.head.MinTicket().VRFProof)
+	return &types.Ticket{
+		VRFProof: newProof[:],
+	}
+}
+
+// makeTipSet generates and executes the next tipset from the given messages. This method:
+//
+// 1. Stores the given messages in the Chainstore.
+// 2. Creates and persists a single block mined by the same miner as the parent.
+// 3. Creates a tipset from this block and executes it.
+// 4. Returns the resulting tipset.
+//
+// This method does _not_ mutate local state (although it does add blocks to the datastore).
+func (sim *Simulation) makeTipSet(ctx context.Context, messages []*types.Message) (*types.TipSet, error) {
+	parentTs := sim.head
+	parentState, parentRec, err := sim.StateManager.TipSetState(ctx, parentTs)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to compute parent tipset: %w", err)
+	}
+	msgsCid, err := sim.storeMessages(ctx, messages)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to store block messages: %w", err)
+	}
+
+	uts := parentTs.MinTimestamp() + build.BlockDelaySecs
+
+	blks := []*types.BlockHeader{{
+		Miner:                 parentTs.MinTicketBlock().Miner, // keep reusing the same miner.
+		Ticket:                sim.nextTicket(),
+		BeaconEntries:         sim.nextBeaconEntries(),
+		Parents:               parentTs.Cids(),
+		Height:                parentTs.Height() + 1,
+		ParentStateRoot:       parentState,
+		ParentMessageReceipts: parentRec,
+		Messages:              msgsCid,
+		ParentBaseFee:         abi.NewTokenAmount(0),
+		Timestamp:             uts,
+		ElectionProof:         &types.ElectionProof{WinCount: 1},
+	}}
+	err = sim.Node.Chainstore.PersistBlockHeaders(blks...)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to persist block headers: %w", err)
+	}
+	newTipSet, err := types.NewTipSet(blks)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to create new tipset: %w", err)
+	}
+	now := time.Now()
+	_, _, err = sim.StateManager.TipSetState(ctx, newTipSet)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to compute new tipset: %w", err)
+	}
+	duration := time.Since(now)
+	log.Infow("computed tipset", "duration", duration, "height", newTipSet.Height())
+
+	return newTipSet, nil
+}

cmd/lotus-sim/simulation/blockbuilder/blockbuilder.go

@@ -0,0 +1,280 @@
+package blockbuilder
+
+import (
+	"context"
+	"math"
+
+	"go.uber.org/zap"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/network"
+
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/account"
+	"github.com/filecoin-project/lotus/chain/state"
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+)
+
+const (
+	// 0.25 is the default, but the number below is from the network.
+	gasOverestimation = 1.0 / 0.808
+	// The number of expected blocks in a tipset. We use this to determine how much gas a tipset
+	// has.
+	// 5 per tipset, but we effectively get 4 blocks worth of messages.
+	expectedBlocks = 4
+	// TODO: This will produce invalid blocks but it will accurately model the amount of gas
+	// we're willing to use per-tipset.
+	// A more correct approach would be to produce 5 blocks. We can do that later.
+	targetGas = build.BlockGasTarget * expectedBlocks
+)
+
+type BlockBuilder struct {
+	ctx    context.Context
+	logger *zap.SugaredLogger
+
+	parentTs *types.TipSet
+	parentSt *state.StateTree
+	vm       *vm.VM
+	sm       *stmgr.StateManager
+
+	gasTotal int64
+	messages []*types.Message
+}
+
+// NewBlockBuilder constructs a new block builder from the parent state. Use this to pack a block
+// with messages.
+//
+// NOTE: The context applies to the life of the block builder itself (but does not need to be canceled).
+func NewBlockBuilder(ctx context.Context, logger *zap.SugaredLogger, sm *stmgr.StateManager, parentTs *types.TipSet) (*BlockBuilder, error) {
+	parentState, _, err := sm.TipSetState(ctx, parentTs)
+	if err != nil {
+		return nil, err
+	}
+	parentSt, err := sm.StateTree(parentState)
+	if err != nil {
+		return nil, err
+	}
+
+	bb := &BlockBuilder{
+		ctx:      ctx,
+		logger:   logger.With("epoch", parentTs.Height()+1),
+		sm:       sm,
+		parentTs: parentTs,
+		parentSt: parentSt,
+	}
+
+	// Then we construct a VM to execute messages for gas estimation.
+	//
+	// Most parts of this VM are "real" except:
+	// 1. We don't charge a fee.
+	// 2. The runtime has "fake" proof logic.
+	// 3. We don't actually save any of the results.
+	r := store.NewChainRand(sm.ChainStore(), parentTs.Cids())
+	vmopt := &vm.VMOpts{
+		StateBase:      parentState,
+		Epoch:          parentTs.Height() + 1,
+		Rand:           r,
+		Bstore:         sm.ChainStore().StateBlockstore(),
+		Syscalls:       sm.ChainStore().VMSys(),
+		CircSupplyCalc: sm.GetVMCirculatingSupply,
+		NtwkVersion:    sm.GetNtwkVersion,
+		BaseFee:        abi.NewTokenAmount(0),
+		LookbackState:  stmgr.LookbackStateGetterForTipset(sm, parentTs),
+	}
+	bb.vm, err = vm.NewVM(bb.ctx, vmopt)
+	if err != nil {
+		return nil, err
+	}
+	return bb, nil
+}
+
+// PushMessages tries to push the specified message into the block.
+//
+// 1. All messages will be executed in-order.
+// 2. Gas computation & nonce selection will be handled internally.
+// 3. The base-fee is 0 so the sender does not need funds.
+// 4. As usual, the sender must be an account (any account).
+// 5. If the message fails to execute, this method will fail.
+//
+// Returns ErrOutOfGas when out of gas. Check BlockBuilder.GasRemaining and try pushing a cheaper
+// message.
+func (bb *BlockBuilder) PushMessage(msg *types.Message) (*types.MessageReceipt, error) {
+	if bb.gasTotal >= targetGas {
+		return nil, new(ErrOutOfGas)
+	}
+
+	st := bb.StateTree()
+	store := bb.ActorStore()
+
+	// Copy the message before we start mutating it.
+	msgCpy := *msg
+	msg = &msgCpy
+
+	actor, err := st.GetActor(msg.From)
+	if err != nil {
+		return nil, err
+	}
+	if !builtin.IsAccountActor(actor.Code) {
+		return nil, xerrors.Errorf(
+			"messags may only be sent from account actors, got message from %s (%s)",
+			msg.From, builtin.ActorNameByCode(actor.Code),
+		)
+	}
+	msg.Nonce = actor.Nonce
+	if msg.From.Protocol() == address.ID {
+		state, err := account.Load(store, actor)
+		if err != nil {
+			return nil, err
+		}
+		msg.From, err = state.PubkeyAddress()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// TODO: Our gas estimation is broken for payment channels due to horrible hacks in
+	// gasEstimateGasLimit.
+	if msg.Value == types.EmptyInt {
+		msg.Value = abi.NewTokenAmount(0)
+	}
+	msg.GasPremium = abi.NewTokenAmount(0)
+	msg.GasFeeCap = abi.NewTokenAmount(0)
+	msg.GasLimit = build.BlockGasTarget
+
+	// We manually snapshot so we can revert nonce changes, etc. on failure.
+	err = st.Snapshot(bb.ctx)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to take a snapshot while estimating message gas: %w", err)
+	}
+	defer st.ClearSnapshot()
+
+	ret, err := bb.vm.ApplyMessage(bb.ctx, msg)
+	if err != nil {
+		_ = st.Revert()
+		return nil, err
+	}
+	if ret.ActorErr != nil {
+		_ = st.Revert()
+		return nil, ret.ActorErr
+	}
+
+	// Sometimes there are bugs. Let's catch them.
+	if ret.GasUsed == 0 {
+		_ = st.Revert()
+		return nil, xerrors.Errorf("used no gas %v -> %v", msg, ret)
+	}
+
+	// Update the gas limit taking overestimation into account.
+	msg.GasLimit = int64(math.Ceil(float64(ret.GasUsed) * gasOverestimation))
+
+	// Did we go over? Yes, revert.
+	newTotal := bb.gasTotal + msg.GasLimit
+	if newTotal > targetGas {
+		_ = st.Revert()
+		return nil, &ErrOutOfGas{Available: targetGas - bb.gasTotal, Required: msg.GasLimit}
+	}
+	bb.gasTotal = newTotal
+
+	bb.messages = append(bb.messages, msg)
+	return &ret.MessageReceipt, nil
+}
+
+// ActorStore returns the VM's current (pending) blockstore.
+func (bb *BlockBuilder) ActorStore() adt.Store {
+	return bb.vm.ActorStore(bb.ctx)
+}
+
+// StateTree returns the VM's current (pending) state-tree. This includes any changes made by
+// successfully pushed messages.
+//
+// You probably want ParentStateTree
+func (bb *BlockBuilder) StateTree() *state.StateTree {
+	return bb.vm.StateTree().(*state.StateTree)
+}
+
+// ParentStateTree returns the parent state-tree (not the paren't tipset's parent state-tree).
+func (bb *BlockBuilder) ParentStateTree() *state.StateTree {
+	return bb.parentSt
+}
+
+// StateTreeByHeight will return a state-tree up through and including the current in-progress
+// epoch.
+//
+// NOTE: This will return the state after the given epoch, not the parent state for the epoch.
+func (bb *BlockBuilder) StateTreeByHeight(epoch abi.ChainEpoch) (*state.StateTree, error) {
+	now := bb.Height()
+	if epoch > now {
+		return nil, xerrors.Errorf(
+			"cannot load state-tree from future: %d > %d", epoch, bb.Height(),
+		)
+	} else if epoch <= 0 {
+		return nil, xerrors.Errorf(
+			"cannot load state-tree: epoch %d <= 0", epoch,
+		)
+	}
+
+	// Manually handle "now" and "previous".
+	switch epoch {
+	case now:
+		return bb.StateTree(), nil
+	case now - 1:
+		return bb.ParentStateTree(), nil
+	}
+
+	// Get the tipset of the block _after_ the target epoch so we can use its parent state.
+	targetTs, err := bb.sm.ChainStore().GetTipsetByHeight(bb.ctx, epoch+1, bb.parentTs, false)
+	if err != nil {
+		return nil, err
+	}
+
+	return bb.sm.StateTree(targetTs.ParentState())
+}
+
+// Messages returns all messages currently packed into the next block.
+// 1. DO NOT modify the slice, copy it.
+// 2. DO NOT retain the slice, copy it.
+func (bb *BlockBuilder) Messages() []*types.Message {
+	return bb.messages
+}
+
+// GasRemaining returns the amount of remaining gas in the next block.
+func (bb *BlockBuilder) GasRemaining() int64 {
+	return targetGas - bb.gasTotal
+}
+
+// ParentTipSet returns the parent tipset.
+func (bb *BlockBuilder) ParentTipSet() *types.TipSet {
+	return bb.parentTs
+}
+
+// Height returns the epoch for the target block.
+func (bb *BlockBuilder) Height() abi.ChainEpoch {
+	return bb.parentTs.Height() + 1
+}
+
+// NetworkVersion returns the network version for the target block.
+func (bb *BlockBuilder) NetworkVersion() network.Version {
+	return bb.sm.GetNtwkVersion(bb.ctx, bb.Height())
+}
+
+// StateManager returns the stmgr.StateManager.
+func (bb *BlockBuilder) StateManager() *stmgr.StateManager {
+	return bb.sm
+}
+
+// ActorsVersion returns the actors version for the target block.
+func (bb *BlockBuilder) ActorsVersion() actors.Version {
+	return actors.VersionForNetwork(bb.NetworkVersion())
+}
+
+func (bb *BlockBuilder) L() *zap.SugaredLogger {
+	return bb.logger
+}

cmd/lotus-sim/simulation/blockbuilder/errors.go

@@ -0,0 +1,25 @@
+package blockbuilder
+
+import (
+	"errors"
+	"fmt"
+)
+
+// ErrOutOfGas is returned from BlockBuilder.PushMessage when the block does not have enough gas to
+// fit the given message.
+type ErrOutOfGas struct {
+	Available, Required int64
+}
+
+func (e *ErrOutOfGas) Error() string {
+	if e.Available == 0 {
+		return "out of gas: block full"
+	}
+	return fmt.Sprintf("out of gas: %d < %d", e.Required, e.Available)
+}
+
+// IsOutOfGas returns true if the error is an "out of gas" error.
+func IsOutOfGas(err error) bool {
+	var oog *ErrOutOfGas
+	return errors.As(err, &oog)
+}

cmd/lotus-sim/simulation/messages.go

@@ -0,0 +1,58 @@
+package simulation
+
+import (
+	"context"
+
+	"github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
+
+	blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
+
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+// toArray converts the given set of CIDs to an AMT. This is usually used to pack messages into blocks.
+func toArray(store blockadt.Store, cids []cid.Cid) (cid.Cid, error) {
+	arr := blockadt.MakeEmptyArray(store)
+	for i, c := range cids {
+		oc := cbg.CborCid(c)
+		if err := arr.Set(uint64(i), &oc); err != nil {
+			return cid.Undef, err
+		}
+	}
+	return arr.Root()
+}
+
+// storeMessages packs a set of messages into a types.MsgMeta and returns the resulting CID. The
+// resulting CID is valid for the BlocKHeader's Messages field.
+func (sim *Simulation) storeMessages(ctx context.Context, messages []*types.Message) (cid.Cid, error) {
+	// We store all messages as "bls" messages so they're executed in-order. This ensures
+	// accurate gas accounting. It also ensures we don't, e.g., try to fund a miner after we
+	// fail a pre-commit...
+	var msgCids []cid.Cid
+	for _, msg := range messages {
+		c, err := sim.Node.Chainstore.PutMessage(msg)
+		if err != nil {
+			return cid.Undef, err
+		}
+		msgCids = append(msgCids, c)
+	}
+	adtStore := sim.Node.Chainstore.ActorStore(ctx)
+	blsMsgArr, err := toArray(adtStore, msgCids)
+	if err != nil {
+		return cid.Undef, err
+	}
+	sekpMsgArr, err := toArray(adtStore, nil)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	msgsCid, err := adtStore.Put(adtStore.Context(), &types.MsgMeta{
+		BlsMessages:   blsMsgArr,
+		SecpkMessages: sekpMsgArr,
+	})
+	if err != nil {
+		return cid.Undef, err
+	}
+	return msgsCid, nil
+}

cmd/lotus-sim/simulation/mock/mock.go

@@ -0,0 +1,179 @@
+package mock
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"fmt"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/ipfs/go-cid"
+	logging "github.com/ipfs/go-log/v2"
+
+	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
+	proof5 "github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
+	tutils "github.com/filecoin-project/specs-actors/v5/support/testing"
+
+	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+)
+
+// Ideally, we'd use extern/sector-storage/mock. Unfortunately, those mocks are a bit _too_ accurate
+// and would force us to load sector info for window post proofs.
+
+const (
+	mockSealProofPrefix          = "valid seal proof:"
+	mockAggregateSealProofPrefix = "valid aggregate seal proof:"
+	mockPoStProofPrefix          = "valid post proof:"
+)
+
+var log = logging.Logger("simulation-mock")
+
+// mockVerifier is a simple mock for verifying "fake" proofs.
+type mockVerifier struct{}
+
+var Verifier ffiwrapper.Verifier = mockVerifier{}
+
+func (mockVerifier) VerifySeal(proof proof5.SealVerifyInfo) (bool, error) {
+	addr, err := address.NewIDAddress(uint64(proof.Miner))
+	if err != nil {
+		return false, err
+	}
+	mockProof, err := MockSealProof(proof.SealProof, addr)
+	if err != nil {
+		return false, err
+	}
+	if bytes.Equal(proof.Proof, mockProof) {
+		return true, nil
+	}
+	log.Debugw("invalid seal proof", "expected", mockProof, "actual", proof.Proof, "miner", addr)
+	return false, nil
+}
+
+func (mockVerifier) VerifyAggregateSeals(aggregate proof5.AggregateSealVerifyProofAndInfos) (bool, error) {
+	addr, err := address.NewIDAddress(uint64(aggregate.Miner))
+	if err != nil {
+		return false, err
+	}
+	mockProof, err := MockAggregateSealProof(aggregate.SealProof, addr, len(aggregate.Infos))
+	if err != nil {
+		return false, err
+	}
+	if bytes.Equal(aggregate.Proof, mockProof) {
+		return true, nil
+	}
+	log.Debugw("invalid aggregate seal proof",
+		"expected", mockProof,
+		"actual", aggregate.Proof,
+		"count", len(aggregate.Infos),
+		"miner", addr,
+	)
+	return false, nil
+}
+func (mockVerifier) VerifyWinningPoSt(ctx context.Context, info proof5.WinningPoStVerifyInfo) (bool, error) {
+	panic("should not be called")
+}
+func (mockVerifier) VerifyWindowPoSt(ctx context.Context, info proof5.WindowPoStVerifyInfo) (bool, error) {
+	if len(info.Proofs) != 1 {
+		return false, fmt.Errorf("expected exactly one proof")
+	}
+	proof := info.Proofs[0]
+	addr, err := address.NewIDAddress(uint64(info.Prover))
+	if err != nil {
+		return false, err
+	}
+	mockProof, err := MockWindowPoStProof(proof.PoStProof, addr)
+	if err != nil {
+		return false, err
+	}
+	if bytes.Equal(proof.ProofBytes, mockProof) {
+		return true, nil
+	}
+
+	log.Debugw("invalid window post proof",
+		"expected", mockProof,
+		"actual", info.Proofs[0],
+		"miner", addr,
+	)
+	return false, nil
+}
+
+func (mockVerifier) GenerateWinningPoStSectorChallenge(context.Context, abi.RegisteredPoStProof, abi.ActorID, abi.PoStRandomness, uint64) ([]uint64, error) {
+	panic("should not be called")
+}
+
+// MockSealProof generates a mock "seal" proof tied to the specified proof type and the given miner.
+func MockSealProof(proofType abi.RegisteredSealProof, minerAddr address.Address) ([]byte, error) {
+	plen, err := proofType.ProofSize()
+	if err != nil {
+		return nil, err
+	}
+	proof := make([]byte, plen)
+	i := copy(proof, mockSealProofPrefix)
+	binary.BigEndian.PutUint64(proof[i:], uint64(proofType))
+	i += 8
+	i += copy(proof[i:], minerAddr.Bytes())
+	return proof, nil
+}
+
+// MockAggregateSealProof generates a mock "seal" aggregate proof tied to the specified proof type,
+// the given miner, and the number of proven sectors.
+func MockAggregateSealProof(proofType abi.RegisteredSealProof, minerAddr address.Address, count int) ([]byte, error) {
+	proof := make([]byte, aggProofLen(count))
+	i := copy(proof, mockAggregateSealProofPrefix)
+	binary.BigEndian.PutUint64(proof[i:], uint64(proofType))
+	i += 8
+	binary.BigEndian.PutUint64(proof[i:], uint64(count))
+	i += 8
+	i += copy(proof[i:], minerAddr.Bytes())
+
+	return proof, nil
+}
+
+// MockWindowPoStProof generates a mock "window post" proof tied to the specified proof type, and the
+// given miner.
+func MockWindowPoStProof(proofType abi.RegisteredPoStProof, minerAddr address.Address) ([]byte, error) {
+	plen, err := proofType.ProofSize()
+	if err != nil {
+		return nil, err
+	}
+	proof := make([]byte, plen)
+	i := copy(proof, mockPoStProofPrefix)
+	i += copy(proof[i:], minerAddr.Bytes())
+	return proof, nil
+}
+
+// makeCommR generates a "fake" but valid CommR for a sector. It is unique for the given sector/miner.
+func MockCommR(minerAddr address.Address, sno abi.SectorNumber) cid.Cid {
+	return tutils.MakeCID(fmt.Sprintf("%s:%d", minerAddr, sno), &miner5.SealedCIDPrefix)
+}
+
+// TODO: dedup
+func aggProofLen(nproofs int) int {
+	switch {
+	case nproofs <= 8:
+		return 11220
+	case nproofs <= 16:
+		return 14196
+	case nproofs <= 32:
+		return 17172
+	case nproofs <= 64:
+		return 20148
+	case nproofs <= 128:
+		return 23124
+	case nproofs <= 256:
+		return 26100
+	case nproofs <= 512:
+		return 29076
+	case nproofs <= 1024:
+		return 32052
+	case nproofs <= 2048:
+		return 35028
+	case nproofs <= 4096:
+		return 38004
+	case nproofs <= 8192:
+		return 40980
+	default:
+		panic("too many proofs")
+	}
+}

cmd/lotus-sim/simulation/node.go

@@ -0,0 +1,241 @@
+package simulation
+
+import (
+	"context"
+	"strings"
+
+	"go.uber.org/multierr"
+	"golang.org/x/xerrors"
+
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore/query"
+
+	"github.com/filecoin-project/lotus/blockstore"
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/mock"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/stages"
+	"github.com/filecoin-project/lotus/node/repo"
+)
+
+// Node represents the local lotus node, or at least the part of it we care about.
+type Node struct {
+	repo       repo.LockedRepo
+	Blockstore blockstore.Blockstore
+	MetadataDS datastore.Batching
+	Chainstore *store.ChainStore
+}
+
+// OpenNode opens the local lotus node for writing. This will fail if the node is online.
+func OpenNode(ctx context.Context, path string) (*Node, error) {
+	r, err := repo.NewFS(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewNode(ctx, r)
+}
+
+// NewNode constructs a new node from the given repo.
+func NewNode(ctx context.Context, r repo.Repo) (nd *Node, _err error) {
+	lr, err := r.Lock(repo.FullNode)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if _err != nil {
+			_ = lr.Close()
+		}
+	}()
+
+	bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
+	if err != nil {
+		return nil, err
+	}
+
+	ds, err := lr.Datastore(ctx, "/metadata")
+	if err != nil {
+		return nil, err
+	}
+	return &Node{
+		repo:       lr,
+		Chainstore: store.NewChainStore(bs, bs, ds, vm.Syscalls(mock.Verifier), nil),
+		MetadataDS: ds,
+		Blockstore: bs,
+	}, err
+}
+
+// Close cleanly close the repo. Please call this on shutdown to make sure everything is flushed.
+func (nd *Node) Close() error {
+	if nd.repo != nil {
+		return nd.repo.Close()
+	}
+	return nil
+}
+
+// LoadSim loads
+func (nd *Node) LoadSim(ctx context.Context, name string) (*Simulation, error) {
+	stages, err := stages.DefaultPipeline()
+	if err != nil {
+		return nil, err
+	}
+	sim := &Simulation{
+		Node:   nd,
+		name:   name,
+		stages: stages,
+	}
+
+	sim.head, err = sim.loadNamedTipSet("head")
+	if err != nil {
+		return nil, err
+	}
+	sim.start, err = sim.loadNamedTipSet("start")
+	if err != nil {
+		return nil, err
+	}
+
+	err = sim.loadConfig()
+	if err != nil {
+		return nil, xerrors.Errorf("failed to load config for simulation %s: %w", name, err)
+	}
+
+	us, err := sim.config.upgradeSchedule()
+	if err != nil {
+		return nil, xerrors.Errorf("failed to create upgrade schedule for simulation %s: %w", name, err)
+	}
+	sim.StateManager, err = stmgr.NewStateManagerWithUpgradeSchedule(nd.Chainstore, us)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to create state manager for simulation %s: %w", name, err)
+	}
+	return sim, nil
+}
+
+// Create creates a new simulation.
+//
+// - This will fail if a simulation already exists with the given name.
+// - Name must not contain a '/'.
+func (nd *Node) CreateSim(ctx context.Context, name string, head *types.TipSet) (*Simulation, error) {
+	if strings.Contains(name, "/") {
+		return nil, xerrors.Errorf("simulation name %q cannot contain a '/'", name)
+	}
+	stages, err := stages.DefaultPipeline()
+	if err != nil {
+		return nil, err
+	}
+	sim := &Simulation{
+		name:         name,
+		Node:         nd,
+		StateManager: stmgr.NewStateManager(nd.Chainstore),
+		stages:       stages,
+	}
+	if has, err := nd.MetadataDS.Has(sim.key("head")); err != nil {
+		return nil, err
+	} else if has {
+		return nil, xerrors.Errorf("simulation named %s already exists", name)
+	}
+
+	if err := sim.storeNamedTipSet("start", head); err != nil {
+		return nil, xerrors.Errorf("failed to set simulation start: %w", err)
+	}
+
+	if err := sim.SetHead(head); err != nil {
+		return nil, err
+	}
+
+	return sim, nil
+}
+
+// ListSims lists all simulations.
+func (nd *Node) ListSims(ctx context.Context) ([]string, error) {
+	prefix := simulationPrefix.ChildString("head").String()
+	items, err := nd.MetadataDS.Query(query.Query{
+		Prefix:   prefix,
+		KeysOnly: true,
+		Orders:   []query.Order{query.OrderByKey{}},
+	})
+	if err != nil {
+		return nil, xerrors.Errorf("failed to list simulations: %w", err)
+	}
+
+	defer func() { _ = items.Close() }()
+
+	var names []string
+	for {
+		select {
+		case result, ok := <-items.Next():
+			if !ok {
+				return names, nil
+			}
+			if result.Error != nil {
+				return nil, xerrors.Errorf("failed to retrieve next simulation: %w", result.Error)
+			}
+			names = append(names, strings.TrimPrefix(result.Key, prefix+"/"))
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+}
+
+var simFields = []string{"head", "start", "config"}
+
+// DeleteSim deletes a simulation and all related metadata.
+//
+// NOTE: This function does not delete associated messages, blocks, or chain state.
+func (nd *Node) DeleteSim(ctx context.Context, name string) error {
+	var err error
+	for _, field := range simFields {
+		key := simulationPrefix.ChildString(field).ChildString(name)
+		err = multierr.Append(err, nd.MetadataDS.Delete(key))
+	}
+	return err
+}
+
+// CopySim copies a simulation.
+func (nd *Node) CopySim(ctx context.Context, oldName, newName string) error {
+	if strings.Contains(newName, "/") {
+		return xerrors.Errorf("simulation name %q cannot contain a '/'", newName)
+	}
+	if strings.Contains(oldName, "/") {
+		return xerrors.Errorf("simulation name %q cannot contain a '/'", oldName)
+	}
+
+	values := make(map[string][]byte)
+	for _, field := range simFields {
+		key := simulationPrefix.ChildString(field).ChildString(oldName)
+		value, err := nd.MetadataDS.Get(key)
+		if err == datastore.ErrNotFound {
+			continue
+		} else if err != nil {
+			return err
+		}
+		values[field] = value
+	}
+
+	if _, ok := values["head"]; !ok {
+		return xerrors.Errorf("simulation named %s not found", oldName)
+	}
+
+	for _, field := range simFields {
+		key := simulationPrefix.ChildString(field).ChildString(newName)
+		var err error
+		if value, ok := values[field]; ok {
+			err = nd.MetadataDS.Put(key, value)
+		} else {
+			err = nd.MetadataDS.Delete(key)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// RenameSim renames a simulation.
+func (nd *Node) RenameSim(ctx context.Context, oldName, newName string) error {
+	if err := nd.CopySim(ctx, oldName, newName); err != nil {
+		return err
+	}
+	return nd.DeleteSim(ctx, oldName)
+}

cmd/lotus-sim/simulation/simulation.go

@@ -0,0 +1,408 @@
+package simulation
+
+import (
+	"context"
+	"encoding/json"
+	"runtime"
+
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/network"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-datastore"
+	logging "github.com/ipfs/go-log/v2"
+
+	blockadt "github.com/filecoin-project/specs-actors/actors/util/adt"
+
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/stages"
+)
+
+var log = logging.Logger("simulation")
+
+// config is the simulation's config, persisted to the local metadata store and loaded on start.
+//
+// See Simulation.loadConfig and Simulation.saveConfig.
+type config struct {
+	Upgrades map[network.Version]abi.ChainEpoch
+}
+
+// upgradeSchedule constructs an stmgr.StateManager upgrade schedule, overriding any network upgrade
+// epochs as specified in the config.
+func (c *config) upgradeSchedule() (stmgr.UpgradeSchedule, error) {
+	upgradeSchedule := stmgr.DefaultUpgradeSchedule()
+	expected := make(map[network.Version]struct{}, len(c.Upgrades))
+	for nv := range c.Upgrades {
+		expected[nv] = struct{}{}
+	}
+
+	// Update network upgrade epochs.
+	newUpgradeSchedule := upgradeSchedule[:0]
+	for _, upgrade := range upgradeSchedule {
+		if height, ok := c.Upgrades[upgrade.Network]; ok {
+			delete(expected, upgrade.Network)
+			if height < 0 {
+				continue
+			}
+			upgrade.Height = height
+		}
+		newUpgradeSchedule = append(newUpgradeSchedule, upgrade)
+	}
+
+	// Make sure we didn't try to configure an unknown network version.
+	if len(expected) > 0 {
+		missing := make([]network.Version, 0, len(expected))
+		for nv := range expected {
+			missing = append(missing, nv)
+		}
+		return nil, xerrors.Errorf("unknown network versions %v in config", missing)
+	}
+
+	// Finally, validate it. This ensures we don't change the order of the upgrade or anything
+	// like that.
+	if err := newUpgradeSchedule.Validate(); err != nil {
+		return nil, err
+	}
+	return newUpgradeSchedule, nil
+}
+
+// Simulation specifies a lotus-sim simulation.
+type Simulation struct {
+	Node         *Node
+	StateManager *stmgr.StateManager
+
+	name   string
+	config config
+	start  *types.TipSet
+
+	// head
+	head *types.TipSet
+
+	stages []stages.Stage
+}
+
+// loadConfig loads a simulation's config from the datastore. This must be called on startup and may
+// be called to restore the config from-disk.
+func (sim *Simulation) loadConfig() error {
+	configBytes, err := sim.Node.MetadataDS.Get(sim.key("config"))
+	if err == nil {
+		err = json.Unmarshal(configBytes, &sim.config)
+	}
+	switch err {
+	case nil:
+	case datastore.ErrNotFound:
+		sim.config = config{}
+	default:
+		return xerrors.Errorf("failed to load config: %w", err)
+	}
+	return nil
+}
+
+// saveConfig saves the current config to the datastore. This must be called whenever the config is
+// changed.
+func (sim *Simulation) saveConfig() error {
+	buf, err := json.Marshal(sim.config)
+	if err != nil {
+		return err
+	}
+	return sim.Node.MetadataDS.Put(sim.key("config"), buf)
+}
+
+var simulationPrefix = datastore.NewKey("/simulation")
+
+// key returns the the key in the form /simulation/<subkey>/<simulation-name>. For example,
+// /simulation/head/default.
+func (sim *Simulation) key(subkey string) datastore.Key {
+	return simulationPrefix.ChildString(subkey).ChildString(sim.name)
+}
+
+// loadNamedTipSet the tipset with the given name (for this simulation)
+func (sim *Simulation) loadNamedTipSet(name string) (*types.TipSet, error) {
+	tskBytes, err := sim.Node.MetadataDS.Get(sim.key(name))
+	if err != nil {
+		return nil, xerrors.Errorf("failed to load tipset %s/%s: %w", sim.name, name, err)
+	}
+	tsk, err := types.TipSetKeyFromBytes(tskBytes)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to parse tipste %v (%s/%s): %w", tskBytes, sim.name, name, err)
+	}
+	ts, err := sim.Node.Chainstore.LoadTipSet(tsk)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to load tipset %s (%s/%s): %w", tsk, sim.name, name, err)
+	}
+	return ts, nil
+}
+
+// storeNamedTipSet stores the tipset at name (relative to the simulation).
+func (sim *Simulation) storeNamedTipSet(name string, ts *types.TipSet) error {
+	if err := sim.Node.MetadataDS.Put(sim.key(name), ts.Key().Bytes()); err != nil {
+		return xerrors.Errorf("failed to store tipset (%s/%s): %w", sim.name, name, err)
+	}
+	return nil
+}
+
+// GetHead returns the current simulation head.
+func (sim *Simulation) GetHead() *types.TipSet {
+	return sim.head
+}
+
+// GetStart returns simulation's parent tipset.
+func (sim *Simulation) GetStart() *types.TipSet {
+	return sim.start
+}
+
+// GetNetworkVersion returns the current network version for the simulation.
+func (sim *Simulation) GetNetworkVersion() network.Version {
+	return sim.StateManager.GetNtwkVersion(context.TODO(), sim.head.Height())
+}
+
+// SetHead updates the current head of the simulation and stores it in the metadata store. This is
+// called for every Simulation.Step.
+func (sim *Simulation) SetHead(head *types.TipSet) error {
+	if err := sim.storeNamedTipSet("head", head); err != nil {
+		return err
+	}
+	sim.head = head
+	return nil
+}
+
+// Name returns the simulation's name.
+func (sim *Simulation) Name() string {
+	return sim.name
+}
+
+// SetUpgradeHeight sets the height of the given network version change (and saves the config).
+//
+// This fails if the specified epoch has already passed or the new upgrade schedule is invalid.
+func (sim *Simulation) SetUpgradeHeight(nv network.Version, epoch abi.ChainEpoch) (_err error) {
+	if epoch <= sim.head.Height() {
+		return xerrors.Errorf("cannot set upgrade height in the past (%d <= %d)", epoch, sim.head.Height())
+	}
+
+	if sim.config.Upgrades == nil {
+		sim.config.Upgrades = make(map[network.Version]abi.ChainEpoch, 1)
+	}
+
+	sim.config.Upgrades[nv] = epoch
+	defer func() {
+		if _err != nil {
+			// try to restore the old config on error.
+			_ = sim.loadConfig()
+		}
+	}()
+
+	newUpgradeSchedule, err := sim.config.upgradeSchedule()
+	if err != nil {
+		return err
+	}
+	sm, err := stmgr.NewStateManagerWithUpgradeSchedule(sim.Node.Chainstore, newUpgradeSchedule)
+	if err != nil {
+		return err
+	}
+	err = sim.saveConfig()
+	if err != nil {
+		return err
+	}
+
+	sim.StateManager = sm
+	return nil
+}
+
+// ListUpgrades returns any future network upgrades.
+func (sim *Simulation) ListUpgrades() (stmgr.UpgradeSchedule, error) {
+	upgrades, err := sim.config.upgradeSchedule()
+	if err != nil {
+		return nil, err
+	}
+	var pending stmgr.UpgradeSchedule
+	for _, upgrade := range upgrades {
+		if upgrade.Height < sim.head.Height() {
+			continue
+		}
+		pending = append(pending, upgrade)
+	}
+	return pending, nil
+}
+
+type AppliedMessage struct {
+	types.Message
+	types.MessageReceipt
+}
+
+// Walk walks the simulation's chain from the current head back to the first tipset.
+func (sim *Simulation) Walk(
+	ctx context.Context,
+	lookback int64,
+	cb func(sm *stmgr.StateManager,
+		ts *types.TipSet,
+		stCid cid.Cid,
+		messages []*AppliedMessage) error,
+) error {
+	store := sim.Node.Chainstore.ActorStore(ctx)
+	minEpoch := sim.start.Height()
+	if lookback != 0 {
+		minEpoch = sim.head.Height() - abi.ChainEpoch(lookback)
+	}
+
+	// Given tha loading messages and receipts can be a little bit slow, we do this in parallel.
+	//
+	// 1. We spin up some number of workers.
+	// 2. We hand tipsets to workers in round-robin order.
+	// 3. We pull "resolved" tipsets in the same round-robin order.
+	// 4. We serially call the callback in reverse-chain order.
+	//
+	// We have a buffer of size 1 for both resolved tipsets and unresolved tipsets. This should
+	// ensure that we never block unecessarily.
+
+	type work struct {
+		ts     *types.TipSet
+		stCid  cid.Cid
+		recCid cid.Cid
+	}
+	type result struct {
+		ts       *types.TipSet
+		stCid    cid.Cid
+		messages []*AppliedMessage
+	}
+
+	// This is more disk bound than CPU bound, but eh...
+	workerCount := runtime.NumCPU() * 2
+
+	workQs := make([]chan *work, workerCount)
+	resultQs := make([]chan *result, workerCount)
+
+	for i := range workQs {
+		workQs[i] = make(chan *work, 1)
+	}
+
+	for i := range resultQs {
+		resultQs[i] = make(chan *result, 1)
+	}
+
+	grp, ctx := errgroup.WithContext(ctx)
+
+	// Walk the chain and fire off work items.
+	grp.Go(func() error {
+		ts := sim.head
+		stCid, recCid, err := sim.StateManager.TipSetState(ctx, ts)
+		if err != nil {
+			return err
+		}
+		i := 0
+		for ts.Height() > minEpoch {
+			if err := ctx.Err(); err != nil {
+				return ctx.Err()
+			}
+
+			select {
+			case workQs[i] <- &work{ts, stCid, recCid}:
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+
+			stCid = ts.MinTicketBlock().ParentStateRoot
+			recCid = ts.MinTicketBlock().ParentMessageReceipts
+			ts, err = sim.Node.Chainstore.LoadTipSet(ts.Parents())
+			if err != nil {
+				return xerrors.Errorf("loading parent: %w", err)
+			}
+			i = (i + 1) % workerCount
+		}
+		for _, q := range workQs {
+			close(q)
+		}
+		return nil
+	})
+
+	// Spin up one worker per queue pair.
+	for i := 0; i < workerCount; i++ {
+		workQ := workQs[i]
+		resultQ := resultQs[i]
+		grp.Go(func() error {
+			for {
+				if err := ctx.Err(); err != nil {
+					return ctx.Err()
+				}
+
+				var job *work
+				var ok bool
+				select {
+				case job, ok = <-workQ:
+				case <-ctx.Done():
+					return ctx.Err()
+				}
+
+				if !ok {
+					break
+				}
+
+				msgs, err := sim.Node.Chainstore.MessagesForTipset(job.ts)
+				if err != nil {
+					return err
+				}
+
+				recs, err := blockadt.AsArray(store, job.recCid)
+				if err != nil {
+					return xerrors.Errorf("amt load: %w", err)
+				}
+				applied := make([]*AppliedMessage, len(msgs))
+				var rec types.MessageReceipt
+				err = recs.ForEach(&rec, func(i int64) error {
+					applied[i] = &AppliedMessage{
+						Message:        *msgs[i].VMMessage(),
+						MessageReceipt: rec,
+					}
+					return nil
+				})
+				if err != nil {
+					return err
+				}
+				select {
+				case resultQ <- &result{
+					ts:       job.ts,
+					stCid:    job.stCid,
+					messages: applied,
+				}:
+				case <-ctx.Done():
+					return ctx.Err()
+				}
+			}
+			close(resultQ)
+			return nil
+		})
+	}
+
+	// Process results in the same order we enqueued them.
+	grp.Go(func() error {
+		qs := resultQs
+		for len(qs) > 0 {
+			newQs := qs[:0]
+			for _, q := range qs {
+				if err := ctx.Err(); err != nil {
+					return ctx.Err()
+				}
+				select {
+				case r, ok := <-q:
+					if !ok {
+						continue
+					}
+					err := cb(sim.StateManager, r.ts, r.stCid, r.messages)
+					if err != nil {
+						return err
+					}
+				case <-ctx.Done():
+					return ctx.Err()
+				}
+				newQs = append(newQs, q)
+			}
+			qs = newQs
+		}
+		return nil
+	})
+
+	// Wait for everything to finish.
+	return grp.Wait()
+}

cmd/lotus-sim/simulation/stages/actor_iter.go

@@ -0,0 +1,38 @@
+package stages
+
+import (
+	"math/rand"
+
+	"github.com/filecoin-project/go-address"
+)
+
+// actorIter is a simple persistent iterator that loops over a set of actors.
+type actorIter struct {
+	actors []address.Address
+	offset int
+}
+
+// shuffle randomly permutes the set of actors.
+func (p *actorIter) shuffle() {
+	rand.Shuffle(len(p.actors), func(i, j int) {
+		p.actors[i], p.actors[j] = p.actors[j], p.actors[i]
+	})
+}
+
+// next returns the next actor's address and advances the iterator.
+func (p *actorIter) next() address.Address {
+	next := p.actors[p.offset]
+	p.offset++
+	p.offset %= len(p.actors)
+	return next
+}
+
+// add adds a new actor to the iterator.
+func (p *actorIter) add(addr address.Address) {
+	p.actors = append(p.actors, addr)
+}
+
+// len returns the number of actors in the iterator.
+func (p *actorIter) len() int {
+	return len(p.actors)
+}

cmd/lotus-sim/simulation/stages/commit_queue.go

@@ -0,0 +1,200 @@
+package stages
+
+import (
+	"sort"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+)
+
+// pendingCommitTracker tracks pending commits per-miner for a single epoch.
+type pendingCommitTracker map[address.Address]minerPendingCommits
+
+// minerPendingCommits tracks a miner's pending commits during a single epoch (grouped by seal proof type).
+type minerPendingCommits map[abi.RegisteredSealProof][]abi.SectorNumber
+
+// finish marks count sectors of the given proof type as "prove-committed".
+func (m minerPendingCommits) finish(proof abi.RegisteredSealProof, count int) {
+	snos := m[proof]
+	if len(snos) < count {
+		panic("not enough sector numbers to finish")
+	} else if len(snos) == count {
+		delete(m, proof)
+	} else {
+		m[proof] = snos[count:]
+	}
+}
+
+// empty returns true if there are no pending commits.
+func (m minerPendingCommits) empty() bool {
+	return len(m) == 0
+}
+
+// count returns the number of pending commits.
+func (m minerPendingCommits) count() int {
+	count := 0
+	for _, snos := range m {
+		count += len(snos)
+	}
+	return count
+}
+
+// commitQueue is used to track pending prove-commits.
+//
+// Miners are processed in round-robin where _all_ commits from a given miner are finished before
+// moving on to the next. This is designed to maximize batching.
+type commitQueue struct {
+	minerQueue []address.Address
+	queue      []pendingCommitTracker
+	offset     abi.ChainEpoch
+}
+
+// ready returns the number of prove-commits ready to be proven at the current epoch. Useful for logging.
+func (q *commitQueue) ready() int {
+	if len(q.queue) == 0 {
+		return 0
+	}
+	count := 0
+	for _, pending := range q.queue[0] {
+		count += pending.count()
+	}
+	return count
+}
+
+// nextMiner returns the next miner to be proved and the set of pending prove commits for that
+// miner. When some number of sectors have successfully been proven, call "finish" so we don't try
+// to prove them again.
+func (q *commitQueue) nextMiner() (address.Address, minerPendingCommits, bool) {
+	if len(q.queue) == 0 {
+		return address.Undef, nil, false
+	}
+	next := q.queue[0]
+
+	// Go through the queue and find the first non-empty batch.
+	for len(q.minerQueue) > 0 {
+		addr := q.minerQueue[0]
+		q.minerQueue = q.minerQueue[1:]
+		pending := next[addr]
+		if !pending.empty() {
+			return addr, pending, true
+		}
+		delete(next, addr)
+	}
+
+	return address.Undef, nil, false
+}
+
+// advanceEpoch will advance to the next epoch. If some sectors were left unproven in the current
+// epoch, they will be "prepended" into the next epochs sector set.
+func (q *commitQueue) advanceEpoch(epoch abi.ChainEpoch) {
+	if epoch < q.offset {
+		panic("cannot roll epoch backwards")
+	}
+	// Now we "roll forwards", merging each epoch we advance over with the next.
+	for len(q.queue) > 1 && q.offset < epoch {
+		curr := q.queue[0]
+		q.queue[0] = nil
+		q.queue = q.queue[1:]
+		q.offset++
+
+		next := q.queue[0]
+
+		// Cleanup empty entries.
+		for addr, pending := range curr {
+			if pending.empty() {
+				delete(curr, addr)
+			}
+		}
+
+		// If the entire level is actually empty, just skip to the next one.
+		if len(curr) == 0 {
+			continue
+		}
+
+		// Otherwise, merge the next into the current.
+		for addr, nextPending := range next {
+			currPending := curr[addr]
+			if currPending.empty() {
+				curr[addr] = nextPending
+				continue
+			}
+			for ty, nextSnos := range nextPending {
+				currSnos := currPending[ty]
+				if len(currSnos) == 0 {
+					currPending[ty] = nextSnos
+					continue
+				}
+				currPending[ty] = append(currSnos, nextSnos...)
+			}
+		}
+		// Now replace next with the merged curr.
+		q.queue[0] = curr
+	}
+	q.offset = epoch
+	if len(q.queue) == 0 {
+		return
+	}
+
+	next := q.queue[0]
+	seenMiners := make(map[address.Address]struct{}, len(q.minerQueue))
+	for _, addr := range q.minerQueue {
+		seenMiners[addr] = struct{}{}
+	}
+
+	// Find the new miners not already in the queue.
+	offset := len(q.minerQueue)
+	for addr, pending := range next {
+		if pending.empty() {
+			delete(next, addr)
+			continue
+		}
+		if _, ok := seenMiners[addr]; ok {
+			continue
+		}
+		q.minerQueue = append(q.minerQueue, addr)
+	}
+
+	// Sort the new miners only.
+	newMiners := q.minerQueue[offset:]
+	sort.Slice(newMiners, func(i, j int) bool {
+		// eh, escape analysis should be fine here...
+		return string(newMiners[i].Bytes()) < string(newMiners[j].Bytes())
+	})
+}
+
+// enquueProveCommit enqueues prove-commit for the given pre-commit for the given miner.
+func (q *commitQueue) enqueueProveCommit(addr address.Address, preCommitEpoch abi.ChainEpoch, info miner.SectorPreCommitInfo) error {
+	// Compute the epoch at which we can start trying to commit.
+	preCommitDelay := policy.GetPreCommitChallengeDelay()
+	minCommitEpoch := preCommitEpoch + preCommitDelay + 1
+
+	// Figure out the offset in the queue.
+	i := int(minCommitEpoch - q.offset)
+	if i < 0 {
+		i = 0
+	}
+
+	// Expand capacity and insert.
+	if cap(q.queue) <= i {
+		pc := make([]pendingCommitTracker, i+1, preCommitDelay*2)
+		copy(pc, q.queue)
+		q.queue = pc
+	} else if len(q.queue) <= i {
+		q.queue = q.queue[:i+1]
+	}
+	tracker := q.queue[i]
+	if tracker == nil {
+		tracker = make(pendingCommitTracker)
+		q.queue[i] = tracker
+	}
+	minerPending := tracker[addr]
+	if minerPending == nil {
+		minerPending = make(minerPendingCommits)
+		tracker[addr] = minerPending
+	}
+	minerPending[info.SealProof] = append(minerPending[info.SealProof], info.SectorNumber)
+	return nil
+}

cmd/lotus-sim/simulation/stages/commit_queue_test.go

@@ -0,0 +1,128 @@
+package stages
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+)
+
+func TestCommitQueue(t *testing.T) {
+	var q commitQueue
+	addr1, err := address.NewIDAddress(1000)
+	require.NoError(t, err)
+	proofType := abi.RegisteredSealProof_StackedDrg64GiBV1_1
+	require.NoError(t, q.enqueueProveCommit(addr1, 0, miner.SectorPreCommitInfo{
+		SealProof:    proofType,
+		SectorNumber: 0,
+	}))
+	require.NoError(t, q.enqueueProveCommit(addr1, 0, miner.SectorPreCommitInfo{
+		SealProof:    proofType,
+		SectorNumber: 1,
+	}))
+	require.NoError(t, q.enqueueProveCommit(addr1, 1, miner.SectorPreCommitInfo{
+		SealProof:    proofType,
+		SectorNumber: 2,
+	}))
+	require.NoError(t, q.enqueueProveCommit(addr1, 1, miner.SectorPreCommitInfo{
+		SealProof:    proofType,
+		SectorNumber: 3,
+	}))
+	require.NoError(t, q.enqueueProveCommit(addr1, 3, miner.SectorPreCommitInfo{
+		SealProof:    proofType,
+		SectorNumber: 4,
+	}))
+	require.NoError(t, q.enqueueProveCommit(addr1, 4, miner.SectorPreCommitInfo{
+		SealProof:    proofType,
+		SectorNumber: 5,
+	}))
+	require.NoError(t, q.enqueueProveCommit(addr1, 6, miner.SectorPreCommitInfo{
+		SealProof:    proofType,
+		SectorNumber: 6,
+	}))
+
+	epoch := abi.ChainEpoch(0)
+	q.advanceEpoch(epoch)
+	_, _, ok := q.nextMiner()
+	require.False(t, ok)
+
+	epoch += policy.GetPreCommitChallengeDelay()
+	q.advanceEpoch(epoch)
+	_, _, ok = q.nextMiner()
+	require.False(t, ok)
+
+	// 0 : empty + non-empty
+	epoch++
+	q.advanceEpoch(epoch)
+	addr, sectors, ok := q.nextMiner()
+	require.True(t, ok)
+	require.Equal(t, sectors.count(), 2)
+	require.Equal(t, addr, addr1)
+	sectors.finish(proofType, 1)
+	require.Equal(t, sectors.count(), 1)
+	require.EqualValues(t, []abi.SectorNumber{1}, sectors[proofType])
+
+	// 1 : non-empty + non-empty
+	epoch++
+	q.advanceEpoch(epoch)
+	addr, sectors, ok = q.nextMiner()
+	require.True(t, ok)
+	require.Equal(t, addr, addr1)
+	require.Equal(t, sectors.count(), 3)
+	require.EqualValues(t, []abi.SectorNumber{1, 2, 3}, sectors[proofType])
+	sectors.finish(proofType, 3)
+	require.Equal(t, sectors.count(), 0)
+
+	// 2 : empty + empty
+	epoch++
+	q.advanceEpoch(epoch)
+	_, _, ok = q.nextMiner()
+	require.False(t, ok)
+
+	// 3 : empty + non-empty
+	epoch++
+	q.advanceEpoch(epoch)
+	_, sectors, ok = q.nextMiner()
+	require.True(t, ok)
+	require.Equal(t, sectors.count(), 1)
+	require.EqualValues(t, []abi.SectorNumber{4}, sectors[proofType])
+
+	// 4 : non-empty + non-empty
+	epoch++
+	q.advanceEpoch(epoch)
+	_, sectors, ok = q.nextMiner()
+	require.True(t, ok)
+	require.Equal(t, sectors.count(), 2)
+	require.EqualValues(t, []abi.SectorNumber{4, 5}, sectors[proofType])
+
+	// 5 : empty + non-empty
+	epoch++
+	q.advanceEpoch(epoch)
+	_, sectors, ok = q.nextMiner()
+	require.True(t, ok)
+	require.Equal(t, sectors.count(), 2)
+	require.EqualValues(t, []abi.SectorNumber{4, 5}, sectors[proofType])
+	sectors.finish(proofType, 1)
+	require.EqualValues(t, []abi.SectorNumber{5}, sectors[proofType])
+
+	// 6
+	epoch++
+	q.advanceEpoch(epoch)
+	_, sectors, ok = q.nextMiner()
+	require.True(t, ok)
+	require.Equal(t, sectors.count(), 2)
+	require.EqualValues(t, []abi.SectorNumber{5, 6}, sectors[proofType])
+
+	// 8
+	epoch += 2
+	q.advanceEpoch(epoch)
+	_, sectors, ok = q.nextMiner()
+	require.True(t, ok)
+	require.Equal(t, sectors.count(), 2)
+	require.EqualValues(t, []abi.SectorNumber{5, 6}, sectors[proofType])
+}

cmd/lotus-sim/simulation/stages/funding_stage.go

@@ -0,0 +1,318 @@
+package stages
+
+import (
+	"bytes"
+	"context"
+	"sort"
+	"time"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/exitcode"
+
+	"github.com/filecoin-project/lotus/chain/actors/aerrors"
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/multisig"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
+)
+
+var (
+	TargetFunds  = abi.TokenAmount(types.MustParseFIL("1000FIL"))
+	MinimumFunds = abi.TokenAmount(types.MustParseFIL("100FIL"))
+)
+
+type FundingStage struct {
+	fundAccount        address.Address
+	taxMin             abi.TokenAmount
+	minFunds, maxFunds abi.TokenAmount
+}
+
+func NewFundingStage() (*FundingStage, error) {
+	// TODO: make all this configurable.
+	addr, err := address.NewIDAddress(100)
+	if err != nil {
+		return nil, err
+	}
+	return &FundingStage{
+		fundAccount: addr,
+		taxMin:      abi.TokenAmount(types.MustParseFIL("1000FIL")),
+		minFunds:    abi.TokenAmount(types.MustParseFIL("1000000FIL")),
+		maxFunds:    abi.TokenAmount(types.MustParseFIL("100000000FIL")),
+	}, nil
+}
+
+func (*FundingStage) Name() string {
+	return "funding"
+}
+
+func (fs *FundingStage) Fund(bb *blockbuilder.BlockBuilder, target address.Address) error {
+	return fs.fund(bb, target, 0)
+}
+
+// sendAndFund "packs" the given message, funding the actor if necessary. It:
+//
+// 1. Tries to send the given message.
+// 2. If that fails, it checks to see if the exit code was ErrInsufficientFunds.
+// 3. If so, it sends 1K FIL from the "burnt funds actor" (because we need to send it from
+//    somewhere) and re-tries the message.0
+func (fs *FundingStage) SendAndFund(bb *blockbuilder.BlockBuilder, msg *types.Message) (res *types.MessageReceipt, err error) {
+	for i := 0; i < 10; i++ {
+		res, err = bb.PushMessage(msg)
+		if err == nil {
+			return res, nil
+		}
+		aerr, ok := err.(aerrors.ActorError)
+		if !ok || aerr.RetCode() != exitcode.ErrInsufficientFunds {
+			return nil, err
+		}
+
+		// Ok, insufficient funds. Let's fund this miner and try again.
+		if err := fs.fund(bb, msg.To, i); err != nil {
+			if !blockbuilder.IsOutOfGas(err) {
+				err = xerrors.Errorf("failed to fund %s: %w", msg.To, err)
+			}
+			return nil, err
+		}
+	}
+	return res, err
+}
+
+// fund funds the target actor with 'TargetFunds << shift' FIL. The "shift" parameter allows us to
+// keep doubling the amount until the intended operation succeeds.
+func (fs *FundingStage) fund(bb *blockbuilder.BlockBuilder, target address.Address, shift int) error {
+	amt := TargetFunds
+	if shift > 0 {
+		if shift >= 8 {
+			shift = 8 // cap
+		}
+		amt = big.Lsh(amt, uint(shift))
+	}
+	_, err := bb.PushMessage(&types.Message{
+		From:   fs.fundAccount,
+		To:     target,
+		Value:  amt,
+		Method: builtin.MethodSend,
+	})
+	return err
+}
+
+func (fs *FundingStage) PackMessages(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
+	st := bb.StateTree()
+	fundAccActor, err := st.GetActor(fs.fundAccount)
+	if err != nil {
+		return err
+	}
+	if fs.minFunds.LessThan(fundAccActor.Balance) {
+		return nil
+	}
+
+	// Ok, we're going to go fund this thing.
+	start := time.Now()
+
+	type actor struct {
+		types.Actor
+		Address address.Address
+	}
+
+	var targets []*actor
+	err = st.ForEach(func(addr address.Address, act *types.Actor) error {
+		// Don't steal from ourselves!
+		if addr == fs.fundAccount {
+			return nil
+		}
+		if act.Balance.LessThan(fs.taxMin) {
+			return nil
+		}
+		if !(builtin.IsAccountActor(act.Code) || builtin.IsMultisigActor(act.Code)) {
+			return nil
+		}
+		targets = append(targets, &actor{*act, addr})
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	balance := fundAccActor.Balance.Copy()
+
+	sort.Slice(targets, func(i, j int) bool {
+		return targets[i].Balance.GreaterThan(targets[j].Balance)
+	})
+
+	store := bb.ActorStore()
+	epoch := bb.Height()
+	actorsVersion := bb.ActorsVersion()
+
+	var accounts, multisigs int
+	defer func() {
+		if _err != nil {
+			return
+		}
+		bb.L().Infow("finished funding the simulation",
+			"duration", time.Since(start),
+			"targets", len(targets),
+			"epoch", epoch,
+			"new-balance", types.FIL(balance),
+			"old-balance", types.FIL(fundAccActor.Balance),
+			"multisigs", multisigs,
+			"accounts", accounts,
+		)
+	}()
+
+	for _, actor := range targets {
+		switch {
+		case builtin.IsAccountActor(actor.Code):
+			if _, err := bb.PushMessage(&types.Message{
+				From:  actor.Address,
+				To:    fs.fundAccount,
+				Value: actor.Balance,
+			}); blockbuilder.IsOutOfGas(err) {
+				return nil
+			} else if err != nil {
+				return err
+			}
+			accounts++
+		case builtin.IsMultisigActor(actor.Code):
+			msigState, err := multisig.Load(store, &actor.Actor)
+			if err != nil {
+				return err
+			}
+
+			threshold, err := msigState.Threshold()
+			if err != nil {
+				return err
+			}
+
+			if threshold > 16 {
+				bb.L().Debugw("ignoring multisig with high threshold",
+					"multisig", actor.Address,
+					"threshold", threshold,
+					"max", 16,
+				)
+				continue
+			}
+
+			locked, err := msigState.LockedBalance(epoch)
+			if err != nil {
+				return err
+			}
+
+			if locked.LessThan(fs.taxMin) {
+				continue // not worth it.
+			}
+
+			allSigners, err := msigState.Signers()
+			if err != nil {
+				return err
+			}
+			signers := make([]address.Address, 0, threshold)
+			for _, signer := range allSigners {
+				actor, err := st.GetActor(signer)
+				if err != nil {
+					return err
+				}
+				if !builtin.IsAccountActor(actor.Code) {
+					// I am so not dealing with this mess.
+					continue
+				}
+				if uint64(len(signers)) >= threshold {
+					break
+				}
+			}
+			// Ok, we're not dealing with this one.
+			if uint64(len(signers)) < threshold {
+				continue
+			}
+
+			available := big.Sub(actor.Balance, locked)
+
+			var txnId uint64
+			{
+				msg, err := multisig.Message(actorsVersion, signers[0]).Propose(
+					actor.Address, fs.fundAccount, available,
+					builtin.MethodSend, nil,
+				)
+				if err != nil {
+					return err
+				}
+				res, err := bb.PushMessage(msg)
+				if err != nil {
+					if blockbuilder.IsOutOfGas(err) {
+						err = nil
+					}
+					return err
+				}
+				var ret multisig.ProposeReturn
+				err = ret.UnmarshalCBOR(bytes.NewReader(res.Return))
+				if err != nil {
+					return err
+				}
+				if ret.Applied {
+					if !ret.Code.IsSuccess() {
+						bb.L().Errorw("failed to tax multisig",
+							"multisig", actor.Address,
+							"exitcode", ret.Code,
+						)
+					}
+					break
+				}
+				txnId = uint64(ret.TxnID)
+			}
+			var ret multisig.ProposeReturn
+			for _, signer := range signers[1:] {
+				msg, err := multisig.Message(actorsVersion, signer).Approve(actor.Address, txnId, nil)
+				if err != nil {
+					return err
+				}
+				res, err := bb.PushMessage(msg)
+				if err != nil {
+					if blockbuilder.IsOutOfGas(err) {
+						err = nil
+					}
+					return err
+				}
+				var ret multisig.ProposeReturn
+				err = ret.UnmarshalCBOR(bytes.NewReader(res.Return))
+				if err != nil {
+					return err
+				}
+				// A bit redundant, but nice.
+				if ret.Applied {
+					break
+				}
+
+			}
+			if !ret.Applied {
+				bb.L().Errorw("failed to apply multisig transaction",
+					"multisig", actor.Address,
+					"txnid", txnId,
+					"signers", len(signers),
+					"threshold", threshold,
+				)
+				continue
+			}
+			if !ret.Code.IsSuccess() {
+				bb.L().Errorw("failed to tax multisig",
+					"multisig", actor.Address,
+					"txnid", txnId,
+					"exitcode", ret.Code,
+				)
+			} else {
+				multisigs++
+			}
+		default:
+			panic("impossible case")
+		}
+		balance = big.Int{Int: balance.Add(balance.Int, actor.Balance.Int)}
+		if balance.GreaterThanEqual(fs.maxFunds) {
+			// There's no need to get greedy.
+			// Well, really, we're trying to avoid messing with state _too_ much.
+			return nil
+		}
+	}
+	return nil
+}

cmd/lotus-sim/simulation/stages/interface.go

@@ -0,0 +1,27 @@
+package stages
+
+import (
+	"context"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
+)
+
+// Stage is a stage of the simulation. It's asked to pack messages for every block.
+type Stage interface {
+	Name() string
+	PackMessages(ctx context.Context, bb *blockbuilder.BlockBuilder) error
+}
+
+type Funding interface {
+	SendAndFund(*blockbuilder.BlockBuilder, *types.Message) (*types.MessageReceipt, error)
+	Fund(*blockbuilder.BlockBuilder, address.Address) error
+}
+
+type Committer interface {
+	EnqueueProveCommit(addr address.Address, preCommitEpoch abi.ChainEpoch, info miner.SectorPreCommitInfo) error
+}

cmd/lotus-sim/simulation/stages/pipeline.go

@@ -0,0 +1,31 @@
+package stages
+
+// DefaultPipeline returns the default stage pipeline. This pipeline.
+//
+// 1. Funds a "funding" actor, if necessary.
+// 2. Submits any ready window posts.
+// 3. Submits any ready prove commits.
+// 4. Submits pre-commits with the remaining gas.
+func DefaultPipeline() ([]Stage, error) {
+	// TODO: make this configurable. E.g., through DI?
+	// Ideally, we'd also be able to change priority, limit throughput (by limiting gas in the
+	// block builder, etc.
+	funding, err := NewFundingStage()
+	if err != nil {
+		return nil, err
+	}
+	wdpost, err := NewWindowPoStStage()
+	if err != nil {
+		return nil, err
+	}
+	provecommit, err := NewProveCommitStage(funding)
+	if err != nil {
+		return nil, err
+	}
+	precommit, err := NewPreCommitStage(funding, provecommit)
+	if err != nil {
+		return nil, err
+	}
+
+	return []Stage{funding, wdpost, provecommit, precommit}, nil
+}

cmd/lotus-sim/simulation/stages/precommit_stage.go

@@ -0,0 +1,347 @@
+package stages
+
+import (
+	"context"
+	"sort"
+	"time"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/network"
+
+	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
+
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/aerrors"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/mock"
+)
+
+const (
+	minPreCommitBatchSize = 1
+	maxPreCommitBatchSize = miner5.PreCommitSectorBatchMaxSize
+)
+
+type PreCommitStage struct {
+	funding   Funding
+	committer Committer
+
+	// The tiers represent the top 1%, top 10%, and everyone else. When sealing sectors, we seal
+	// a group of sectors for the top 1%, a group (half that size) for the top 10%, and one
+	// sector for everyone else. We determine these rates by looking at two power tables.
+	// TODO Ideally we'd "learn" this distribution from the network. But this is good enough for
+	// now.
+	top1, top10, rest actorIter
+	initialized       bool
+}
+
+func NewPreCommitStage(funding Funding, committer Committer) (*PreCommitStage, error) {
+	return &PreCommitStage{
+		funding:   funding,
+		committer: committer,
+	}, nil
+}
+
+func (*PreCommitStage) Name() string {
+	return "pre-commit"
+}
+
+// packPreCommits packs pre-commit messages until the block is full.
+func (stage *PreCommitStage) PackMessages(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
+	if !stage.initialized {
+		if err := stage.load(ctx, bb); err != nil {
+			return err
+		}
+	}
+
+	var (
+		full                             bool
+		top1Count, top10Count, restCount int
+	)
+	start := time.Now()
+	defer func() {
+		if _err != nil {
+			return
+		}
+		bb.L().Debugw("packed pre commits",
+			"done", top1Count+top10Count+restCount,
+			"top1", top1Count,
+			"top10", top10Count,
+			"rest", restCount,
+			"filled-block", full,
+			"duration", time.Since(start),
+		)
+	}()
+
+	var top1Miners, top10Miners, restMiners int
+	for i := 0; ; i++ {
+		var (
+			minerAddr address.Address
+			count     *int
+		)
+
+		// We pre-commit for the top 1%, 10%, and the of the network 1/3rd of the time each.
+		// This won't yield the most accurate distribution... but it'll give us a good
+		// enough distribution.
+		switch {
+		case (i%3) <= 0 && top1Miners < stage.top1.len():
+			count = &top1Count
+			minerAddr = stage.top1.next()
+			top1Miners++
+		case (i%3) <= 1 && top10Miners < stage.top10.len():
+			count = &top10Count
+			minerAddr = stage.top10.next()
+			top10Miners++
+		case (i%3) <= 2 && restMiners < stage.rest.len():
+			count = &restCount
+			minerAddr = stage.rest.next()
+			restMiners++
+		default:
+			// Well, we've run through all miners.
+			return nil
+		}
+
+		var (
+			added int
+			err   error
+		)
+		added, full, err = stage.packMiner(ctx, bb, minerAddr, maxProveCommitBatchSize)
+		if err != nil {
+			return xerrors.Errorf("failed to pack precommits for miner %s: %w", minerAddr, err)
+		}
+		*count += added
+		if full {
+			return nil
+		}
+	}
+}
+
+// packPreCommitsMiner packs count pre-commits for the given miner.
+func (stage *PreCommitStage) packMiner(
+	ctx context.Context, bb *blockbuilder.BlockBuilder,
+	minerAddr address.Address, count int,
+) (int, bool, error) {
+	log := bb.L().With("miner", minerAddr)
+	epoch := bb.Height()
+	nv := bb.NetworkVersion()
+
+	minerActor, err := bb.StateTree().GetActor(minerAddr)
+	if err != nil {
+		return 0, false, err
+	}
+	minerState, err := miner.Load(bb.ActorStore(), minerActor)
+	if err != nil {
+		return 0, false, err
+	}
+
+	minerInfo, err := minerState.Info()
+	if err != nil {
+		return 0, false, err
+	}
+
+	// Make sure the miner is funded.
+	minerBalance, err := minerState.AvailableBalance(minerActor.Balance)
+	if err != nil {
+		return 0, false, err
+	}
+
+	if big.Cmp(minerBalance, MinimumFunds) < 0 {
+		err := stage.funding.Fund(bb, minerAddr)
+		if err != nil {
+			if blockbuilder.IsOutOfGas(err) {
+				return 0, true, nil
+			}
+			return 0, false, err
+		}
+	}
+
+	// Generate pre-commits.
+	sealType, err := miner.PreferredSealProofTypeFromWindowPoStType(
+		nv, minerInfo.WindowPoStProofType,
+	)
+	if err != nil {
+		return 0, false, err
+	}
+
+	sectorNos, err := minerState.UnallocatedSectorNumbers(count)
+	if err != nil {
+		return 0, false, err
+	}
+
+	expiration := epoch + policy.GetMaxSectorExpirationExtension()
+	infos := make([]miner.SectorPreCommitInfo, len(sectorNos))
+	for i, sno := range sectorNos {
+		infos[i] = miner.SectorPreCommitInfo{
+			SealProof:     sealType,
+			SectorNumber:  sno,
+			SealedCID:     mock.MockCommR(minerAddr, sno),
+			SealRandEpoch: epoch - 1,
+			Expiration:    expiration,
+		}
+	}
+
+	// Commit the pre-commits.
+	added := 0
+	if nv >= network.Version13 {
+		targetBatchSize := maxPreCommitBatchSize
+		for targetBatchSize >= minPreCommitBatchSize && len(infos) >= minPreCommitBatchSize {
+			batch := infos
+			if len(batch) > targetBatchSize {
+				batch = batch[:targetBatchSize]
+			}
+			params := miner5.PreCommitSectorBatchParams{
+				Sectors: batch,
+			}
+			enc, err := actors.SerializeParams(&params)
+			if err != nil {
+				return added, false, err
+			}
+			// NOTE: just in-case, sendAndFund will "fund" and re-try for any message
+			// that fails due to "insufficient funds".
+			if _, err := stage.funding.SendAndFund(bb, &types.Message{
+				To:     minerAddr,
+				From:   minerInfo.Worker,
+				Value:  abi.NewTokenAmount(0),
+				Method: miner.Methods.PreCommitSectorBatch,
+				Params: enc,
+			}); blockbuilder.IsOutOfGas(err) {
+				// try again with a smaller batch.
+				targetBatchSize /= 2
+				continue
+			} else if aerr, ok := err.(aerrors.ActorError); ok && !aerr.IsFatal() {
+				// Log the error and move on. No reason to stop.
+				log.Errorw("failed to pre-commit for unknown reasons",
+					"error", aerr,
+					"sectors", batch,
+				)
+				return added, false, nil
+			} else if err != nil {
+				return added, false, err
+			}
+
+			for _, info := range batch {
+				if err := stage.committer.EnqueueProveCommit(minerAddr, epoch, info); err != nil {
+					return added, false, err
+				}
+				added++
+			}
+			infos = infos[len(batch):]
+		}
+	}
+	for _, info := range infos {
+		enc, err := actors.SerializeParams(&info) //nolint
+		if err != nil {
+			return 0, false, err
+		}
+		if _, err := stage.funding.SendAndFund(bb, &types.Message{
+			To:     minerAddr,
+			From:   minerInfo.Worker,
+			Value:  abi.NewTokenAmount(0),
+			Method: miner.Methods.PreCommitSector,
+			Params: enc,
+		}); blockbuilder.IsOutOfGas(err) {
+			return added, true, nil
+		} else if err != nil {
+			return added, false, err
+		}
+
+		if err := stage.committer.EnqueueProveCommit(minerAddr, epoch, info); err != nil {
+			return added, false, err
+		}
+		added++
+	}
+	return added, false, nil
+}
+
+func (stage *PreCommitStage) load(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
+	bb.L().Infow("loading miner power for pre-commits")
+	start := time.Now()
+	defer func() {
+		if _err != nil {
+			return
+		}
+		bb.L().Infow("loaded miner power for pre-commits",
+			"duration", time.Since(start),
+			"top1", stage.top1.len(),
+			"top10", stage.top10.len(),
+			"rest", stage.rest.len(),
+		)
+	}()
+
+	store := bb.ActorStore()
+	st := bb.ParentStateTree()
+	powerState, err := loadPower(store, st)
+	if err != nil {
+		return xerrors.Errorf("failed to power actor: %w", err)
+	}
+
+	type onboardingInfo struct {
+		addr        address.Address
+		sectorCount uint64
+	}
+	var sealList []onboardingInfo
+	err = powerState.ForEachClaim(func(addr address.Address, claim power.Claim) error {
+		if claim.RawBytePower.IsZero() {
+			return nil
+		}
+
+		minerState, err := loadMiner(store, st, addr)
+		if err != nil {
+			return err
+		}
+		info, err := minerState.Info()
+		if err != nil {
+			return err
+		}
+
+		sectorCount := sectorsFromClaim(info.SectorSize, claim)
+
+		if sectorCount > 0 {
+			sealList = append(sealList, onboardingInfo{addr, uint64(sectorCount)})
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	if len(sealList) == 0 {
+		return xerrors.Errorf("simulation has no miners")
+	}
+
+	// Now that we have a list of sealing miners, sort them into percentiles.
+	sort.Slice(sealList, func(i, j int) bool {
+		return sealList[i].sectorCount < sealList[j].sectorCount
+	})
+
+	// reset, just in case.
+	stage.top1 = actorIter{}
+	stage.top10 = actorIter{}
+	stage.rest = actorIter{}
+
+	for i, oi := range sealList {
+		var dist *actorIter
+		if i < len(sealList)/100 {
+			dist = &stage.top1
+		} else if i < len(sealList)/10 {
+			dist = &stage.top10
+		} else {
+			dist = &stage.rest
+		}
+		dist.add(oi.addr)
+	}
+
+	stage.top1.shuffle()
+	stage.top10.shuffle()
+	stage.rest.shuffle()
+
+	stage.initialized = true
+	return nil
+}

cmd/lotus-sim/simulation/stages/provecommit_stage.go

@@ -0,0 +1,372 @@
+package stages
+
+import (
+	"context"
+	"time"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-bitfield"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/exitcode"
+	"github.com/filecoin-project/go-state-types/network"
+
+	miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner"
+	power5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/power"
+
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/aerrors"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/mock"
+)
+
+const (
+	minProveCommitBatchSize = 4
+	maxProveCommitBatchSize = miner5.MaxAggregatedSectors
+)
+
+type ProveCommitStage struct {
+	funding Funding
+	// We track the set of pending commits. On simulation load, and when a new pre-commit is
+	// added to the chain, we put the commit in this queue. advanceEpoch(currentEpoch) should be
+	// called on this queue at every epoch before using it.
+	commitQueue commitQueue
+	initialized bool
+}
+
+func NewProveCommitStage(funding Funding) (*ProveCommitStage, error) {
+	return &ProveCommitStage{
+		funding: funding,
+	}, nil
+}
+
+func (*ProveCommitStage) Name() string {
+	return "prove-commit"
+}
+
+func (stage *ProveCommitStage) EnqueueProveCommit(
+	minerAddr address.Address, preCommitEpoch abi.ChainEpoch, info miner.SectorPreCommitInfo,
+) error {
+	return stage.commitQueue.enqueueProveCommit(minerAddr, preCommitEpoch, info)
+}
+
+// packProveCommits packs all prove-commits for all "ready to be proven" sectors until it fills the
+// block or runs out.
+func (stage *ProveCommitStage) PackMessages(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
+	if !stage.initialized {
+		if err := stage.load(ctx, bb); err != nil {
+			return err
+		}
+	}
+	// Roll the commitQueue forward.
+	stage.commitQueue.advanceEpoch(bb.Height())
+
+	start := time.Now()
+	var failed, done, unbatched, count int
+	defer func() {
+		if _err != nil {
+			return
+		}
+		remaining := stage.commitQueue.ready()
+		bb.L().Debugw("packed prove commits",
+			"remaining", remaining,
+			"done", done,
+			"failed", failed,
+			"unbatched", unbatched,
+			"miners-processed", count,
+			"duration", time.Since(start),
+		)
+	}()
+
+	for {
+		addr, pending, ok := stage.commitQueue.nextMiner()
+		if !ok {
+			return nil
+		}
+
+		res, err := stage.packProveCommitsMiner(ctx, bb, addr, pending)
+		if err != nil {
+			return err
+		}
+		failed += res.failed
+		done += res.done
+		unbatched += res.unbatched
+		count++
+		if res.full {
+			return nil
+		}
+	}
+}
+
+type proveCommitResult struct {
+	done, failed, unbatched int
+	full                    bool
+}
+
+// packProveCommitsMiner enqueues a prove commits from the given miner until it runs out of
+// available prove-commits, batching as much as possible.
+//
+// This function will fund as necessary from the "burnt funds actor" (look, it's convenient).
+func (stage *ProveCommitStage) packProveCommitsMiner(
+	ctx context.Context, bb *blockbuilder.BlockBuilder, minerAddr address.Address,
+	pending minerPendingCommits,
+) (res proveCommitResult, _err error) {
+	minerActor, err := bb.StateTree().GetActor(minerAddr)
+	if err != nil {
+		return res, err
+	}
+	minerState, err := miner.Load(bb.ActorStore(), minerActor)
+	if err != nil {
+		return res, err
+	}
+	info, err := minerState.Info()
+	if err != nil {
+		return res, err
+	}
+
+	log := bb.L().With("miner", minerAddr)
+
+	nv := bb.NetworkVersion()
+	for sealType, snos := range pending {
+		if nv >= network.Version13 {
+			for len(snos) > minProveCommitBatchSize {
+				batchSize := maxProveCommitBatchSize
+				if len(snos) < batchSize {
+					batchSize = len(snos)
+				}
+				batch := snos[:batchSize]
+
+				proof, err := mock.MockAggregateSealProof(sealType, minerAddr, batchSize)
+				if err != nil {
+					return res, err
+				}
+
+				params := miner5.ProveCommitAggregateParams{
+					SectorNumbers:  bitfield.New(),
+					AggregateProof: proof,
+				}
+				for _, sno := range batch {
+					params.SectorNumbers.Set(uint64(sno))
+				}
+
+				enc, err := actors.SerializeParams(&params)
+				if err != nil {
+					return res, err
+				}
+
+				if _, err := stage.funding.SendAndFund(bb, &types.Message{
+					From:   info.Worker,
+					To:     minerAddr,
+					Value:  abi.NewTokenAmount(0),
+					Method: miner.Methods.ProveCommitAggregate,
+					Params: enc,
+				}); err == nil {
+					res.done += len(batch)
+				} else if blockbuilder.IsOutOfGas(err) {
+					res.full = true
+					return res, nil
+				} else if aerr, ok := err.(aerrors.ActorError); !ok || aerr.IsFatal() {
+					// If we get a random error, or a fatal actor error, bail.
+					return res, err
+				} else if aerr.RetCode() == exitcode.ErrNotFound || aerr.RetCode() == exitcode.ErrIllegalArgument {
+					// If we get a "not-found" or illegal argument error, try to
+					// remove any missing prove-commits and continue. This can
+					// happen either because:
+					//
+					// 1. The pre-commit failed on execution (but not when
+					// packing). This shouldn't happen, but we might as well
+					// gracefully handle it.
+					// 2. The pre-commit has expired. We'd have to be really
+					// backloged to hit this case, but we might as well handle
+					// it.
+					// First, split into "good" and "missing"
+					good, err := stage.filterProveCommits(ctx, bb, minerAddr, batch)
+					if err != nil {
+						log.Errorw("failed to filter prove commits", "error", err)
+						// fail with the original error.
+						return res, aerr
+					}
+					removed := len(batch) - len(good)
+					if removed == 0 {
+						log.Errorw("failed to prove-commit for unknown reasons",
+							"error", aerr,
+							"sectors", batch,
+						)
+						res.failed += len(batch)
+					} else if len(good) == 0 {
+						log.Errorw("failed to prove commit missing pre-commits",
+							"error", aerr,
+							"discarded", removed,
+						)
+						res.failed += len(batch)
+					} else {
+						// update the pending sector numbers in-place to remove the expired ones.
+						snos = snos[removed:]
+						copy(snos, good)
+						pending.finish(sealType, removed)
+
+						log.Errorw("failed to prove commit expired/missing pre-commits",
+							"error", aerr,
+							"discarded", removed,
+							"kept", len(good),
+						)
+						res.failed += removed
+
+						// Then try again.
+						continue
+					}
+				} else {
+					log.Errorw("failed to prove commit sector(s)",
+						"error", err,
+						"sectors", batch,
+					)
+					res.failed += len(batch)
+				}
+				pending.finish(sealType, len(batch))
+				snos = snos[len(batch):]
+			}
+		}
+		for len(snos) > 0 && res.unbatched < power5.MaxMinerProveCommitsPerEpoch {
+			sno := snos[0]
+			snos = snos[1:]
+
+			proof, err := mock.MockSealProof(sealType, minerAddr)
+			if err != nil {
+				return res, err
+			}
+			params := miner.ProveCommitSectorParams{
+				SectorNumber: sno,
+				Proof:        proof,
+			}
+			enc, err := actors.SerializeParams(&params)
+			if err != nil {
+				return res, err
+			}
+			if _, err := stage.funding.SendAndFund(bb, &types.Message{
+				From:   info.Worker,
+				To:     minerAddr,
+				Value:  abi.NewTokenAmount(0),
+				Method: miner.Methods.ProveCommitSector,
+				Params: enc,
+			}); err == nil {
+				res.unbatched++
+				res.done++
+			} else if blockbuilder.IsOutOfGas(err) {
+				res.full = true
+				return res, nil
+			} else if aerr, ok := err.(aerrors.ActorError); !ok || aerr.IsFatal() {
+				return res, err
+			} else {
+				log.Errorw("failed to prove commit sector(s)",
+					"error", err,
+					"sectors", []abi.SectorNumber{sno},
+				)
+				res.failed++
+			}
+			// mark it as "finished" regardless so we skip it.
+			pending.finish(sealType, 1)
+		}
+		// if we get here, we can't pre-commit anything more.
+	}
+	return res, nil
+}
+
+// loadMiner enqueue all pending prove-commits for the given miner. This is called on load to
+// populate the commitQueue and should not need to be called later.
+//
+// It will drop any pre-commits that have already expired.
+func (stage *ProveCommitStage) loadMiner(ctx context.Context, bb *blockbuilder.BlockBuilder, addr address.Address) error {
+	epoch := bb.Height()
+	av := bb.ActorsVersion()
+	minerState, err := loadMiner(bb.ActorStore(), bb.ParentStateTree(), addr)
+	if err != nil {
+		return err
+	}
+
+	// Find all pending prove commits and group by proof type. Really, there should never
+	// (except during upgrades be more than one type.
+	var total, dropped int
+	err = minerState.ForEachPrecommittedSector(func(info miner.SectorPreCommitOnChainInfo) error {
+		total++
+		msd := policy.GetMaxProveCommitDuration(av, info.Info.SealProof)
+		if epoch > info.PreCommitEpoch+msd {
+			dropped++
+			return nil
+		}
+		return stage.commitQueue.enqueueProveCommit(addr, info.PreCommitEpoch, info.Info)
+	})
+	if err != nil {
+		return err
+	}
+	if dropped > 0 {
+		bb.L().Warnw("dropped expired pre-commits on load",
+			"miner", addr,
+			"total", total,
+			"expired", dropped,
+		)
+	}
+	return nil
+}
+
+// filterProveCommits filters out expired and/or missing pre-commits.
+func (stage *ProveCommitStage) filterProveCommits(
+	ctx context.Context, bb *blockbuilder.BlockBuilder,
+	minerAddr address.Address, snos []abi.SectorNumber,
+) ([]abi.SectorNumber, error) {
+	act, err := bb.StateTree().GetActor(minerAddr)
+	if err != nil {
+		return nil, err
+	}
+
+	minerState, err := miner.Load(bb.ActorStore(), act)
+	if err != nil {
+		return nil, err
+	}
+
+	nextEpoch := bb.Height()
+	av := bb.ActorsVersion()
+
+	good := make([]abi.SectorNumber, 0, len(snos))
+	for _, sno := range snos {
+		info, err := minerState.GetPrecommittedSector(sno)
+		if err != nil {
+			return nil, err
+		}
+		if info == nil {
+			continue
+		}
+		msd := policy.GetMaxProveCommitDuration(av, info.Info.SealProof)
+		if nextEpoch > info.PreCommitEpoch+msd {
+			continue
+		}
+		good = append(good, sno)
+	}
+	return good, nil
+}
+
+func (stage *ProveCommitStage) load(ctx context.Context, bb *blockbuilder.BlockBuilder) error {
+	stage.initialized = false // in case something failes while we're doing this.
+	stage.commitQueue = commitQueue{offset: bb.Height()}
+	powerState, err := loadPower(bb.ActorStore(), bb.ParentStateTree())
+	if err != nil {
+		return err
+	}
+
+	err = powerState.ForEachClaim(func(minerAddr address.Address, claim power.Claim) error {
+		// TODO: If we want to finish pre-commits for "new" miners, we'll need to change
+		// this.
+		if claim.RawBytePower.IsZero() {
+			return nil
+		}
+		return stage.loadMiner(ctx, bb, minerAddr)
+	})
+	if err != nil {
+		return err
+	}
+
+	stage.initialized = true
+	return nil
+}

cmd/lotus-sim/simulation/stages/util.go

@@ -0,0 +1,51 @@
+package stages
+
+import (
+	"context"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/crypto"
+
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
+)
+
+func loadMiner(store adt.Store, st types.StateTree, addr address.Address) (miner.State, error) {
+	minerActor, err := st.GetActor(addr)
+	if err != nil {
+		return nil, err
+	}
+	return miner.Load(store, minerActor)
+}
+
+func loadPower(store adt.Store, st types.StateTree) (power.State, error) {
+	powerActor, err := st.GetActor(power.Address)
+	if err != nil {
+		return nil, err
+	}
+	return power.Load(store, powerActor)
+}
+
+// Compute the number of sectors a miner has from their power claim.
+func sectorsFromClaim(sectorSize abi.SectorSize, c power.Claim) int64 {
+	if c.RawBytePower.Int == nil {
+		return 0
+	}
+	sectorCount := big.Div(c.RawBytePower, big.NewIntUnsigned(uint64(sectorSize)))
+	if !sectorCount.IsInt64() {
+		panic("impossible number of sectors")
+	}
+	return sectorCount.Int64()
+}
+
+func postChainCommitInfo(ctx context.Context, bb *blockbuilder.BlockBuilder, epoch abi.ChainEpoch) (abi.Randomness, error) {
+	cs := bb.StateManager().ChainStore()
+	ts := bb.ParentTipSet()
+	commitRand, err := cs.GetChainRandomness(ctx, ts.Cids(), crypto.DomainSeparationTag_PoStChainCommit, epoch, nil, true)
+	return commitRand, err
+}

cmd/lotus-sim/simulation/stages/windowpost_stage.go

@@ -0,0 +1,317 @@
+package stages
+
+import (
+	"context"
+	"math"
+	"time"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+
+	proof5 "github.com/filecoin-project/specs-actors/v5/actors/runtime/proof"
+
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/aerrors"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
+	"github.com/filecoin-project/lotus/chain/actors/policy"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/mock"
+)
+
+type WindowPoStStage struct {
+	// We track the window post periods per miner and assume that no new miners are ever added.
+
+	// We record all pending window post messages, and the epoch up through which we've
+	// generated window post messages.
+	pendingWposts  []*types.Message
+	wpostPeriods   [][]address.Address // (epoch % (epochs in a deadline)) -> miner
+	nextWpostEpoch abi.ChainEpoch
+}
+
+func NewWindowPoStStage() (*WindowPoStStage, error) {
+	return new(WindowPoStStage), nil
+}
+
+func (*WindowPoStStage) Name() string {
+	return "window-post"
+}
+
+// packWindowPoSts packs window posts until either the block is full or all healty sectors
+// have been proven. It does not recover sectors.
+func (stage *WindowPoStStage) PackMessages(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
+	// Push any new window posts into the queue.
+	if err := stage.tick(ctx, bb); err != nil {
+		return err
+	}
+	done := 0
+	failed := 0
+	defer func() {
+		if _err != nil {
+			return
+		}
+
+		bb.L().Debugw("packed window posts",
+			"done", done,
+			"failed", failed,
+			"remaining", len(stage.pendingWposts),
+		)
+	}()
+	// Then pack as many as we can.
+	for len(stage.pendingWposts) > 0 {
+		next := stage.pendingWposts[0]
+		if _, err := bb.PushMessage(next); err != nil {
+			if blockbuilder.IsOutOfGas(err) {
+				return nil
+			}
+			if aerr, ok := err.(aerrors.ActorError); !ok || aerr.IsFatal() {
+				return err
+			}
+			bb.L().Errorw("failed to submit windowed post",
+				"error", err,
+				"miner", next.To,
+			)
+			failed++
+		} else {
+			done++
+		}
+
+		stage.pendingWposts = stage.pendingWposts[1:]
+	}
+	stage.pendingWposts = nil
+	return nil
+}
+
+// stepWindowPoStsMiner enqueues all missing window posts for the current epoch for the given miner.
+func (stage *WindowPoStStage) queueMiner(
+	ctx context.Context, bb *blockbuilder.BlockBuilder,
+	addr address.Address, minerState miner.State,
+	commitEpoch abi.ChainEpoch, commitRand abi.Randomness,
+) error {
+
+	if active, err := minerState.DeadlineCronActive(); err != nil {
+		return err
+	} else if !active {
+		return nil
+	}
+
+	minerInfo, err := minerState.Info()
+	if err != nil {
+		return err
+	}
+
+	di, err := minerState.DeadlineInfo(bb.Height())
+	if err != nil {
+		return err
+	}
+	di = di.NextNotElapsed()
+
+	dl, err := minerState.LoadDeadline(di.Index)
+	if err != nil {
+		return err
+	}
+
+	provenBf, err := dl.PartitionsPoSted()
+	if err != nil {
+		return err
+	}
+	proven, err := provenBf.AllMap(math.MaxUint64)
+	if err != nil {
+		return err
+	}
+
+	poStBatchSize, err := policy.GetMaxPoStPartitions(bb.NetworkVersion(), minerInfo.WindowPoStProofType)
+	if err != nil {
+		return err
+	}
+
+	var (
+		partitions      []miner.PoStPartition
+		partitionGroups [][]miner.PoStPartition
+	)
+	// Only prove partitions with live sectors.
+	err = dl.ForEachPartition(func(idx uint64, part miner.Partition) error {
+		if proven[idx] {
+			return nil
+		}
+		// NOTE: We're mimicing the behavior of wdpost_run.go here.
+		if len(partitions) > 0 && idx%uint64(poStBatchSize) == 0 {
+			partitionGroups = append(partitionGroups, partitions)
+			partitions = nil
+
+		}
+		live, err := part.LiveSectors()
+		if err != nil {
+			return err
+		}
+		liveCount, err := live.Count()
+		if err != nil {
+			return err
+		}
+		faulty, err := part.FaultySectors()
+		if err != nil {
+			return err
+		}
+		faultyCount, err := faulty.Count()
+		if err != nil {
+			return err
+		}
+		if liveCount-faultyCount > 0 {
+			partitions = append(partitions, miner.PoStPartition{Index: idx})
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	if len(partitions) > 0 {
+		partitionGroups = append(partitionGroups, partitions)
+		partitions = nil
+	}
+
+	proof, err := mock.MockWindowPoStProof(minerInfo.WindowPoStProofType, addr)
+	if err != nil {
+		return err
+	}
+	for _, group := range partitionGroups {
+		params := miner.SubmitWindowedPoStParams{
+			Deadline:   di.Index,
+			Partitions: group,
+			Proofs: []proof5.PoStProof{{
+				PoStProof:  minerInfo.WindowPoStProofType,
+				ProofBytes: proof,
+			}},
+			ChainCommitEpoch: commitEpoch,
+			ChainCommitRand:  commitRand,
+		}
+		enc, aerr := actors.SerializeParams(&params)
+		if aerr != nil {
+			return xerrors.Errorf("could not serialize submit window post parameters: %w", aerr)
+		}
+		msg := &types.Message{
+			To:     addr,
+			From:   minerInfo.Worker,
+			Method: miner.Methods.SubmitWindowedPoSt,
+			Params: enc,
+			Value:  types.NewInt(0),
+		}
+		stage.pendingWposts = append(stage.pendingWposts, msg)
+	}
+	return nil
+}
+
+func (stage *WindowPoStStage) load(ctx context.Context, bb *blockbuilder.BlockBuilder) (_err error) {
+	bb.L().Info("loading window post info")
+
+	start := time.Now()
+	defer func() {
+		if _err != nil {
+			return
+		}
+
+		bb.L().Infow("loaded window post info", "duration", time.Since(start))
+	}()
+
+	// reset
+	stage.wpostPeriods = make([][]address.Address, miner.WPoStChallengeWindow)
+	stage.pendingWposts = nil
+	stage.nextWpostEpoch = bb.Height() + 1
+
+	st := bb.ParentStateTree()
+	store := bb.ActorStore()
+
+	powerState, err := loadPower(store, st)
+	if err != nil {
+		return err
+	}
+
+	commitEpoch := bb.ParentTipSet().Height()
+	commitRand, err := postChainCommitInfo(ctx, bb, commitEpoch)
+	if err != nil {
+		return err
+	}
+
+	return powerState.ForEachClaim(func(minerAddr address.Address, claim power.Claim) error {
+		// TODO: If we start recovering power, we'll need to change this.
+		if claim.RawBytePower.IsZero() {
+			return nil
+		}
+
+		minerState, err := loadMiner(store, st, minerAddr)
+		if err != nil {
+			return err
+		}
+
+		// Shouldn't be necessary if the miner has power, but we might as well be safe.
+		if active, err := minerState.DeadlineCronActive(); err != nil {
+			return err
+		} else if !active {
+			return nil
+		}
+
+		// Record when we need to prove for this miner.
+		dinfo, err := minerState.DeadlineInfo(bb.Height())
+		if err != nil {
+			return err
+		}
+		dinfo = dinfo.NextNotElapsed()
+
+		ppOffset := int(dinfo.PeriodStart % miner.WPoStChallengeWindow)
+		stage.wpostPeriods[ppOffset] = append(stage.wpostPeriods[ppOffset], minerAddr)
+
+		return stage.queueMiner(ctx, bb, minerAddr, minerState, commitEpoch, commitRand)
+	})
+}
+
+func (stage *WindowPoStStage) tick(ctx context.Context, bb *blockbuilder.BlockBuilder) error {
+	// If this is our first time, load from scratch.
+	if stage.wpostPeriods == nil {
+		return stage.load(ctx, bb)
+	}
+
+	targetHeight := bb.Height()
+	now := time.Now()
+	was := len(stage.pendingWposts)
+	count := 0
+	defer func() {
+		bb.L().Debugw("computed window posts",
+			"miners", count,
+			"count", len(stage.pendingWposts)-was,
+			"duration", time.Since(now),
+		)
+	}()
+
+	st := bb.ParentStateTree()
+	store := bb.ActorStore()
+
+	// Perform a bit of catch up. This lets us do things like skip blocks at upgrades then catch
+	// up to make the simulation easier.
+	for ; stage.nextWpostEpoch <= targetHeight; stage.nextWpostEpoch++ {
+		if stage.nextWpostEpoch+miner.WPoStChallengeWindow < targetHeight {
+			bb.L().Warnw("skipping old window post", "deadline-open", stage.nextWpostEpoch)
+			continue
+		}
+		commitEpoch := stage.nextWpostEpoch - 1
+		commitRand, err := postChainCommitInfo(ctx, bb, commitEpoch)
+		if err != nil {
+			return err
+		}
+
+		for _, addr := range stage.wpostPeriods[int(stage.nextWpostEpoch%miner.WPoStChallengeWindow)] {
+			minerState, err := loadMiner(store, st, addr)
+			if err != nil {
+				return err
+			}
+
+			if err := stage.queueMiner(ctx, bb, addr, minerState, commitEpoch, commitRand); err != nil {
+				return err
+			}
+			count++
+		}
+
+	}
+	return nil
+}

cmd/lotus-sim/simulation/step.go

@@ -0,0 +1,71 @@
+package simulation
+
+import (
+	"context"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation/blockbuilder"
+)
+
+// Step steps the simulation forward one step. This may move forward by more than one epoch.
+func (sim *Simulation) Step(ctx context.Context) (*types.TipSet, error) {
+	log.Infow("step", "epoch", sim.head.Height()+1)
+	messages, err := sim.popNextMessages(ctx)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to select messages for block: %w", err)
+	}
+	head, err := sim.makeTipSet(ctx, messages)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to make tipset: %w", err)
+	}
+	if err := sim.SetHead(head); err != nil {
+		return nil, xerrors.Errorf("failed to update head: %w", err)
+	}
+	return head, nil
+}
+
+// popNextMessages generates/picks a set of messages to be included in the next block.
+//
+// - This function is destructive and should only be called once per epoch.
+// - This function does not store anything in the repo.
+// - This function handles all gas estimation. The returned messages should all fit in a single
+//   block.
+func (sim *Simulation) popNextMessages(ctx context.Context) ([]*types.Message, error) {
+	parentTs := sim.head
+
+	// First we make sure we don't have an upgrade at this epoch. If we do, we return no
+	// messages so we can just create an empty block at that epoch.
+	//
+	// This isn't what the network does, but it makes things easier. Otherwise, we'd need to run
+	// migrations before this epoch and I'd rather not deal with that.
+	nextHeight := parentTs.Height() + 1
+	prevVer := sim.StateManager.GetNtwkVersion(ctx, nextHeight-1)
+	nextVer := sim.StateManager.GetNtwkVersion(ctx, nextHeight)
+	if nextVer != prevVer {
+		log.Warnw("packing no messages for version upgrade block",
+			"old", prevVer,
+			"new", nextVer,
+			"epoch", nextHeight,
+		)
+		return nil, nil
+	}
+
+	bb, err := blockbuilder.NewBlockBuilder(
+		ctx, log.With("simulation", sim.name),
+		sim.StateManager, parentTs,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, stage := range sim.stages {
+		// We're intentionally ignoring the "full" signal so we can try to pack a few more
+		// messages.
+		if err := stage.PackMessages(ctx, bb); err != nil && !blockbuilder.IsOutOfGas(err) {
+			return nil, xerrors.Errorf("when packing messages with %s: %w", stage.Name(), err)
+		}
+	}
+	return bb.Messages(), nil
+}

cmd/lotus-sim/upgrade.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/network"
+)
+
+var upgradeCommand = &cli.Command{
+	Name:        "upgrade",
+	Description: "Modifies network upgrade heights.",
+	Subcommands: []*cli.Command{
+		upgradeSetCommand,
+		upgradeList,
+	},
+}
+
+var upgradeList = &cli.Command{
+	Name:        "list",
+	Description: "Lists all pending upgrades.",
+	Subcommands: []*cli.Command{
+		upgradeSetCommand,
+	},
+	Action: func(cctx *cli.Context) (err error) {
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
+		if err != nil {
+			return err
+		}
+		upgrades, err := sim.ListUpgrades()
+		if err != nil {
+			return err
+		}
+
+		tw := tabwriter.NewWriter(cctx.App.Writer, 8, 8, 0, ' ', 0)
+		fmt.Fprintf(tw, "version\theight\tepochs\tmigration\texpensive")
+		epoch := sim.GetHead().Height()
+		for _, upgrade := range upgrades {
+			fmt.Fprintf(
+				tw, "%d\t%d\t%+d\t%t\t%t",
+				upgrade.Network, upgrade.Height, upgrade.Height-epoch,
+				upgrade.Migration != nil,
+				upgrade.Expensive,
+			)
+		}
+		return nil
+	},
+}
+
+var upgradeSetCommand = &cli.Command{
+	Name:        "set",
+	ArgsUsage:   "<network-version> [+]<epochs>",
+	Description: "Set a network upgrade height. Prefix with '+' to set it relative to the last epoch.",
+	Action: func(cctx *cli.Context) (err error) {
+		args := cctx.Args()
+		if args.Len() != 2 {
+			return fmt.Errorf("expected 2 arguments")
+		}
+		nvString := args.Get(0)
+		networkVersion, err := strconv.ParseUint(nvString, 10, 32)
+		if err != nil {
+			return fmt.Errorf("failed to parse network version %q: %w", nvString, err)
+		}
+		heightString := args.Get(1)
+		relative := false
+		if strings.HasPrefix(heightString, "+") {
+			heightString = heightString[1:]
+			relative = true
+		}
+		height, err := strconv.ParseInt(heightString, 10, 64)
+		if err != nil {
+			return fmt.Errorf("failed to parse height version %q: %w", heightString, err)
+		}
+
+		node, err := open(cctx)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if cerr := node.Close(); err == nil {
+				err = cerr
+			}
+		}()
+
+		sim, err := node.LoadSim(cctx.Context, cctx.String("simulation"))
+		if err != nil {
+			return err
+		}
+		if relative {
+			height += int64(sim.GetHead().Height())
+		}
+		return sim.SetUpgradeHeight(network.Version(networkVersion), abi.ChainEpoch(height))
+	},
+}

cmd/lotus-sim/util.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/lotus/cmd/lotus-sim/simulation"
+	"github.com/filecoin-project/lotus/lib/ulimit"
+)
+
+func open(cctx *cli.Context) (*simulation.Node, error) {
+	_, _, err := ulimit.ManageFdLimit()
+	if err != nil {
+		fmt.Fprintf(cctx.App.ErrWriter, "ERROR: failed to raise ulimit: %s\n", err)
+	}
+	return simulation.OpenNode(cctx.Context, cctx.String("repo"))
+}