Merge remote-tracking branch 'origin/master' into nonsense/add-cancel-retrieval-deal-cmd

.golangci.yml

@@ -16,6 +16,12 @@ linters:
     - deadcode
     - scopelint
 
+# We don't want to skip builtin/
+skip-dirs-use-default: false
+skip-dirs:
+  - vendor$
+  - testdata$
+  - examples$
 
 issues:
   exclude:

api/api_storage.go

@@ -5,6 +5,8 @@ import (
 	"context"
 	"time"
 
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
+
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -165,6 +167,8 @@ type StorageMiner interface {
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 
 	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
+
+	ComputeProof(ctx context.Context, ssi []builtin.SectorInfo, rand abi.PoStRandomness) ([]builtin.PoStProof, error) //perm:read
 }
 
 var _ storiface.WorkerReturn = *new(StorageMiner)

api/apistruct/struct.go

@@ -20,6 +20,7 @@ import (
 	"github.com/filecoin-project/go-state-types/dline"
 	"github.com/filecoin-project/lotus/api"
 	apitypes "github.com/filecoin-project/lotus/api/types"
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -534,6 +535,8 @@ type StorageMinerStruct struct {
 
 		CheckProvable func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storage.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) `perm:"admin"`
 
+		ComputeProof func(p0 context.Context, p1 []builtin.SectorInfo, p2 abi.PoStRandomness) ([]builtin.PoStProof, error) `perm:"read"`
+
 		CreateBackup func(p0 context.Context, p1 string) error `perm:"admin"`
 
 		DealsConsiderOfflineRetrievalDeals func(p0 context.Context) (bool, error) `perm:"admin"`
@@ -1712,6 +1715,10 @@ func (s *StorageMinerStruct) CheckProvable(p0 context.Context, p1 abi.Registered
 	return s.Internal.CheckProvable(p0, p1, p2, p3)
 }
 
+func (s *StorageMinerStruct) ComputeProof(p0 context.Context, p1 []builtin.SectorInfo, p2 abi.PoStRandomness) ([]builtin.PoStProof, error) {
+	return s.Internal.ComputeProof(p0, p1, p2)
+}
+
 func (s *StorageMinerStruct) CreateBackup(p0 context.Context, p1 string) error {
 	return s.Internal.CreateBackup(p0, p1)
 }

blockstore/badger/blockstore.go

@@ -85,11 +85,11 @@ const (
 // operation calls after Close() has returned, but it may not happen for
 // operations in progress. Those are likely to fail with a different error.
 type Blockstore struct {
-	DB *badger.DB
+	// state is accessed atomically
-
-	// state is guarded by atomic.
 	state int64
 
+	DB *badger.DB
+
 	prefixing bool
 	prefix    []byte
 	prefixLen int

build/params_shared_vals.go

@@ -118,5 +118,5 @@ const PackingEfficiencyNum = 4
 const PackingEfficiencyDenom = 5
 
 // Actor consts
-// TODO: Pull from actors when its made not private
+// TODO: pieceSize unused from actors
-var MinDealDuration = abi.ChainEpoch(180 * builtin2.EpochsInDay)
+var MinDealDuration, MaxDealDuration = policy.DealDurationBounds(0)

build/params_testground.go

@@ -72,8 +72,8 @@ var (
 	}()
 
 	// Actor consts
-	// TODO: Pull from actors when its made not private
+	// TODO: pieceSize unused from actors
-	MinDealDuration = abi.ChainEpoch(180 * builtin2.EpochsInDay)
+	MinDealDuration, MaxDealDuration = policy.DealDurationBounds(0)
 
 	PackingEfficiencyNum   int64 = 4
 	PackingEfficiencyDenom int64 = 5

build/version.go

@@ -29,7 +29,7 @@ func buildType() string {
 }
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "1.6.0-dev"
+const BuildVersion = "1.7.0-dev"
 
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit

chain/actors/builtin/builtin.go

@@ -52,7 +52,7 @@ type PoStProof = proof0.PoStProof
 type FilterEstimate = smoothing0.FilterEstimate
 
 func FromV0FilterEstimate(v0 smoothing0.FilterEstimate) FilterEstimate {
-	return (FilterEstimate)(v0)
+	return (FilterEstimate)(v0) //nolint:unconvert
 }
 
 // Doesn't change between actors v0, v2, and v3.

chain/actors/builtin/market/market.go

@@ -61,6 +61,7 @@ type State interface {
 	VerifyDealsForActivation(
 		minerAddr address.Address, deals []abi.DealID, currEpoch, sectorExpiry abi.ChainEpoch,
 	) (weight, verifiedWeight abi.DealWeight, err error)
+	NextID() (abi.DealID, error)
 }
 
 type BalanceTable interface {

chain/actors/builtin/market/v0.go

@@ -105,6 +105,10 @@ func (s *state0) VerifyDealsForActivation(
 	return market0.ValidateDealsForActivation(&s.State, s.store, deals, minerAddr, sectorExpiry, currEpoch)
 }
 
+func (s *state0) NextID() (abi.DealID, error) {
+	return s.State.NextID, nil
+}
+
 type balanceTable0 struct {
 	*adt0.BalanceTable
 }

chain/actors/builtin/market/v2.go

@@ -106,6 +106,10 @@ func (s *state2) VerifyDealsForActivation(
 	return w, vw, err
 }
 
+func (s *state2) NextID() (abi.DealID, error) {
+	return s.State.NextID, nil
+}
+
 type balanceTable2 struct {
 	*adt2.BalanceTable
 }

chain/actors/builtin/market/v3.go

@@ -106,6 +106,10 @@ func (s *state3) VerifyDealsForActivation(
 	return w, vw, err
 }
 
+func (s *state3) NextID() (abi.DealID, error) {
+	return s.State.NextID, nil
+}
+
 type balanceTable3 struct {
 	*adt3.BalanceTable
 }

chain/actors/builtin/multisig/state0.go

@@ -68,7 +68,7 @@ func (s *state0) ForEachPendingTxn(cb func(id int64, txn Transaction) error) err
 		if n <= 0 {
 			return xerrors.Errorf("invalid pending transaction key: %v", key)
 		}
-		return cb(txid, (Transaction)(out))
+		return cb(txid, (Transaction)(out)) //nolint:unconvert
 	})
 }
 

chain/actors/builtin/multisig/state2.go

@@ -67,7 +67,7 @@ func (s *state2) ForEachPendingTxn(cb func(id int64, txn Transaction) error) err
 		if n <= 0 {
 			return xerrors.Errorf("invalid pending transaction key: %v", key)
 		}
-		return cb(txid, (Transaction)(out))
+		return cb(txid, (Transaction)(out)) //nolint:unconvert
 	})
 }
 

chain/actors/builtin/multisig/state3.go

@@ -69,7 +69,7 @@ func (s *state3) ForEachPendingTxn(cb func(id int64, txn Transaction) error) err
 		if n <= 0 {
 			return xerrors.Errorf("invalid pending transaction key: %v", key)
 		}
-		return cb(txid, (Transaction)(out))
+		return cb(txid, (Transaction)(out)) //nolint:unconvert
 	})
 }
 

chain/actors/policy/policy.go

@@ -28,6 +28,7 @@ const (
 	ChainFinality                  = miner3.ChainFinality
 	SealRandomnessLookback         = ChainFinality
 	PaychSettleDelay               = paych3.SettleDelay
+	MaxPreCommitRandomnessLookback = builtin3.EpochsInDay + SealRandomnessLookback
 )
 
 // SetSupportedProofTypes sets supported proof types, across all actor versions.
@@ -136,6 +137,10 @@ func DealProviderCollateralBounds(
 	}
 }
 
+func DealDurationBounds(pieceSize abi.PaddedPieceSize) (min, max abi.ChainEpoch) {
+	return market2.DealDurationBounds(pieceSize)
+}
+
 // Sets the challenge window and scales the proving period to match (such that
 // there are always 48 challenge windows in a proving period).
 func SetWPoStChallengeWindow(period abi.ChainEpoch) {

chain/stmgr/call.go

@@ -255,7 +255,7 @@ func (sm *StateManager) Replay(ctx context.Context, ts *types.TipSet, mcid cid.C
 		}
 		return nil
 	})
-	if err != nil && err != errHaltExecution {
+	if err != nil && !xerrors.Is(err, errHaltExecution) {
 		return nil, nil, xerrors.Errorf("unexpected error during execution: %w", err)
 	}
 

cli/auth.go

@@ -13,16 +13,16 @@ import (
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
-var authCmd = &cli.Command{
+var AuthCmd = &cli.Command{
 	Name:  "auth",
 	Usage: "Manage RPC permissions",
 	Subcommands: []*cli.Command{
-		authCreateAdminToken,
+		AuthCreateAdminToken,
-		authApiInfoToken,
+		AuthApiInfoToken,
 	},
 }
 
-var authCreateAdminToken = &cli.Command{
+var AuthCreateAdminToken = &cli.Command{
 	Name:  "create-token",
 	Usage: "Create token",
 	Flags: []cli.Flag{
@@ -70,7 +70,7 @@ var authCreateAdminToken = &cli.Command{
 	},
 }
 
-var authApiInfoToken = &cli.Command{
+var AuthApiInfoToken = &cli.Command{
 	Name:  "api-info",
 	Usage: "Get token with API info required to connect to this node",
 	Flags: []cli.Flag{
@@ -90,7 +90,7 @@ var authApiInfoToken = &cli.Command{
 		ctx := ReqContext(cctx)
 
 		if !cctx.IsSet("perm") {
-			return xerrors.New("--perm flag not set")
+			return xerrors.New("--perm flag not set, use with one of: read, write, sign, admin")
 		}
 
 		perm := cctx.String("perm")

cli/chain.go

@@ -38,31 +38,31 @@ import (
 	types "github.com/filecoin-project/lotus/chain/types"
 )
 
-var chainCmd = &cli.Command{
+var ChainCmd = &cli.Command{
 	Name:  "chain",
 	Usage: "Interact with filecoin blockchain",
 	Subcommands: []*cli.Command{
-		chainHeadCmd,
+		ChainHeadCmd,
-		chainGetBlock,
+		ChainGetBlock,
-		chainReadObjCmd,
+		ChainReadObjCmd,
-		chainDeleteObjCmd,
+		ChainDeleteObjCmd,
-		chainStatObjCmd,
+		ChainStatObjCmd,
-		chainGetMsgCmd,
+		ChainGetMsgCmd,
-		chainSetHeadCmd,
+		ChainSetHeadCmd,
-		chainListCmd,
+		ChainListCmd,
-		chainGetCmd,
+		ChainGetCmd,
-		chainBisectCmd,
+		ChainBisectCmd,
-		chainExportCmd,
+		ChainExportCmd,
-		slashConsensusFault,
+		SlashConsensusFault,
-		chainGasPriceCmd,
+		ChainGasPriceCmd,
-		chainInspectUsage,
+		ChainInspectUsage,
-		chainDecodeCmd,
+		ChainDecodeCmd,
-		chainEncodeCmd,
+		ChainEncodeCmd,
-		chainDisputeSetCmd,
+		ChainDisputeSetCmd,
 	},
 }
 
-var chainHeadCmd = &cli.Command{
+var ChainHeadCmd = &cli.Command{
 	Name:  "head",
 	Usage: "Print chain head",
 	Action: func(cctx *cli.Context) error {
@@ -85,7 +85,7 @@ var chainHeadCmd = &cli.Command{
 	},
 }
 
-var chainGetBlock = &cli.Command{
+var ChainGetBlock = &cli.Command{
 	Name:      "getblock",
 	Usage:     "Get a block and print its details",
 	ArgsUsage: "[blockCid]",
@@ -176,7 +176,7 @@ func apiMsgCids(in []lapi.Message) []cid.Cid {
 	return out
 }
 
-var chainReadObjCmd = &cli.Command{
+var ChainReadObjCmd = &cli.Command{
 	Name:      "read-obj",
 	Usage:     "Read the raw bytes of an object",
 	ArgsUsage: "[objectCid]",
@@ -203,7 +203,7 @@ var chainReadObjCmd = &cli.Command{
 	},
 }
 
-var chainDeleteObjCmd = &cli.Command{
+var ChainDeleteObjCmd = &cli.Command{
 	Name:        "delete-obj",
 	Usage:       "Delete an object from the chain blockstore",
 	Description: "WARNING: Removing wrong objects from the chain blockstore may lead to sync issues",
@@ -240,7 +240,7 @@ var chainDeleteObjCmd = &cli.Command{
 	},
 }
 
-var chainStatObjCmd = &cli.Command{
+var ChainStatObjCmd = &cli.Command{
 	Name:      "stat-obj",
 	Usage:     "Collect size and ipld link counts for objs",
 	ArgsUsage: "[cid]",
@@ -287,7 +287,7 @@ var chainStatObjCmd = &cli.Command{
 	},
 }
 
-var chainGetMsgCmd = &cli.Command{
+var ChainGetMsgCmd = &cli.Command{
 	Name:      "getmessage",
 	Usage:     "Get and print a message by its cid",
 	ArgsUsage: "[messageCid]",
@@ -335,7 +335,7 @@ var chainGetMsgCmd = &cli.Command{
 	},
 }
 
-var chainSetHeadCmd = &cli.Command{
+var ChainSetHeadCmd = &cli.Command{
 	Name:      "sethead",
 	Usage:     "manually set the local nodes head tipset (Caution: normally only used for recovery)",
 	ArgsUsage: "[tipsetkey]",
@@ -384,7 +384,7 @@ var chainSetHeadCmd = &cli.Command{
 	},
 }
 
-var chainInspectUsage = &cli.Command{
+var ChainInspectUsage = &cli.Command{
 	Name:  "inspect-usage",
 	Usage: "Inspect block space usage of a given tipset",
 	Flags: []cli.Flag{
@@ -529,7 +529,7 @@ var chainInspectUsage = &cli.Command{
 	},
 }
 
-var chainListCmd = &cli.Command{
+var ChainListCmd = &cli.Command{
 	Name:    "list",
 	Aliases: []string{"love"},
 	Usage:   "View a segment of the chain",
@@ -658,7 +658,7 @@ var chainListCmd = &cli.Command{
 	},
 }
 
-var chainGetCmd = &cli.Command{
+var ChainGetCmd = &cli.Command{
 	Name:      "get",
 	Usage:     "Get chain DAG node by path",
 	ArgsUsage: "[path]",
@@ -905,7 +905,7 @@ func printTipSet(format string, ts *types.TipSet) {
 	fmt.Println(format)
 }
 
-var chainBisectCmd = &cli.Command{
+var ChainBisectCmd = &cli.Command{
 	Name:      "bisect",
 	Usage:     "bisect chain for an event",
 	ArgsUsage: "[minHeight maxHeight path shellCommand <shellCommandArgs (if any)>]",
@@ -1028,7 +1028,7 @@ var chainBisectCmd = &cli.Command{
 	},
 }
 
-var chainExportCmd = &cli.Command{
+var ChainExportCmd = &cli.Command{
 	Name:      "export",
 	Usage:     "export chain to a car file",
 	ArgsUsage: "[outputPath]",
@@ -1106,7 +1106,7 @@ var chainExportCmd = &cli.Command{
 	},
 }
 
-var slashConsensusFault = &cli.Command{
+var SlashConsensusFault = &cli.Command{
 	Name:      "slash-consensus",
 	Usage:     "Report consensus fault",
 	ArgsUsage: "[blockCid1 blockCid2]",
@@ -1227,7 +1227,7 @@ var slashConsensusFault = &cli.Command{
 	},
 }
 
-var chainGasPriceCmd = &cli.Command{
+var ChainGasPriceCmd = &cli.Command{
 	Name:  "gas-price",
 	Usage: "Estimate gas prices",
 	Action: func(cctx *cli.Context) error {
@@ -1254,7 +1254,7 @@ var chainGasPriceCmd = &cli.Command{
 	},
 }
 
-var chainDecodeCmd = &cli.Command{
+var ChainDecodeCmd = &cli.Command{
 	Name:  "decode",
 	Usage: "decode various types",
 	Subcommands: []*cli.Command{
@@ -1334,7 +1334,7 @@ var chainDecodeParamsCmd = &cli.Command{
 	},
 }
 
-var chainEncodeCmd = &cli.Command{
+var ChainEncodeCmd = &cli.Command{
 	Name:  "encode",
 	Usage: "encode various types",
 	Subcommands: []*cli.Command{

cli/client.go

@@ -388,6 +388,9 @@ var clientDealCmd = &cli.Command{
 		if abi.ChainEpoch(dur) < build.MinDealDuration {
 			return xerrors.Errorf("minimum deal duration is %d blocks", build.MinDealDuration)
 		}
+		if abi.ChainEpoch(dur) > build.MaxDealDuration {
+			return xerrors.Errorf("maximum deal duration is %d blocks", build.MaxDealDuration)
+		}
 
 		var a address.Address
 		if from := cctx.String("from"); from != "" {
@@ -499,9 +502,10 @@ func interactiveDeal(cctx *cli.Context) error {
 	var ds lapi.DataCIDSize
 
 	// find
-	var candidateAsks []*storagemarket.StorageAsk
+	var candidateAsks []QueriedAsk
 	var budget types.FIL
 	var dealCount int64
+	var medianPing, maxAcceptablePing time.Duration
 
 	var a address.Address
 	if from := cctx.String("from"); from != "" {
@@ -661,6 +665,18 @@ uiLoop:
 				return err
 			}
 
+			if len(asks) == 0 {
+				printErr(xerrors.Errorf("no asks found"))
+				continue uiLoop
+			}
+
+			medianPing = asks[len(asks)/2].Ping
+			var avgPing time.Duration
+			for _, ask := range asks {
+				avgPing += ask.Ping
+			}
+			avgPing /= time.Duration(len(asks))
+
 			for _, ask := range asks {
 				if ask.Ask.MinPieceSize > ds.PieceSize {
 					continue
@@ -668,10 +684,48 @@ uiLoop:
 				if ask.Ask.MaxPieceSize < ds.PieceSize {
 					continue
 				}
-				candidateAsks = append(candidateAsks, ask.Ask)
+				candidateAsks = append(candidateAsks, ask)
 			}
 
 			afmt.Printf("Found %d candidate asks\n", len(candidateAsks))
+			afmt.Printf("Average network latency: %s; Median latency: %s\n", avgPing.Truncate(time.Millisecond), medianPing.Truncate(time.Millisecond))
+			state = "max-ping"
+		case "max-ping":
+			maxAcceptablePing = medianPing
+
+			afmt.Printf("Maximum network latency (default: %s) (ms): ", maxAcceptablePing.Truncate(time.Millisecond))
+			_latStr, _, err := rl.ReadLine()
+			latStr := string(_latStr)
+			if err != nil {
+				printErr(xerrors.Errorf("reading maximum latency: %w", err))
+				continue
+			}
+
+			if latStr != "" {
+				maxMs, err := strconv.ParseInt(latStr, 10, 64)
+				if err != nil {
+					printErr(xerrors.Errorf("parsing FIL: %w", err))
+					continue uiLoop
+				}
+
+				maxAcceptablePing = time.Millisecond * time.Duration(maxMs)
+			}
+
+			var goodAsks []QueriedAsk
+			for _, candidateAsk := range candidateAsks {
+				if candidateAsk.Ping < maxAcceptablePing {
+					goodAsks = append(goodAsks, candidateAsk)
+				}
+			}
+
+			if len(goodAsks) == 0 {
+				afmt.Printf("no asks left after filtering for network latency\n")
+				continue uiLoop
+			}
+
+			afmt.Printf("%d asks left after filtering for network latency\n", len(goodAsks))
+			candidateAsks = goodAsks
+
 			state = "find-budget"
 		case "find-budget":
 			afmt.Printf("Proposing from %s, Current Balance: %s\n", a, types.FIL(fromBal))
@@ -690,11 +744,11 @@ uiLoop:
 				continue uiLoop
 			}
 
-			var goodAsks []*storagemarket.StorageAsk
+			var goodAsks []QueriedAsk
 			for _, ask := range candidateAsks {
-				p := ask.Price
+				p := ask.Ask.Price
 				if verified {
-					p = ask.VerifiedPrice
+					p = ask.Ask.VerifiedPrice
 				}
 
 				epochPrice := types.BigDiv(types.BigMul(p, types.NewInt(uint64(ds.PieceSize))), gib)
@@ -734,9 +788,9 @@ uiLoop:
 				pickedAsks = []*storagemarket.StorageAsk{}
 
 				for _, ask := range candidateAsks {
-					p := ask.Price
+					p := ask.Ask.Price
 					if verified {
-						p = ask.VerifiedPrice
+						p = ask.Ask.VerifiedPrice
 					}
 
 					epochPrice := types.BigDiv(types.BigMul(p, types.NewInt(uint64(ds.PieceSize))), gib)
@@ -746,7 +800,7 @@ uiLoop:
 						continue
 					}
 
-					pickedAsks = append(pickedAsks, ask)
+					pickedAsks = append(pickedAsks, ask.Ask)
 					remainingBudget = big.Sub(remainingBudget, totalPrice)
 
 					if len(pickedAsks) == int(dealCount) {

cli/cmd.go

@@ -56,12 +56,12 @@ var GetStorageMinerAPI = cliutil.GetStorageMinerAPI
 var GetWorkerAPI = cliutil.GetWorkerAPI
 
 var CommonCommands = []*cli.Command{
-	netCmd,
+	NetCmd,
-	authCmd,
+	AuthCmd,
-	logCmd,
+	LogCmd,
-	waitApiCmd,
+	WaitApiCmd,
-	fetchParamCmd,
+	FetchParamCmd,
-	pprofCmd,
+	PprofCmd,
 	VersionCmd,
 }
 
@@ -71,16 +71,16 @@ var Commands = []*cli.Command{
 	WithCategory("basic", clientCmd),
 	WithCategory("basic", multisigCmd),
 	WithCategory("basic", paychCmd),
-	WithCategory("developer", authCmd),
+	WithCategory("developer", AuthCmd),
-	WithCategory("developer", mpoolCmd),
+	WithCategory("developer", MpoolCmd),
-	WithCategory("developer", stateCmd),
+	WithCategory("developer", StateCmd),
-	WithCategory("developer", chainCmd),
+	WithCategory("developer", ChainCmd),
-	WithCategory("developer", logCmd),
+	WithCategory("developer", LogCmd),
-	WithCategory("developer", waitApiCmd),
+	WithCategory("developer", WaitApiCmd),
-	WithCategory("developer", fetchParamCmd),
+	WithCategory("developer", FetchParamCmd),
-	WithCategory("network", netCmd),
+	WithCategory("network", NetCmd),
-	WithCategory("network", syncCmd),
+	WithCategory("network", SyncCmd),
-	pprofCmd,
+	PprofCmd,
 	VersionCmd,
 }
 

cli/disputer.go

@@ -35,7 +35,7 @@ type minerDeadline struct {
 	index uint64
 }
 
-var chainDisputeSetCmd = &cli.Command{
+var ChainDisputeSetCmd = &cli.Command{
 	Name:  "disputer",
 	Usage: "interact with the window post disputer",
 	Flags: []cli.Flag{

cli/log.go

@@ -7,16 +7,16 @@ import (
 	"golang.org/x/xerrors"
 )
 
-var logCmd = &cli.Command{
+var LogCmd = &cli.Command{
 	Name:  "log",
 	Usage: "Manage logging",
 	Subcommands: []*cli.Command{
-		logList,
+		LogList,
-		logSetLevel,
+		LogSetLevel,
 	},
 }
 
-var logList = &cli.Command{
+var LogList = &cli.Command{
 	Name:  "list",
 	Usage: "List log systems",
 	Action: func(cctx *cli.Context) error {
@@ -41,7 +41,7 @@ var logList = &cli.Command{
 	},
 }
 
-var logSetLevel = &cli.Command{
+var LogSetLevel = &cli.Command{
 	Name:      "set-level",
 	Usage:     "Set log level",
 	ArgsUsage: "[level]",

cli/mpool.go

@@ -22,22 +22,22 @@ import (
 	"github.com/filecoin-project/lotus/node/config"
 )
 
-var mpoolCmd = &cli.Command{
+var MpoolCmd = &cli.Command{
 	Name:  "mpool",
 	Usage: "Manage message pool",
 	Subcommands: []*cli.Command{
-		mpoolPending,
+		MpoolPending,
-		mpoolClear,
+		MpoolClear,
-		mpoolSub,
+		MpoolSub,
-		mpoolStat,
+		MpoolStat,
-		mpoolReplaceCmd,
+		MpoolReplaceCmd,
-		mpoolFindCmd,
+		MpoolFindCmd,
-		mpoolConfig,
+		MpoolConfig,
-		mpoolGasPerfCmd,
+		MpoolGasPerfCmd,
 	},
 }
 
-var mpoolPending = &cli.Command{
+var MpoolPending = &cli.Command{
 	Name:  "pending",
 	Usage: "Get pending messages",
 	Flags: []cli.Flag{
@@ -132,7 +132,7 @@ var mpoolPending = &cli.Command{
 	},
 }
 
-var mpoolClear = &cli.Command{
+var MpoolClear = &cli.Command{
 	Name:  "clear",
 	Usage: "Clear all pending messages from the mpool (USE WITH CARE)",
 	Flags: []cli.Flag{
@@ -165,7 +165,7 @@ var mpoolClear = &cli.Command{
 	},
 }
 
-var mpoolSub = &cli.Command{
+var MpoolSub = &cli.Command{
 	Name:  "sub",
 	Usage: "Subscribe to mpool changes",
 	Action: func(cctx *cli.Context) error {
@@ -197,7 +197,7 @@ var mpoolSub = &cli.Command{
 	},
 }
 
-var mpoolStat = &cli.Command{
+var MpoolStat = &cli.Command{
 	Name:  "stat",
 	Usage: "print mempool stats",
 	Flags: []cli.Flag{
@@ -356,7 +356,7 @@ var mpoolStat = &cli.Command{
 	},
 }
 
-var mpoolReplaceCmd = &cli.Command{
+var MpoolReplaceCmd = &cli.Command{
 	Name:  "replace",
 	Usage: "replace a message in the mempool",
 	Flags: []cli.Flag{
@@ -509,7 +509,7 @@ var mpoolReplaceCmd = &cli.Command{
 	},
 }
 
-var mpoolFindCmd = &cli.Command{
+var MpoolFindCmd = &cli.Command{
 	Name:  "find",
 	Usage: "find a message in the mempool",
 	Flags: []cli.Flag{
@@ -592,7 +592,7 @@ var mpoolFindCmd = &cli.Command{
 	},
 }
 
-var mpoolConfig = &cli.Command{
+var MpoolConfig = &cli.Command{
 	Name:      "config",
 	Usage:     "get or set current mpool configuration",
 	ArgsUsage: "[new-config]",
@@ -637,7 +637,7 @@ var mpoolConfig = &cli.Command{
 	},
 }
 
-var mpoolGasPerfCmd = &cli.Command{
+var MpoolGasPerfCmd = &cli.Command{
 	Name:  "gas-perf",
 	Usage: "Check gas performance of messages in mempool",
 	Flags: []cli.Flag{

cli/net.go

@@ -23,16 +23,16 @@ import (
 	"github.com/filecoin-project/lotus/lib/addrutil"
 )
 
-var netCmd = &cli.Command{
+var NetCmd = &cli.Command{
 	Name:  "net",
 	Usage: "Manage P2P Network",
 	Subcommands: []*cli.Command{
 		NetPeers,
-		netConnect,
+		NetConnect,
 		NetListen,
 		NetId,
-		netFindPeer,
+		NetFindPeer,
-		netScores,
+		NetScores,
 		NetReachability,
 		NetBandwidthCmd,
 		NetBlockCmd,
@@ -112,7 +112,7 @@ var NetPeers = &cli.Command{
 	},
 }
 
-var netScores = &cli.Command{
+var NetScores = &cli.Command{
 	Name:  "scores",
 	Usage: "Print peers' pubsub scores",
 	Flags: []cli.Flag{
@@ -175,7 +175,7 @@ var NetListen = &cli.Command{
 	},
 }
 
-var netConnect = &cli.Command{
+var NetConnect = &cli.Command{
 	Name:      "connect",
 	Usage:     "Connect to a peer",
 	ArgsUsage: "[peerMultiaddr|minerActorAddress]",
@@ -264,7 +264,7 @@ var NetId = &cli.Command{
 	},
 }
 
-var netFindPeer = &cli.Command{
+var NetFindPeer = &cli.Command{
 	Name:      "findpeer",
 	Usage:     "Find the addresses of a given peerID",
 	ArgsUsage: "[peerId]",

cli/params.go

@@ -9,7 +9,7 @@ import (
 	"github.com/filecoin-project/lotus/build"
 )
 
-var fetchParamCmd = &cli.Command{
+var FetchParamCmd = &cli.Command{
 	Name:      "fetch-params",
 	Usage:     "Fetch proving parameters",
 	ArgsUsage: "[sectorSize]",

cli/pprof.go

@@ -11,7 +11,7 @@ import (
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
-var pprofCmd = &cli.Command{
+var PprofCmd = &cli.Command{
 	Name:   "pprof",
 	Hidden: true,
 	Subcommands: []*cli.Command{

cli/state.go

@@ -41,7 +41,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
-var stateCmd = &cli.Command{
+var StateCmd = &cli.Command{
 	Name:  "state",
 	Usage: "Interact with and query filecoin chain state",
 	Flags: []cli.Flag{
@@ -51,32 +51,32 @@ var stateCmd = &cli.Command{
 		},
 	},
 	Subcommands: []*cli.Command{
-		statePowerCmd,
+		StatePowerCmd,
-		stateSectorsCmd,
+		StateSectorsCmd,
-		stateActiveSectorsCmd,
+		StateActiveSectorsCmd,
-		stateListActorsCmd,
+		StateListActorsCmd,
-		stateListMinersCmd,
+		StateListMinersCmd,
-		stateCircSupplyCmd,
+		StateCircSupplyCmd,
-		stateSectorCmd,
+		StateSectorCmd,
-		stateGetActorCmd,
+		StateGetActorCmd,
-		stateLookupIDCmd,
+		StateLookupIDCmd,
-		stateReplayCmd,
+		StateReplayCmd,
-		stateSectorSizeCmd,
+		StateSectorSizeCmd,
-		stateReadStateCmd,
+		StateReadStateCmd,
-		stateListMessagesCmd,
+		StateListMessagesCmd,
-		stateComputeStateCmd,
+		StateComputeStateCmd,
-		stateCallCmd,
+		StateCallCmd,
-		stateGetDealSetCmd,
+		StateGetDealSetCmd,
-		stateWaitMsgCmd,
+		StateWaitMsgCmd,
-		stateSearchMsgCmd,
+		StateSearchMsgCmd,
-		stateMinerInfo,
+		StateMinerInfo,
-		stateMarketCmd,
+		StateMarketCmd,
-		stateExecTraceCmd,
+		StateExecTraceCmd,
-		stateNtwkVersionCmd,
+		StateNtwkVersionCmd,
 	},
 }
 
-var stateMinerInfo = &cli.Command{
+var StateMinerInfo = &cli.Command{
 	Name:      "miner-info",
 	Usage:     "Retrieve miner information",
 	ArgsUsage: "[minerAddress]",
@@ -219,7 +219,7 @@ func ParseTipSetRef(ctx context.Context, api api.FullNode, tss string) (*types.T
 	return ts, nil
 }
 
-var statePowerCmd = &cli.Command{
+var StatePowerCmd = &cli.Command{
 	Name:      "power",
 	Usage:     "Query network or miner power",
 	ArgsUsage: "[<minerAddress> (optional)]",
@@ -263,7 +263,7 @@ var statePowerCmd = &cli.Command{
 	},
 }
 
-var stateSectorsCmd = &cli.Command{
+var StateSectorsCmd = &cli.Command{
 	Name:      "sectors",
 	Usage:     "Query the sector set of a miner",
 	ArgsUsage: "[minerAddress]",
@@ -303,7 +303,7 @@ var stateSectorsCmd = &cli.Command{
 	},
 }
 
-var stateActiveSectorsCmd = &cli.Command{
+var StateActiveSectorsCmd = &cli.Command{
 	Name:      "active-sectors",
 	Usage:     "Query the active sector set of a miner",
 	ArgsUsage: "[minerAddress]",
@@ -343,7 +343,7 @@ var stateActiveSectorsCmd = &cli.Command{
 	},
 }
 
-var stateExecTraceCmd = &cli.Command{
+var StateExecTraceCmd = &cli.Command{
 	Name:      "exec-trace",
 	Usage:     "Get the execution trace of a given message",
 	ArgsUsage: "<messageCid>",
@@ -411,7 +411,7 @@ var stateExecTraceCmd = &cli.Command{
 	},
 }
 
-var stateReplayCmd = &cli.Command{
+var StateReplayCmd = &cli.Command{
 	Name:      "replay",
 	Usage:     "Replay a particular message",
 	ArgsUsage: "<messageCid>",
@@ -476,7 +476,7 @@ var stateReplayCmd = &cli.Command{
 	},
 }
 
-var stateGetDealSetCmd = &cli.Command{
+var StateGetDealSetCmd = &cli.Command{
 	Name:      "get-deal",
 	Usage:     "View on-chain deal info",
 	ArgsUsage: "[dealId]",
@@ -518,7 +518,7 @@ var stateGetDealSetCmd = &cli.Command{
 	},
 }
 
-var stateListMinersCmd = &cli.Command{
+var StateListMinersCmd = &cli.Command{
 	Name:  "list-miners",
 	Usage: "list all miners in the network",
 	Flags: []cli.Flag{
@@ -590,7 +590,7 @@ func getDealsCounts(ctx context.Context, lapi api.FullNode) (map[address.Address
 	return out, nil
 }
 
-var stateListActorsCmd = &cli.Command{
+var StateListActorsCmd = &cli.Command{
 	Name:  "list-actors",
 	Usage: "list all actors in the network",
 	Action: func(cctx *cli.Context) error {
@@ -620,7 +620,7 @@ var stateListActorsCmd = &cli.Command{
 	},
 }
 
-var stateGetActorCmd = &cli.Command{
+var StateGetActorCmd = &cli.Command{
 	Name:      "get-actor",
 	Usage:     "Print actor information",
 	ArgsUsage: "[actorrAddress]",
@@ -664,7 +664,7 @@ var stateGetActorCmd = &cli.Command{
 	},
 }
 
-var stateLookupIDCmd = &cli.Command{
+var StateLookupIDCmd = &cli.Command{
 	Name:      "lookup",
 	Usage:     "Find corresponding ID address",
 	ArgsUsage: "[address]",
@@ -715,7 +715,7 @@ var stateLookupIDCmd = &cli.Command{
 	},
 }
 
-var stateSectorSizeCmd = &cli.Command{
+var StateSectorSizeCmd = &cli.Command{
 	Name:      "sector-size",
 	Usage:     "Look up miners sector size",
 	ArgsUsage: "[minerAddress]",
@@ -752,7 +752,7 @@ var stateSectorSizeCmd = &cli.Command{
 	},
 }
 
-var stateReadStateCmd = &cli.Command{
+var StateReadStateCmd = &cli.Command{
 	Name:      "read-state",
 	Usage:     "View a json representation of an actors state",
 	ArgsUsage: "[actorAddress]",
@@ -794,7 +794,7 @@ var stateReadStateCmd = &cli.Command{
 	},
 }
 
-var stateListMessagesCmd = &cli.Command{
+var StateListMessagesCmd = &cli.Command{
 	Name:  "list-messages",
 	Usage: "list messages on chain matching given criteria",
 	Flags: []cli.Flag{
@@ -907,7 +907,7 @@ var stateListMessagesCmd = &cli.Command{
 	},
 }
 
-var stateComputeStateCmd = &cli.Command{
+var StateComputeStateCmd = &cli.Command{
 	Name:  "compute-state",
 	Usage: "Perform state computations",
 	Flags: []cli.Flag{
@@ -1365,7 +1365,7 @@ func jsonReturn(code cid.Cid, method abi.MethodNum, ret []byte) (string, error)
 	return string(b), err
 }
 
-var stateWaitMsgCmd = &cli.Command{
+var StateWaitMsgCmd = &cli.Command{
 	Name:      "wait-msg",
 	Usage:     "Wait for a message to appear on chain",
 	ArgsUsage: "[messageCid]",
@@ -1407,7 +1407,7 @@ var stateWaitMsgCmd = &cli.Command{
 	},
 }
 
-var stateSearchMsgCmd = &cli.Command{
+var StateSearchMsgCmd = &cli.Command{
 	Name:      "search-msg",
 	Usage:     "Search to see whether a message has appeared on chain",
 	ArgsUsage: "[messageCid]",
@@ -1484,7 +1484,7 @@ func printMsg(ctx context.Context, api api.FullNode, msg cid.Cid, mw *lapi.MsgLo
 	return nil
 }
 
-var stateCallCmd = &cli.Command{
+var StateCallCmd = &cli.Command{
 	Name:      "call",
 	Usage:     "Invoke a method on an actor locally",
 	ArgsUsage: "[toAddress methodId <param1 param2 ...> (optional)]",
@@ -1692,7 +1692,7 @@ func parseParamsForMethod(act cid.Cid, method uint64, args []string) ([]byte, er
 	return buf.Bytes(), nil
 }
 
-var stateCircSupplyCmd = &cli.Command{
+var StateCircSupplyCmd = &cli.Command{
 	Name:  "circulating-supply",
 	Usage: "Get the exact current circulating supply of Filecoin",
 	Flags: []cli.Flag{
@@ -1741,7 +1741,7 @@ var stateCircSupplyCmd = &cli.Command{
 	},
 }
 
-var stateSectorCmd = &cli.Command{
+var StateSectorCmd = &cli.Command{
 	Name:      "sector",
 	Usage:     "Get miner sector info",
 	ArgsUsage: "[miner address] [sector number]",
@@ -1815,7 +1815,7 @@ var stateSectorCmd = &cli.Command{
 	},
 }
 
-var stateMarketCmd = &cli.Command{
+var StateMarketCmd = &cli.Command{
 	Name:  "market",
 	Usage: "Inspect the storage market actor",
 	Subcommands: []*cli.Command{
@@ -1861,7 +1861,7 @@ var stateMarketBalanceCmd = &cli.Command{
 	},
 }
 
-var stateNtwkVersionCmd = &cli.Command{
+var StateNtwkVersionCmd = &cli.Command{
 	Name:  "network-version",
 	Usage: "Returns the network version",
 	Action: func(cctx *cli.Context) error {

cli/sync.go

@@ -15,20 +15,20 @@ import (
 	"github.com/filecoin-project/lotus/build"
 )
 
-var syncCmd = &cli.Command{
+var SyncCmd = &cli.Command{
 	Name:  "sync",
 	Usage: "Inspect or interact with the chain syncer",
 	Subcommands: []*cli.Command{
-		syncStatusCmd,
+		SyncStatusCmd,
-		syncWaitCmd,
+		SyncWaitCmd,
-		syncMarkBadCmd,
+		SyncMarkBadCmd,
-		syncUnmarkBadCmd,
+		SyncUnmarkBadCmd,
-		syncCheckBadCmd,
+		SyncCheckBadCmd,
-		syncCheckpointCmd,
+		SyncCheckpointCmd,
 	},
 }
 
-var syncStatusCmd = &cli.Command{
+var SyncStatusCmd = &cli.Command{
 	Name:  "status",
 	Usage: "check sync status",
 	Action: func(cctx *cli.Context) error {
@@ -81,7 +81,7 @@ var syncStatusCmd = &cli.Command{
 	},
 }
 
-var syncWaitCmd = &cli.Command{
+var SyncWaitCmd = &cli.Command{
 	Name:  "wait",
 	Usage: "Wait for sync to be complete",
 	Flags: []cli.Flag{
@@ -102,7 +102,7 @@ var syncWaitCmd = &cli.Command{
 	},
 }
 
-var syncMarkBadCmd = &cli.Command{
+var SyncMarkBadCmd = &cli.Command{
 	Name:      "mark-bad",
 	Usage:     "Mark the given block as bad, will prevent syncing to a chain that contains it",
 	ArgsUsage: "[blockCid]",
@@ -127,7 +127,7 @@ var syncMarkBadCmd = &cli.Command{
 	},
 }
 
-var syncUnmarkBadCmd = &cli.Command{
+var SyncUnmarkBadCmd = &cli.Command{
 	Name:  "unmark-bad",
 	Usage: "Unmark the given block as bad, makes it possible to sync to a chain containing it",
 	Flags: []cli.Flag{
@@ -162,7 +162,7 @@ var syncUnmarkBadCmd = &cli.Command{
 	},
 }
 
-var syncCheckBadCmd = &cli.Command{
+var SyncCheckBadCmd = &cli.Command{
 	Name:      "check-bad",
 	Usage:     "check if the given block was marked bad, and for what reason",
 	ArgsUsage: "[blockCid]",
@@ -198,7 +198,7 @@ var syncCheckBadCmd = &cli.Command{
 	},
 }
 
-var syncCheckpointCmd = &cli.Command{
+var SyncCheckpointCmd = &cli.Command{
 	Name:      "checkpoint",
 	Usage:     "mark a certain tipset as checkpointed; the node will never fork away from this tipset",
 	ArgsUsage: "[tipsetKey]",

cli/wait.go

@@ -7,7 +7,7 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-var waitApiCmd = &cli.Command{
+var WaitApiCmd = &cli.Command{
 	Name:  "wait-api",
 	Usage: "Wait for lotus api to come online",
 	Action: func(cctx *cli.Context) error {

cmd/lotus-shed/base64.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/go-address"
+
+	"github.com/urfave/cli/v2"
+)
+
+var base64Cmd = &cli.Command{
+	Name:        "base64",
+	Description: "multiformats base64",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "decodeAddr",
+			Value: false,
+			Usage: "Decode a base64 addr",
+		},
+		&cli.BoolFlag{
+			Name:  "decodeBig",
+			Value: false,
+			Usage: "Decode a base64 big",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		var input io.Reader
+
+		if cctx.Args().Len() == 0 {
+			input = os.Stdin
+		} else {
+			input = strings.NewReader(cctx.Args().First())
+		}
+
+		bytes, err := ioutil.ReadAll(input)
+		if err != nil {
+			return nil
+		}
+
+		decoded, err := base64.RawStdEncoding.DecodeString(strings.TrimSpace(string(bytes)))
+		if err != nil {
+			return err
+		}
+
+		if cctx.Bool("decodeAddr") {
+			addr, err := address.NewFromBytes(decoded)
+			if err != nil {
+				return err
+			}
+
+			fmt.Println(addr)
+
+			return nil
+		}
+
+		if cctx.Bool("decodeBig") {
+			var val abi.TokenAmount
+			err = val.UnmarshalBinary(decoded)
+			if err != nil {
+				return err
+			}
+
+			fmt.Println(val)
+		}
+
+		return nil
+	},
+}

cmd/lotus-shed/datastore.go

@@ -180,8 +180,11 @@ var datastoreBackupStatCmd = &cli.Command{
 		}
 		defer f.Close() // nolint:errcheck
 
-		var keys, kbytes, vbytes uint64
+		var keys, logs, kbytes, vbytes uint64
-		err = backupds.ReadBackup(f, func(key datastore.Key, value []byte) error {
+		err = backupds.ReadBackup(f, func(key datastore.Key, value []byte, log bool) error {
+			if log {
+				logs++
+			}
 			keys++
 			kbytes += uint64(len(key.String()))
 			vbytes += uint64(len(value))
@@ -192,6 +195,7 @@ var datastoreBackupStatCmd = &cli.Command{
 		}
 
 		fmt.Println("Keys:        ", keys)
+		fmt.Println("Log values:  ", log)
 		fmt.Println("Key bytes:   ", units.BytesSize(float64(kbytes)))
 		fmt.Println("Value bytes: ", units.BytesSize(float64(vbytes)))
 
@@ -225,7 +229,7 @@ var datastoreBackupListCmd = &cli.Command{
 		defer f.Close() // nolint:errcheck
 
 		printKv := kvPrinter(cctx.Bool("top-level"), cctx.String("get-enc"))
-		err = backupds.ReadBackup(f, func(key datastore.Key, value []byte) error {
+		err = backupds.ReadBackup(f, func(key datastore.Key, value []byte, _ bool) error {
 			return printKv(key.String(), value)
 		})
 		if err != nil {

cmd/lotus-shed/main.go

@@ -16,6 +16,7 @@ func main() {
 	logging.SetLogLevel("*", "INFO")
 
 	local := []*cli.Command{
+		base64Cmd,
 		base32Cmd,
 		base16Cmd,
 		bitFieldCmd,
@@ -52,6 +53,7 @@ func main() {
 		rpcCmd,
 		cidCmd,
 		blockmsgidCmd,
+		signaturesCmd,
 	}
 
 	app := &cli.App{

cmd/lotus-shed/sectors.go

@@ -6,6 +6,7 @@ import (
 
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
@@ -34,6 +35,10 @@ var terminateSectorCmd = &cli.Command{
 	Usage:     "Forcefully terminate a sector (WARNING: This means losing power and pay a one-time termination penalty(including collateral) for the terminated sector)",
 	ArgsUsage: "[sectorNum1 sectorNum2 ...]",
 	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "actor",
+			Usage: "specify the address of miner actor",
+		},
 		&cli.BoolFlag{
 			Name:  "really-do-it",
 			Usage: "pass this flag if you know what you are doing",
@@ -44,6 +49,15 @@ var terminateSectorCmd = &cli.Command{
 			return fmt.Errorf("at least one sector must be specified")
 		}
 
+		var maddr address.Address
+		if act := cctx.String("actor"); act != "" {
+			var err error
+			maddr, err = address.NewFromString(act)
+			if err != nil {
+				return fmt.Errorf("parsing address %s: %w", act, err)
+			}
+		}
+
 		if !cctx.Bool("really-do-it") {
 			return fmt.Errorf("this is a command for advanced users, only use it if you are sure of what you are doing")
 		}
@@ -54,18 +68,20 @@ var terminateSectorCmd = &cli.Command{
 		}
 		defer closer()
 
+		ctx := lcli.ReqContext(cctx)
+
+		if maddr.Empty() {
 			api, acloser, err := lcli.GetStorageMinerAPI(cctx)
 			if err != nil {
 				return err
 			}
 			defer acloser()
 
-		ctx := lcli.ReqContext(cctx)
+			maddr, err = api.ActorAddress(ctx)
-
-		maddr, err := api.ActorAddress(ctx)
 			if err != nil {
 				return err
 			}
+		}
 
 		mi, err := nodeApi.StateMinerInfo(ctx, maddr, types.EmptyTSK)
 		if err != nil {
@@ -147,29 +163,46 @@ var terminateSectorPenaltyEstimationCmd = &cli.Command{
 	Name:      "termination-estimate",
 	Usage:     "Estimate the termination penalty",
 	ArgsUsage: "[sectorNum1 sectorNum2 ...]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "actor",
+			Usage: "specify the address of miner actor",
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.Args().Len() < 1 {
 			return fmt.Errorf("at least one sector must be specified")
 		}
 
+		var maddr address.Address
+		if act := cctx.String("actor"); act != "" {
+			var err error
+			maddr, err = address.NewFromString(act)
+			if err != nil {
+				return fmt.Errorf("parsing address %s: %w", act, err)
+			}
+		}
+
 		nodeApi, closer, err := lcli.GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
 		}
 		defer closer()
 
+		ctx := lcli.ReqContext(cctx)
+
+		if maddr.Empty() {
 			api, acloser, err := lcli.GetStorageMinerAPI(cctx)
 			if err != nil {
 				return err
 			}
 			defer acloser()
 
-		ctx := lcli.ReqContext(cctx)
+			maddr, err = api.ActorAddress(ctx)
-
-		maddr, err := api.ActorAddress(ctx)
 			if err != nil {
 				return err
 			}
+		}
 
 		mi, err := nodeApi.StateMinerInfo(ctx, maddr, types.EmptyTSK)
 		if err != nil {

cmd/lotus-shed/signatures.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"encoding/hex"
+	"fmt"
+	"strconv"
+
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/lotus/lib/sigs"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/xerrors"
+)
+
+var signaturesCmd = &cli.Command{
+	Name:  "signatures",
+	Usage: "tools involving signatures",
+	Subcommands: []*cli.Command{
+		sigsVerifyVoteCmd,
+	},
+}
+
+var sigsVerifyVoteCmd = &cli.Command{
+	Name:        "verify-vote",
+	Description: "can be used to verify signed votes being submitted for FILPolls",
+	Usage:       "<FIPnumber> <signingAddress> <signature>",
+	Action: func(cctx *cli.Context) error {
+
+		if cctx.Args().Len() != 3 {
+			return xerrors.Errorf("usage: verify-vote <FIPnumber> <signingAddress> <signature>")
+		}
+
+		fip, err := strconv.ParseInt(cctx.Args().First(), 10, 64)
+		if err != nil {
+			return xerrors.Errorf("couldn't parse FIP number: %w", err)
+		}
+
+		addr, err := address.NewFromString(cctx.Args().Get(1))
+		if err != nil {
+			return xerrors.Errorf("couldn't parse signing address: %w", err)
+		}
+
+		sigBytes, err := hex.DecodeString(cctx.Args().Get(2))
+		if err != nil {
+			return xerrors.Errorf("couldn't parse sig: %w", err)
+		}
+
+		var sig crypto.Signature
+		if err := sig.UnmarshalBinary(sigBytes); err != nil {
+			return xerrors.Errorf("couldn't unmarshal sig: %w", err)
+		}
+
+		switch fip {
+		case 14:
+			approve := []byte("7 - Approve")
+
+			if sigs.Verify(&sig, addr, approve) == nil {
+				fmt.Println("valid vote for approving FIP-0014")
+				return nil
+			}
+
+			reject := []byte("7 - Reject")
+			if sigs.Verify(&sig, addr, reject) == nil {
+				fmt.Println("valid vote for rejecting FIP-0014")
+				return nil
+			}
+
+			return xerrors.Errorf("invalid vote for FIP-0014!")
+		default:
+			return xerrors.Errorf("unrecognized FIP number")
+		}
+	},
+}

documentation/en/api-methods-miner.md

@@ -14,6 +14,8 @@
   * [AuthVerify](#AuthVerify)
 * [Check](#Check)
   * [CheckProvable](#CheckProvable)
+* [Compute](#Compute)
+  * [ComputeProof](#ComputeProof)
 * [Create](#Create)
   * [CreateBackup](#CreateBackup)
 * [Deals](#Deals)
@@ -295,6 +297,24 @@ Response:
 }
 ```
 
+## Compute
+
+
+### ComputeProof
+
+
+Perms: read
+
+Inputs:
+```json
+[
+  null,
+  null
+]
+```
+
+Response: `null`
+
 ## Create
 
 

extern/storage-sealing/checks.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 
-	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/policy"
 
 	proof2 "github.com/filecoin-project/specs-actors/v2/actors/runtime/proof"
@@ -94,14 +93,9 @@ func checkPrecommit(ctx context.Context, maddr address.Address, si SectorInfo, t
 		return &ErrBadCommD{xerrors.Errorf("on chain CommD differs from sector: %s != %s", commD, si.CommD)}
 	}
 
-	nv, err := api.StateNetworkVersion(ctx, tok)
+	ticketEarliest := height - policy.MaxPreCommitRandomnessLookback
-	if err != nil {
-		return &ErrApi{xerrors.Errorf("calling StateNetworkVersion: %w", err)}
-	}
 
-	msd := policy.GetMaxProveCommitDuration(actors.VersionForNetwork(nv), si.SectorType)
+	if si.TicketEpoch < ticketEarliest {
-
-	if height-(si.TicketEpoch+policy.SealRandomnessLookback) > msd {
 		return &ErrExpiredTicket{xerrors.Errorf("ticket expired: seal height: %d, head: %d", si.TicketEpoch+policy.SealRandomnessLookback, height)}
 	}
 

extern/storage-sealing/states_sealing.go

@@ -12,7 +12,6 @@ import (
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/filecoin-project/go-state-types/exitcode"
 	"github.com/filecoin-project/go-statemachine"
-	builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-storage/storage"
 
 	"github.com/filecoin-project/lotus/api"
@@ -22,7 +21,7 @@ import (
 )
 
 var DealSectorPriority = 1024
-var MaxTicketAge = abi.ChainEpoch(builtin0.EpochsInDay * 2)
+var MaxTicketAge = policy.MaxPreCommitRandomnessLookback
 
 func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) error {
 	m.inputLk.Lock()

go.mod

@@ -30,18 +30,18 @@ require (
 	github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 // indirect
 	github.com/filecoin-project/go-bitfield v0.2.4
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
-	github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434
+	github.com/filecoin-project/go-commp-utils v0.1.0
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
-	github.com/filecoin-project/go-data-transfer v1.2.7
+	github.com/filecoin-project/go-data-transfer v1.4.0
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a
-	github.com/filecoin-project/go-fil-markets v1.1.9
+	github.com/filecoin-project/go-fil-markets v1.2.3
 	github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
 	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20
 	github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261
 	github.com/filecoin-project/go-state-types v0.1.0
 	github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe
-	github.com/filecoin-project/go-statestore v0.1.1-0.20210311122610-6c7a5aedbdea
+	github.com/filecoin-project/go-statestore v0.1.1
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
 	github.com/filecoin-project/specs-actors v0.9.13
 	github.com/filecoin-project/specs-actors/v2 v2.3.4

go.sum

@@ -260,21 +260,22 @@ github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW
 github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
-github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434 h1:0kHszkYP3hgApcjl5x4rpwONhN9+j7XDobf6at5XfHs=
 github.com/filecoin-project/go-commp-utils v0.0.0-20201119054358-b88f7a96a434/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U=
+github.com/filecoin-project/go-commp-utils v0.1.0 h1:PaDxoXYh1TXnnz5kA/xSObpAQwcJSUs4Szb72nuaNdk=
+github.com/filecoin-project/go-commp-utils v0.1.0/go.mod h1:6s95K91mCyHY51RPWECZieD3SGWTqIFLf1mPOes9l5U=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-data-transfer v1.0.1/go.mod h1:UxvfUAY9v3ub0a21BSK9u3pB2aq30Y0KMsG+w9/ysyo=
-github.com/filecoin-project/go-data-transfer v1.2.7 h1:WE5Cpp9eMt5BDoWOVR64QegSn6bwHQaDzyyjVU377Y0=
+github.com/filecoin-project/go-data-transfer v1.4.0 h1:SRpFUp7WQdJe6iSmt7HfhMGDk7tniTVIlfmvQVBZhN8=
-github.com/filecoin-project/go-data-transfer v1.2.7/go.mod h1:mvjZ+C3NkBX10JP4JMu27DCjUouHFjHwUGh+Xc4yvDA=
+github.com/filecoin-project/go-data-transfer v1.4.0/go.mod h1:n8kbDQXWrY1c4UgfMa9KERxNCWbOTDwdNhf2MpN9dpo=
 github.com/filecoin-project/go-ds-versioning v0.1.0 h1:y/X6UksYTsK8TLCI7rttCKEvl8btmWxyFMEeeWGUxIQ=
 github.com/filecoin-project/go-ds-versioning v0.1.0/go.mod h1:mp16rb4i2QPmxBnmanUx8i/XANp+PFCCJWiAb+VW4/s=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a h1:hyJ+pUm/4U4RdEZBlg6k8Ma4rDiuvqyGpoICXAxwsTg=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
 github.com/filecoin-project/go-fil-markets v1.0.5-0.20201113164554-c5eba40d5335/go.mod h1:AJySOJC00JRWEZzRG2KsfUnqEf5ITXxeX09BE9N4f9c=
-github.com/filecoin-project/go-fil-markets v1.1.9 h1:sA0NIEOpy7brZaeXeNgdXg5pvHaBtD5OTRlraOUbI0w=
+github.com/filecoin-project/go-fil-markets v1.2.3 h1:JDbGKQf60tB00TFKG/nofSyHyIEbqowZqoLOfda7yTs=
-github.com/filecoin-project/go-fil-markets v1.1.9/go.mod h1:0yQu5gvrjFoAIyzPSSJ+xUdCG83vjInAFbTswIB5/hk=
+github.com/filecoin-project/go-fil-markets v1.2.3/go.mod h1:p5BIKl6sEoeOCKFa3/nfy66Q95rifEkJyGQgaNjPsno=
 github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM=
 github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24=
 github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM=
@@ -298,8 +299,8 @@ github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psS
 github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe h1:dF8u+LEWeIcTcfUcCf3WFVlc81Fr2JKg8zPzIbBDKDw=
 github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
 github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
-github.com/filecoin-project/go-statestore v0.1.1-0.20210311122610-6c7a5aedbdea h1:EvmiCrHRrPF2mxVMIRdtsxZGByqwT24aJLw7mdzEFxE=
+github.com/filecoin-project/go-statestore v0.1.1 h1:ufMFq00VqnT2CAuDpcGnwLnCX1I/c3OROw/kXVNSTZk=
-github.com/filecoin-project/go-statestore v0.1.1-0.20210311122610-6c7a5aedbdea/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
+github.com/filecoin-project/go-statestore v0.1.1/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
 github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4=
@@ -595,7 +596,6 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28
 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE=
 github.com/ipfs/go-graphsync v0.4.2/go.mod h1:/VmbZTUdUMTbNkgzAiCEucIIAU3BkLE2cZrDCVUhyi0=
 github.com/ipfs/go-graphsync v0.4.3/go.mod h1:mPOwDYv128gf8gxPFgXnz4fNrSYPsWyqisJ7ych+XDY=
-github.com/ipfs/go-graphsync v0.5.2/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk=
 github.com/ipfs/go-graphsync v0.6.0 h1:x6UvDUGA7wjaKNqx5Vbo7FGT8aJ5ryYA0dMQ5jN3dF0=
 github.com/ipfs/go-graphsync v0.6.0/go.mod h1:e2ZxnClqBBYAtd901g9vXMJzS47labjAtOzsWtOzKNk=
 github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk=

lib/backupds/log.go

@@ -55,7 +55,7 @@ func (d *Datastore) startLog(logdir string) error {
 			return xerrors.Errorf("creating log: %w", err)
 		}
 	} else {
-		l, err = d.openLog(filepath.Join(logdir, latest))
+		l, latest, err = d.openLog(filepath.Join(logdir, latest))
 		if err != nil {
 			return xerrors.Errorf("opening log: %w", err)
 		}
@@ -97,6 +97,8 @@ type logfile struct {
 	file *os.File
 }
 
+var compactThresh = 2
+
 func (d *Datastore) createLog(logdir string) (*logfile, string, error) {
 	p := filepath.Join(logdir, strconv.FormatInt(time.Now().Unix(), 10)+".log.cbor")
 	log.Infow("creating log", "file", p)
@@ -119,32 +121,36 @@ func (d *Datastore) createLog(logdir string) (*logfile, string, error) {
 	}, filepath.Base(p), nil
 }
 
-func (d *Datastore) openLog(p string) (*logfile, error) {
+func (d *Datastore) openLog(p string) (*logfile, string, error) {
 	log.Infow("opening log", "file", p)
 	lh, err := d.child.Get(loghead)
 	if err != nil {
-		return nil, xerrors.Errorf("checking log head (logfile '%s'): %w", p, err)
+		return nil, "", xerrors.Errorf("checking log head (logfile '%s'): %w", p, err)
 	}
 
 	lhp := strings.Split(string(lh), ";")
 	if len(lhp) != 3 {
-		return nil, xerrors.Errorf("expected loghead to have 3 parts")
+		return nil, "", xerrors.Errorf("expected loghead to have 3 parts")
 	}
 
 	if lhp[0] != filepath.Base(p) {
-		return nil, xerrors.Errorf("loghead log file doesn't match, opening %s, expected %s", p, lhp[0])
+		return nil, "", xerrors.Errorf("loghead log file doesn't match, opening %s, expected %s", p, lhp[0])
 	}
 
 	f, err := os.OpenFile(p, os.O_RDWR, 0644)
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
 
 	var lastLogHead string
-	var openCount, logvals int64
+	var openCount, vals, logvals int64
 	// check file integrity
-	err = ReadBackup(f, func(k datastore.Key, v []byte) error {
+	err = ReadBackup(f, func(k datastore.Key, v []byte, log bool) error {
+		if log {
 			logvals++
+		} else {
+			vals++
+		}
 		if k == loghead {
 			lastLogHead = string(v)
 			openCount++
@@ -152,32 +158,53 @@ func (d *Datastore) openLog(p string) (*logfile, error) {
 		return nil
 	})
 	if err != nil {
-		return nil, xerrors.Errorf("reading backup part of the logfile: %w", err)
+		return nil, "", xerrors.Errorf("reading backup part of the logfile: %w", err)
 	}
 	if string(lh) != lastLogHead {
-		return nil, xerrors.Errorf("loghead didn't match, expected '%s', last in logfile '%s'", string(lh), lastLogHead)
+		return nil, "", xerrors.Errorf("loghead didn't match, expected '%s', last in logfile '%s'", string(lh), lastLogHead)
 	}
 
 	// make sure we're at the end of the file
 	at, err := f.Seek(0, io.SeekCurrent)
 	if err != nil {
-		return nil, xerrors.Errorf("get current logfile offset: %w", err)
+		return nil, "", xerrors.Errorf("get current logfile offset: %w", err)
 	}
 	end, err := f.Seek(0, io.SeekEnd)
 	if err != nil {
-		return nil, xerrors.Errorf("get current logfile offset: %w", err)
+		return nil, "", xerrors.Errorf("get current logfile offset: %w", err)
 	}
 	if at != end {
-		return nil, xerrors.Errorf("logfile %s validated %d bytes, but the file has %d bytes (%d more)", p, at, end, end-at)
+		return nil, "", xerrors.Errorf("logfile %s validated %d bytes, but the file has %d bytes (%d more)", p, at, end, end-at)
 	}
 
-	log.Infow("log opened", "file", p, "openCount", openCount, "logValues", logvals)
+	compact := logvals > vals*int64(compactThresh)
+	if compact {
+		log.Infow("compacting log", "current", p, "openCount", openCount, "baseValues", vals, "logValues", logvals)
+		if err := f.Close(); err != nil {
+			return nil, "", xerrors.Errorf("closing current log: %w", err)
+		}
+
+		l, latest, err := d.createLog(filepath.Dir(p))
+		if err != nil {
+			return nil, "", xerrors.Errorf("creating compacted log: %w", err)
+		}
+
+		log.Infow("compacted log created, cleaning up old", "old", p, "new", latest)
+		if err := os.Remove(p); err != nil {
+			l.Close() // nolint
+			return nil, "", xerrors.Errorf("cleaning up old logfile: %w", err)
+		}
+
+		return l, latest, nil
+	}
+
+	log.Infow("log opened", "file", p, "openCount", openCount, "baseValues", vals, "logValues", logvals)
 
 	// todo: maybe write a magic 'opened at' entry; pad the log to filesystem page to prevent more exotic types of corruption
 
 	return &logfile{
 		file: f,
-	}, nil
+	}, filepath.Base(p), nil
 }
 
 func (l *logfile) writeLogHead(logname string, ds datastore.Batching) error {

lib/backupds/read.go

@@ -11,7 +11,7 @@ import (
 	"golang.org/x/xerrors"
 )
 
-func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) error {
+func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte, log bool) error) error {
 	scratch := make([]byte, 9)
 
 	// read array[2](
@@ -61,7 +61,7 @@ func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) err
 			return xerrors.Errorf("reading value: %w", err)
 		}
 
-		if err := cb(key, value); err != nil {
+		if err := cb(key, value, false); err != nil {
 			return err
 		}
 	}
@@ -110,7 +110,7 @@ func ReadBackup(r io.Reader, cb func(key datastore.Key, value []byte) error) err
 
 		key := datastore.NewKey(string(ent.Key))
 
-		if err := cb(key, ent.Value); err != nil {
+		if err := cb(key, ent.Value, true); err != nil {
 			return err
 		}
 	}
@@ -122,7 +122,7 @@ func RestoreInto(r io.Reader, dest datastore.Batching) error {
 		return xerrors.Errorf("creating batch: %w", err)
 	}
 
-	err = ReadBackup(r, func(key datastore.Key, value []byte) error {
+	err = ReadBackup(r, func(key datastore.Key, value []byte, _ bool) error {
 		if err := batch.Put(key, value); err != nil {
 			return xerrors.Errorf("put key: %w", err)
 		}

miner/miner.go

@@ -37,7 +37,13 @@ const (
 	evtTypeBlockMined = iota
 )
 
-// returns a callback reporting whether we mined a blocks in this round
+// waitFunc is expected to pace block mining at the configured network rate.
+//
+// baseTime is the timestamp of the mining base, i.e. the timestamp
+// of the tipset we're planning to construct upon.
+//
+// Upon each mining loop iteration, the returned callback is called reporting
+// whether we mined a block in this round or not.
 type waitFunc func(ctx context.Context, baseTime uint64) (func(bool, abi.ChainEpoch, error), abi.ChainEpoch, error)
 
 func randTimeOffset(width time.Duration) time.Duration {
@@ -48,6 +54,8 @@ func randTimeOffset(width time.Duration) time.Duration {
 	return val - (width / 2)
 }
 
+// NewMiner instantiates a miner with a concrete WinningPoStProver and a miner
+// address (which can be different from the worker's address).
 func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address, sf *slashfilter.SlashFilter, j journal.Journal) *Miner {
 	arc, err := lru.NewARC(10000)
 	if err != nil {
@@ -59,7 +67,16 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address,
 		epp:     epp,
 		address: addr,
 		waitFunc: func(ctx context.Context, baseTime uint64) (func(bool, abi.ChainEpoch, error), abi.ChainEpoch, error) {
-			// Wait around for half the block time in case other parents come in
+			// wait around for half the block time in case other parents come in
+			//
+			// if we're mining a block in the past via catch-up/rush mining,
+			// such as when recovering from a network halt, this sleep will be
+			// for a negative duration, and therefore **will return
+			// immediately**.
+			//
+			// the result is that we WILL NOT wait, therefore fast-forwarding
+			// and thus healing the chain by backfilling it with null rounds
+			// rapidly.
 			deadline := baseTime + build.PropagationDelaySecs
 			baseT := time.Unix(int64(deadline), 0)
 
@@ -79,6 +96,9 @@ func NewMiner(api api.FullNode, epp gen.WinningPoStProver, addr address.Address,
 	}
 }
 
+// Miner encapsulates the mining processes of the system.
+//
+// Refer to the godocs on mineOne and mine methods for more detail.
 type Miner struct {
 	api api.FullNode
 
@@ -91,15 +111,20 @@ type Miner struct {
 
 	waitFunc waitFunc
 
+	// lastWork holds the last MiningBase we built upon.
 	lastWork *MiningBase
 
 	sf *slashfilter.SlashFilter
+	// minedBlockHeights is a safeguard that caches the last heights we mined.
+	// It is consulted before publishing a newly mined block, for a sanity check
+	// intended to avoid slashings in case of a bug.
 	minedBlockHeights *lru.ARCCache
 
 	evtTypes [1]journal.EventType
 	journal  journal.Journal
 }
 
+// Address returns the address of the miner.
 func (m *Miner) Address() address.Address {
 	m.lk.Lock()
 	defer m.lk.Unlock()
@@ -107,7 +132,9 @@ func (m *Miner) Address() address.Address {
 	return m.address
 }
 
-func (m *Miner) Start(ctx context.Context) error {
+// Start starts the mining operation. It spawns a goroutine and returns
+// immediately. Start is not idempotent.
+func (m *Miner) Start(_ context.Context) error {
 	m.lk.Lock()
 	defer m.lk.Unlock()
 	if m.stop != nil {
@@ -118,6 +145,8 @@ func (m *Miner) Start(ctx context.Context) error {
 	return nil
 }
 
+// Stop stops the mining operation. It is not idempotent, and multiple adjacent
+// calls to Stop will fail.
 func (m *Miner) Stop(ctx context.Context) error {
 	m.lk.Lock()
 
@@ -145,6 +174,28 @@ func (m *Miner) niceSleep(d time.Duration) bool {
 	}
 }
 
+// mine runs the mining loop. It performs the following:
+//
+//  1.  Queries our current best currently-known mining candidate (tipset to
+//      build upon).
+//  2.  Waits until the propagation delay of the network has elapsed (currently
+//      6 seconds). The waiting is done relative to the timestamp of the best
+//      candidate, which means that if it's way in the past, we won't wait at
+//      all (e.g. in catch-up or rush mining).
+//  3.  After the wait, we query our best mining candidate. This will be the one
+//      we'll work with.
+//  4.  Sanity check that we _actually_ have a new mining base to mine on. If
+//      not, wait one epoch + propagation delay, and go back to the top.
+//  5.  We attempt to mine a block, by calling mineOne (refer to godocs). This
+//      method will either return a block if we were eligible to mine, or nil
+//      if we weren't.
+//  6a. If we mined a block, we update our state and push it out to the network
+//      via gossipsub.
+//  6b. If we didn't mine a block, we consider this to be a nil round on top of
+//      the mining base we selected. If other miner or miners on the network
+//      were eligible to mine, we will receive their blocks via gossipsub and
+//      we will select that tipset on the next iteration of the loop, thus
+//      discarding our null round.
 func (m *Miner) mine(ctx context.Context) {
 	ctx, span := trace.StartSpan(ctx, "/mine")
 	defer span.End()
@@ -305,11 +356,19 @@ minerLoop:
 	}
 }
 
+// MiningBase is the tipset on top of which we plan to construct our next block.
+// Refer to godocs on GetBestMiningCandidate.
 type MiningBase struct {
 	TipSet     *types.TipSet
 	NullRounds abi.ChainEpoch
 }
 
+// GetBestMiningCandidate implements the fork choice rule from a miner's
+// perspective.
+//
+// It obtains the current chain head (HEAD), and compares it to the last tipset
+// we selected as our mining base (LAST). If HEAD's weight is larger than
+// LAST's weight, it selects HEAD to build on. Else, it selects LAST.
 func (m *Miner) GetBestMiningCandidate(ctx context.Context) (*MiningBase, error) {
 	m.lk.Lock()
 	defer m.lk.Unlock()

node/impl/storminer.go

@@ -8,6 +8,9 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/filecoin-project/lotus/chain/actors/builtin"
+	"github.com/filecoin-project/lotus/chain/gen"
+
 	"github.com/filecoin-project/lotus/build"
 	"github.com/google/uuid"
 	"github.com/ipfs/go-cid"
@@ -63,6 +66,7 @@ type StorageMinerAPI struct {
 	AddrSel       *storage.AddressSelector
 	DealPublisher *storageadapter.DealPublisher
 
+	Epp gen.WinningPoStProver
 	DS  dtypes.MetadataDS
 
 	ConsiderOnlineStorageDealsConfigFunc        dtypes.ConsiderOnlineStorageDealsConfigFunc
@@ -696,4 +700,8 @@ func (sm *StorageMinerAPI) Discover(ctx context.Context) (apitypes.OpenRPCDocume
 	return build.OpenRPCDiscoverJSON_Miner(), nil
 }
 
+func (sm *StorageMinerAPI) ComputeProof(ctx context.Context, ssi []builtin.SectorInfo, rand abi.PoStRandomness) ([]builtin.PoStProof, error) {
+	return sm.Epp.ComputeProof(ctx, ssi, rand)
+}
+
 var _ api.StorageMiner = &StorageMinerAPI{}

node/modules/client.go

@@ -7,12 +7,10 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/filecoin-project/go-multistore"
+	"go.uber.org/fx"
-	"github.com/filecoin-project/go-state-types/abi"
 	"golang.org/x/xerrors"
 
-	"go.uber.org/fx"
+	"github.com/filecoin-project/go-data-transfer/channelmonitor"
-
 	dtimpl "github.com/filecoin-project/go-data-transfer/impl"
 	dtnet "github.com/filecoin-project/go-data-transfer/network"
 	dtgstransport "github.com/filecoin-project/go-data-transfer/transport/graphsync"
@@ -25,6 +23,8 @@ import (
 	storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
 	"github.com/filecoin-project/go-fil-markets/storagemarket/impl/requestvalidation"
 	smnet "github.com/filecoin-project/go-fil-markets/storagemarket/network"
+	"github.com/filecoin-project/go-multistore"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-storedcounter"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
@@ -121,8 +121,6 @@ func RegisterClientValidator(crv dtypes.ClientRequestValidator, dtm dtypes.Clien
 // NewClientGraphsyncDataTransfer returns a data transfer manager that just
 // uses the clients's Client DAG service for transfers
 func NewClientGraphsyncDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.Graphsync, ds dtypes.MetadataDS, r repo.LockedRepo) (dtypes.ClientDataTransfer, error) {
-	sc := storedcounter.New(ds, datastore.NewKey("/datatransfer/client/counter"))
-
 	// go-data-transfer protocol retries:
 	// 1s, 5s, 25s, 2m5s, 5m x 11 ~= 1 hour
 	dtRetryParams := dtnet.RetryParameters(time.Second, 5*time.Minute, 15, 5)
@@ -135,9 +133,30 @@ func NewClientGraphsyncDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.Grap
 		return nil, err
 	}
 
-	// data-transfer push channel restart configuration
+	// data-transfer push / pull channel restart configuration:
-	dtRestartConfig := dtimpl.PushChannelRestartConfig(time.Minute, 10, 1024, 10*time.Minute, 3)
+	dtRestartConfig := dtimpl.ChannelRestartConfig(channelmonitor.Config{
-	dt, err := dtimpl.NewDataTransfer(dtDs, filepath.Join(r.Path(), "data-transfer"), net, transport, sc, dtRestartConfig)
+		// For now only monitor push channels (for storage deals)
+		MonitorPushChannels: true,
+		// TODO: Enable pull channel monitoring (for retrievals) when the
+		//  following issue has been fixed:
+		// https://github.com/filecoin-project/go-data-transfer/issues/172
+		MonitorPullChannels: false,
+		// Wait up to 30s for the other side to respond to an Open channel message
+		AcceptTimeout: 30 * time.Second,
+		// Send a restart message if the data rate falls below 1024 bytes / minute
+		Interval:            time.Minute,
+		MinBytesTransferred: 1024,
+		// Perform check 10 times / minute
+		ChecksPerInterval: 10,
+		// After sending a restart, wait for at least 1 minute before sending another
+		RestartBackoff: time.Minute,
+		// After trying to restart 3 times, give up and fail the transfer
+		MaxConsecutiveRestarts: 3,
+		// Wait up to 30s for the other side to send a Complete message once all
+		// data has been sent / received
+		CompleteTimeout: 30 * time.Second,
+	})
+	dt, err := dtimpl.NewDataTransfer(dtDs, filepath.Join(r.Path(), "data-transfer"), net, transport, dtRestartConfig)
 	if err != nil {
 		return nil, err
 	}

node/modules/storageminer.go

@@ -330,7 +330,6 @@ func HandleMigrateProviderFunds(lc fx.Lifecycle, ds dtypes.MetadataDS, node api.
 // NewProviderDAGServiceDataTransfer returns a data transfer manager that just
 // uses the provider's Staging DAG service for transfers
 func NewProviderDAGServiceDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.StagingGraphsync, ds dtypes.MetadataDS, r repo.LockedRepo) (dtypes.ProviderDataTransfer, error) {
-	sc := storedcounter.New(ds, datastore.NewKey("/datatransfer/provider/counter"))
 	net := dtnet.NewFromLibp2pHost(h)
 
 	dtDs := namespace.Wrap(ds, datastore.NewKey("/datatransfer/provider/transfers"))
@@ -340,7 +339,7 @@ func NewProviderDAGServiceDataTransfer(lc fx.Lifecycle, h host.Host, gs dtypes.S
 		return nil, err
 	}
 
-	dt, err := dtimpl.NewDataTransfer(dtDs, filepath.Join(r.Path(), "data-transfer"), net, transport, sc)
+	dt, err := dtimpl.NewDataTransfer(dtDs, filepath.Join(r.Path(), "data-transfer"), net, transport)
 	if err != nil {
 		return nil, err
 	}

paychmgr/paych_test.go

@@ -23,6 +23,7 @@ import (
 	paychmock "github.com/filecoin-project/lotus/chain/actors/builtin/paych/mock"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/lib/sigs"
+	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
 )
 
 func TestCheckVoucherValid(t *testing.T) {

tools/packer/etc/motd

@@ -52,6 +52,13 @@ You only need to do this once, after which, you can enable and start the miner.
 
 
 
+Do you want to access your lotus daemon remotely? Learn how to setup token authentication
+and use client libraries from lotus docs.
+
+https://docs.filecoin.io/build/lotus/enable-remote-api-access/
+
+
+
 For more information, see             https://docs.filecoin.io/
 Found a bug? let us know!             https://github.com/filecoin-project/lotus
 Chat with us on slack!                https://filecoinproject.slack.com/archives/CEGN061C5

tools/packer/homedir/bashrc

@@ -1,5 +1,12 @@
 PS1="[\h \w] ⨎ "
 
 export PROMT_DIRTRIM=1
+
+# Where to find the lotus repo
 export LOTUS_PATH=/var/lib/lotus
-export LOTUS_MINER_PATH=/var/lib/lotus-miner
+
+# The miner is not running in this image by default.
+# export LOTUS_MINER_PATH=/var/lib/lotus-miner
+
+# To access the lotus node remotely, the following environment variable may be used.
+# export FULLNODE_API_INFO=<token>:/ip4/<public-ip>/tcp/1234/http

tools/packer/lotus.pkr.hcl

@@ -20,7 +20,23 @@ locals {
 source "amazon-ebs" "lotus" {
   ami_name      = "lotus-${var.lotus_network}-${var.git_tag}-${local.timestamp}"
   ami_regions = [
+    "ap-east-1",
+    "ap-northeast-1",
+    "ap-northeast-2",
+    "ap-northeast-3",
+    "ap-south-1",
+    "ap-southeast-1",
+    "ap-southeast-2",
+    "ca-central-1",
+    "eu-central-1",
+    "eu-north-1",
+    "eu-west-1",
+    "eu-west-2",
+    "eu-west-3",
+    "sa-east-1",
     "us-east-1",
+    "us-east-2",
+    "us-west-1",
     "us-west-2",
   ]
   ami_groups = [
@@ -37,7 +53,7 @@ source "amazon-ebs" "lotus" {
   instance_type = "t2.micro"
   source_ami_filter {
     filters = {
-      name = "ubuntu-minimal/images/*ubuntu-focal-20.04-amd64-minimal*"
+      name = "ubuntu/images/*ubuntu-focal-20.04-amd64-server-*"
       root-device-type = "ebs"
       virtualization-type = "hvm"
     }
@@ -85,6 +101,10 @@ build {
     source = "./tools/packer/systemd/lotus-miner.service"
     destination = "lotus-miner.service"
   }
+  provisioner "file" {
+    source = "./tools/packer/repo/config.toml"
+    destination = "config.toml"
+  }
   provisioner "file" {
     source = "./tools/packer/etc/motd"
     destination = "motd"

tools/packer/repo/config.toml

@@ -0,0 +1,36 @@
+[API]
+ListenAddress = "/ip4/0.0.0.0/tcp/1234/http"
+#  RemoteListenAddress = ""
+#  Timeout = "30s"
+#
+[Libp2p]
+ListenAddresses = ["/ip4/0.0.0.0/tcp/5678", "/ip6/::/tcp/5678"]
+#  AnnounceAddresses = []
+#  NoAnnounceAddresses = []
+#  ConnMgrLow = 150
+#  ConnMgrHigh = 180
+#  ConnMgrGrace = "20s"
+#
+[Pubsub]
+#  Bootstrapper = false
+#  RemoteTracer = "/dns4/pubsub-tracer.filecoin.io/tcp/4001/p2p/QmTd6UvR47vUidRNZ1ZKXHrAFhqTJAD27rKL9XYghEKgKX"
+#
+[Client]
+#  UseIpfs = false
+#  IpfsOnlineMode = false
+#  IpfsMAddr = ""
+#  IpfsUseForRetrieval = false
+#  SimultaneousTransfers = 20
+#
+[Metrics]
+#  Nickname = ""
+#  HeadNotifs = false
+#
+[Wallet]
+#  RemoteBackend = ""
+#  EnableLedger = false
+#  DisableLocal = false
+#
+[Fees]
+#  DefaultMaxFee = "0.007 FIL"
+#

tools/packer/setup.sh

@@ -22,11 +22,14 @@ MANAGED_FILES=(
   /lib/systemd/system/lotus-daemon.service
 	/lib/systemd/system/lotus-miner.service
 	/etc/motd
+	/var/lib/lotus/config.toml
 )
 
 # install libs.
-apt update
+export DEBIAN_FRONTEND=noninteractive
-apt -y install libhwloc15 ocl-icd-libopencl1
+apt-get update
+apt-get -y install libhwloc15 ocl-icd-libopencl1 ufw
+apt-get -y upgrade -q -y -u -o Dpkg::Options::="--force-confold"
 ln -s /usr/lib/x86_64-linux-gnu/libhwloc.so.15 /usr/lib/x86_64-linux-gnu/libhwloc.so.5
 
 # Create lotus user
@@ -55,3 +58,10 @@ done
 # Enable services
 systemctl daemon-reload
 systemctl enable lotus-daemon
+
+# Setup firewall
+ufw enable
+ufw default deny incoming
+ufw default allow outgoing
+ufw allow ssh
+ufw allow 5678   #libp2p