Merge pull request #3501 from filecoin-project/asr/changelog

Lotus version 0.5.8

.github/labels.yml

@@ -152,6 +152,9 @@
 - name: impact/test-flakiness
   color: DDE1E4
   description: "Impact: Test Flakiness"
+- name: impact/consensus
+  color: b20014
+  description: "Impact: Consensus"
 
 ###
 ### Topics

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Lotus changelog
 
+# 0.5.8 / 2020-09-02
+
+This patch includes some bugfixes to the sector sealing process, and updates go-fil-markets. It also improves the performance of blocksync, adds a method to export chain state trees, and improves chainwatch.
+
+## Changes
+
+- Upgrade markets to v0.5.9 (https://github.com/filecoin-project/lotus/pull/3496)
+- Improve blocksync to load fewer messages: (https://github.com/filecoin-project/lotus/pull/3494)
+- Fix a panic in the ffi-wrapper's `ReadPiece` (https://github.com/filecoin-project/lotus/pull/3492/files)
+- Fix a deadlock in the sealing scheduler (https://github.com/filecoin-project/lotus/pull/3489)
+- Add test vectors for tipset tests (https://github.com/filecoin-project/lotus/pull/3485/files)
+- Improve the advance-block debug command (https://github.com/filecoin-project/lotus/pull/3476)
+- Add toggle for message processing to Lotus PCR (https://github.com/filecoin-project/lotus/pull/3470)
+- Allow exporting recent chain state trees (https://github.com/filecoin-project/lotus/pull/3463)
+- Remove height from chain rand (https://github.com/filecoin-project/lotus/pull/3458)
+- Disable GC on chain badger datastore (https://github.com/filecoin-project/lotus/pull/3457)
+- Account for `GasPremium` in `GasEstimateFeeCap` (https://github.com/filecoin-project/lotus/pull/3456)
+- Update go-libp2p-pubsub to `master` (https://github.com/filecoin-project/lotus/pull/3455)
+- Chainwatch improvements (https://github.com/filecoin-project/lotus/pull/3442)
+
 # 0.5.7 / 2020-08-31
 
 This patch release includes some bugfixes and enhancements to the sector lifecycle and message pool logic. 

api/api_full.go

@@ -109,7 +109,10 @@ type FullNode interface {
 	ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error)
 
 	// ChainExport returns a stream of bytes with CAR dump of chain data.
-	ChainExport(context.Context, types.TipSetKey) (<-chan []byte, error)
+	// The exported chain data includes the header chain from the given tipset
+	// back to genesis, the entire genesis state, and the most recent 'nroots'
+	// state trees.
+	ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error)
 
 	// MethodGroup: Beacon
 	// The Beacon method group contains methods for interacting with the random beacon (DRAND)

api/apistruct/struct.go

@@ -86,7 +86,7 @@ type FullNodeStruct struct {
 		ChainGetNode                  func(ctx context.Context, p string) (*api.IpldObject, error)                                                       `perm:"read"`
 		ChainGetMessage               func(context.Context, cid.Cid) (*types.Message, error)                                                             `perm:"read"`
 		ChainGetPath                  func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error)                                 `perm:"read"`
-		ChainExport                   func(context.Context, types.TipSetKey) (<-chan []byte, error)                                                      `perm:"read"`
+		ChainExport                   func(context.Context, abi.ChainEpoch, types.TipSetKey) (<-chan []byte, error)                                      `perm:"read"`
 
 		BeaconGetEntry func(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) `perm:"read"`
 
@@ -654,8 +654,8 @@ func (c *FullNodeStruct) ChainGetPath(ctx context.Context, from types.TipSetKey,
 	return c.Internal.ChainGetPath(ctx, from, to)
 }
 
-func (c *FullNodeStruct) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan []byte, error) {
-	return c.Internal.ChainExport(ctx, tsk)
+func (c *FullNodeStruct) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) {
+	return c.Internal.ChainExport(ctx, nroots, tsk)
 }
 
 func (c *FullNodeStruct) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) {

build/version.go

@@ -25,7 +25,7 @@ func buildType() string {
 }
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "0.5.7"
+const BuildVersion = "0.5.8"
 
 func UserVersion() string {
 	return BuildVersion + buildType() + CurrentCommit

chain/blocksync/server.go

@@ -221,37 +221,36 @@ func collectChainSegment(
 func gatherMessages(cs *store.ChainStore, ts *types.TipSet) ([]*types.Message, [][]uint64, []*types.SignedMessage, [][]uint64, error) {
 	blsmsgmap := make(map[cid.Cid]uint64)
 	secpkmsgmap := make(map[cid.Cid]uint64)
-	var secpkmsgs []*types.SignedMessage
-	var blsmsgs []*types.Message
 	var secpkincl, blsincl [][]uint64
 
+	var blscids, secpkcids []cid.Cid
 	for _, block := range ts.Blocks() {
-		bmsgs, smsgs, err := cs.MessagesForBlock(block)
+		bc, sc, err := cs.ReadMsgMetaCids(block.Messages)
 		if err != nil {
 			return nil, nil, nil, nil, err
 		}
 
 		// FIXME: DRY. Use `chain.Message` interface.
-		bmi := make([]uint64, 0, len(bmsgs))
-		for _, m := range bmsgs {
-			i, ok := blsmsgmap[m.Cid()]
+		bmi := make([]uint64, 0, len(bc))
+		for _, m := range bc {
+			i, ok := blsmsgmap[m]
 			if !ok {
-				i = uint64(len(blsmsgs))
-				blsmsgs = append(blsmsgs, m)
-				blsmsgmap[m.Cid()] = i
+				i = uint64(len(blscids))
+				blscids = append(blscids, m)
+				blsmsgmap[m] = i
 			}
 
 			bmi = append(bmi, i)
 		}
 		blsincl = append(blsincl, bmi)
 
-		smi := make([]uint64, 0, len(smsgs))
-		for _, m := range smsgs {
-			i, ok := secpkmsgmap[m.Cid()]
+		smi := make([]uint64, 0, len(sc))
+		for _, m := range sc {
+			i, ok := secpkmsgmap[m]
 			if !ok {
-				i = uint64(len(secpkmsgs))
-				secpkmsgs = append(secpkmsgs, m)
-				secpkmsgmap[m.Cid()] = i
+				i = uint64(len(secpkcids))
+				secpkcids = append(secpkcids, m)
+				secpkmsgmap[m] = i
 			}
 
 			smi = append(smi, i)
@@ -259,5 +258,15 @@ func gatherMessages(cs *store.ChainStore, ts *types.TipSet) ([]*types.Message, [
 		secpkincl = append(secpkincl, smi)
 	}
 
+	blsmsgs, err := cs.LoadMessagesFromCids(blscids)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
+	secpkmsgs, err := cs.LoadSignedMessagesFromCids(secpkcids)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+
 	return blsmsgs, blsincl, secpkmsgs, secpkincl, nil
 }

chain/stmgr/call.go

@@ -95,7 +95,7 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 
 	state := ts.ParentState()
 
-	r := store.NewChainRand(sm.cs, ts.Cids(), ts.Height())
+	r := store.NewChainRand(sm.cs, ts.Cids())
 
 	return sm.CallRaw(ctx, msg, state, r, ts.Height())
 }
@@ -113,7 +113,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		return nil, xerrors.Errorf("computing tipset state: %w", err)
 	}
 
-	r := store.NewChainRand(sm.cs, ts.Cids(), ts.Height())
+	r := store.NewChainRand(sm.cs, ts.Cids())
 
 	if span.IsRecordingEvents() {
 		span.AddAttributes(

chain/stmgr/stmgr.go

@@ -338,7 +338,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, ts *types.TipSet
 		cids[i] = v.Cid()
 	}
 
-	r := store.NewChainRand(sm.cs, cids, blks[0].Height)
+	r := store.NewChainRand(sm.cs, cids)
 
 	blkmsgs, err := sm.cs.BlockMsgsForTipset(ts)
 	if err != nil {

chain/stmgr/utils.go

@@ -432,7 +432,7 @@ func ComputeState(ctx context.Context, sm *StateManager, height abi.ChainEpoch,
 		return cid.Undef, nil, err
 	}
 
-	r := store.NewChainRand(sm.cs, ts.Cids(), height)
+	r := store.NewChainRand(sm.cs, ts.Cids())
 	vmopt := &vm.VMOpts{
 		StateBase:      base,
 		Epoch:          height,

chain/store/store.go

@@ -49,6 +49,7 @@ var chainHeadKey = dstore.NewKey("head")
 var blockValidationCacheKeyPrefix = dstore.NewKey("blockValidation")
 
 var DefaultTipSetCacheSize = 8192
+var DefaultMsgMetaCacheSize = 2048
 
 func init() {
 	if s := os.Getenv("LOTUS_CHAIN_TIPSET_CACHE"); s != "" {
@@ -58,6 +59,14 @@ func init() {
 		}
 		DefaultTipSetCacheSize = tscs
 	}
+
+	if s := os.Getenv("LOTUS_CHAIN_MSGMETA_CACHE"); s != "" {
+		mmcs, err := strconv.Atoi(s)
+		if err != nil {
+			log.Errorf("failed to parse 'LOTUS_CHAIN_MSGMETA_CACHE' env var: %s", err)
+		}
+		DefaultMsgMetaCacheSize = mmcs
+	}
 }
 
 // ReorgNotifee represents a callback that gets called upon reorgs.
@@ -97,7 +106,7 @@ type ChainStore struct {
 }
 
 func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls vm.SyscallBuilder) *ChainStore {
-	c, _ := lru.NewARC(2048)
+	c, _ := lru.NewARC(DefaultMsgMetaCacheSize)
 	tsc, _ := lru.NewARC(DefaultTipSetCacheSize)
 	cs := &ChainStore{
 		bs:       bs,
@@ -834,7 +843,7 @@ type mmCids struct {
 	secpk []cid.Cid
 }
 
-func (cs *ChainStore) readMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error) {
+func (cs *ChainStore) ReadMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error) {
 	o, ok := cs.mmCache.Get(mmc)
 	if ok {
 		mmcids := o.(*mmCids)
@@ -890,7 +899,7 @@ func (cs *ChainStore) GetPath(ctx context.Context, from types.TipSetKey, to type
 }
 
 func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
-	blscids, secpkcids, err := cs.readMsgMetaCids(b.Messages)
+	blscids, secpkcids, err := cs.ReadMsgMetaCids(b.Messages)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -1114,7 +1123,7 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t
 	return cs.LoadTipSet(lbts.Parents())
 }
 
-func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) {
+func recurseLinks(bs bstore.Blockstore, walked *cid.Set, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) {
 	if root.Prefix().Codec != cid.DagCBOR {
 		return in, nil
 	}
@@ -1131,9 +1140,14 @@ func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid,
 			return
 		}
 
+		// traversed this already...
+		if !walked.Visit(c) {
+			return
+		}
+
 		in = append(in, c)
 		var err error
-		in, err = recurseLinks(bs, c, in)
+		in, err = recurseLinks(bs, walked, c, in)
 		if err != nil {
 			rerr = err
 		}
@@ -1145,12 +1159,13 @@ func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid,
 	return in, rerr
 }
 
-func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer) error {
+func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, w io.Writer) error {
 	if ts == nil {
 		ts = cs.GetHeaviestTipSet()
 	}
 
 	seen := cid.NewSet()
+	walked := cid.NewSet()
 
 	h := &car.CarHeader{
 		Roots:   ts.Cids(),
@@ -1182,7 +1197,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer)
 			return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err)
 		}
 
-		cids, err := recurseLinks(cs.bs, b.Messages, []cid.Cid{b.Messages})
+		cids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages})
 		if err != nil {
 			return xerrors.Errorf("recursing messages failed: %w", err)
 		}
@@ -1198,8 +1213,8 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer)
 
 		out := cids
 
-		if b.Height == 0 {
-			cids, err := recurseLinks(cs.bs, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
+		if b.Height == 0 || b.Height > ts.Height()-inclRecentRoots {
+			cids, err := recurseLinks(cs.bs, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot})
 			if err != nil {
 				return xerrors.Errorf("recursing genesis state failed: %w", err)
 			}
@@ -1282,14 +1297,12 @@ func (cs *ChainStore) GetLatestBeaconEntry(ts *types.TipSet) (*types.BeaconEntry
 type chainRand struct {
 	cs   *ChainStore
 	blks []cid.Cid
-	bh   abi.ChainEpoch
 }
 
-func NewChainRand(cs *ChainStore, blks []cid.Cid, bheight abi.ChainEpoch) vm.Rand {
+func NewChainRand(cs *ChainStore, blks []cid.Cid) vm.Rand {
 	return &chainRand{
 		cs:   cs,
 		blks: blks,
-		bh:   bheight,
 	}
 }
 

chain/store/store_test.go

@@ -96,7 +96,7 @@ func TestChainExportImport(t *testing.T) {
 	}
 
 	buf := new(bytes.Buffer)
-	if err := cg.ChainStore().Export(context.TODO(), last, buf); err != nil {
+	if err := cg.ChainStore().Export(context.TODO(), last, 0, buf); err != nil {
 		t.Fatal(err)
 	}
 

chain/store/weight.go

@@ -41,7 +41,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn
 
 		var st power.State
 		if err := cst.Get(ctx, act.Head, &st); err != nil {
-			return types.NewInt(0), xerrors.Errorf("get power actor head: %w", err)
+			return types.NewInt(0), xerrors.Errorf("get power actor head (%s, height=%d): %w", act.Head, ts.Height(), err)
 		}
 		tpow = st.TotalQualityAdjPower // TODO: REVIEW: Is this correct?
 	}

cli/chain.go

@@ -859,6 +859,10 @@ var chainExportCmd = &cli.Command{
 		&cli.StringFlag{
 			Name: "tipset",
 		},
+		&cli.Int64Flag{
+			Name:  "recent-stateroots",
+			Usage: "specify the number of recent state roots to include in the export",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := GetFullNodeAPI(cctx)
@@ -872,6 +876,11 @@ var chainExportCmd = &cli.Command{
 			return fmt.Errorf("must specify filename to export chain to")
 		}
 
+		rsrs := abi.ChainEpoch(cctx.Int64("recent-stateroots"))
+		if cctx.IsSet("recent-stateroots") && rsrs < build.Finality {
+			return fmt.Errorf("\"recent-stateroots\" has to be greater than %d", build.Finality)
+		}
+
 		fi, err := os.Create(cctx.Args().First())
 		if err != nil {
 			return err
@@ -888,7 +897,7 @@ var chainExportCmd = &cli.Command{
 			return err
 		}
 
-		stream, err := api.ChainExport(ctx, ts.Key())
+		stream, err := api.ChainExport(ctx, rsrs, ts.Key())
 		if err != nil {
 			return err
 		}

cmd/lotus-chainwatch/processor/processor.go

@@ -369,7 +369,9 @@ where rnk <= $1
 			maxBlock = bh.Height
 		}
 	}
-	log.Infow("Gathered Blocks to Process", "start", minBlock, "end", maxBlock)
+	if minBlock <= maxBlock {
+		log.Infow("Gathered Blocks to Process", "start", minBlock, "end", maxBlock)
+	}
 	return out, rows.Close()
 }
 

cmd/lotus-chainwatch/syncer/blockssub.go

@@ -11,16 +11,17 @@ import (
 func (s *Syncer) subBlocks(ctx context.Context) {
 	sub, err := s.node.SyncIncomingBlocks(ctx)
 	if err != nil {
-		log.Error(err)
+		log.Errorf("opening incoming block channel: %+v", err)
 		return
 	}
 
+	log.Infow("Capturing incoming blocks")
 	for bh := range sub {
 		err := s.storeHeaders(map[cid.Cid]*types.BlockHeader{
 			bh.Cid(): bh,
 		}, false, time.Now())
 		if err != nil {
-			log.Errorf("%+v", err)
+			log.Errorf("storing incoming block header: %+v", err)
 		}
 	}
 }

cmd/lotus-chainwatch/syncer/sync.go

@@ -160,23 +160,37 @@ func (s *Syncer) Start(ctx context.Context) {
 		log.Fatal(err)
 	}
 
+	// capture all reported blocks
+	go s.subBlocks(ctx)
+
+	// we need to ensure that on a restart we don't reprocess the whole flarping chain
+	var sinceEpoch uint64
+	blkCID, height, err := s.mostRecentlySyncedBlockHeight()
+	if err != nil {
+		log.Fatalw("failed to find most recently synced block", "error", err)
+	} else {
+		if height > 0 {
+			log.Infow("Found starting point for syncing", "blockCID", blkCID.String(), "height", height)
+			sinceEpoch = uint64(height)
+		}
+	}
+
 	// continue to keep the block headers table up to date.
 	notifs, err := s.node.ChainNotify(ctx)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	// we need to ensure that on a restart we don't reprocess the whole flarping chain
-	blkCID, height, err := s.mostRecentlySyncedBlockHeight()
-	if err != nil {
-		log.Fatalw("failed to find most recently synced block", "error", err)
-	}
-	log.Infow("Found starting point for syncing", "blockCID", blkCID.String(), "height", height)
-	sinceEpoch := uint64(height)
 	go func() {
 		for notif := range notifs {
 			for _, change := range notif {
 				switch change.Type {
+				case store.HCCurrent:
+					// This case is important for capturing the initial state of a node
+					// which might be on a dead network with no new blocks being produced.
+					// It also allows a fresh Chainwatch instance to start walking the
+					// chain without waiting for a new block to come along.
+					fallthrough
 				case store.HCApply:
 					unsynced, err := s.unsyncedBlocks(ctx, change.Val, sinceEpoch)
 					if err != nil {

cmd/lotus-pcr/main.go

@@ -133,6 +133,18 @@ var runCmd = &cli.Command{
 			Usage:   "do not send any messages",
 			Value:   false,
 		},
+		&cli.BoolFlag{
+			Name:    "pre-commit",
+			EnvVars: []string{"LOTUS_PCR_PRE_COMMIT"},
+			Usage:   "process PreCommitSector messages",
+			Value:   true,
+		},
+		&cli.BoolFlag{
+			Name:    "prove-commit",
+			EnvVars: []string{"LOTUS_PCR_PROVE_COMMIT"},
+			Usage:   "process ProveCommitSector messages",
+			Value:   true,
+		},
 		&cli.IntFlag{
 			Name:    "head-delay",
 			EnvVars: []string{"LOTUS_PCR_HEAD_DELAY"},
@@ -180,12 +192,16 @@ var runCmd = &cli.Command{
 		percentExtra := cctx.Int("percent-extra")
 		maxMessageQueue := cctx.Int("max-message-queue")
 		dryRun := cctx.Bool("dry-run")
+		preCommitEnabled := cctx.Bool("pre-commit")
+		proveCommitEnabled := cctx.Bool("prove-commit")
 
 		rf := &refunder{
-			api:          api,
-			wallet:       from,
-			percentExtra: percentExtra,
-			dryRun:       dryRun,
+			api:                api,
+			wallet:             from,
+			percentExtra:       percentExtra,
+			dryRun:             dryRun,
+			preCommitEnabled:   preCommitEnabled,
+			proveCommitEnabled: proveCommitEnabled,
 		}
 
 		for tipset := range tipsetsCh {
@@ -281,10 +297,12 @@ type refunderNodeApi interface {
 }
 
 type refunder struct {
-	api          refunderNodeApi
-	wallet       address.Address
-	percentExtra int
-	dryRun       bool
+	api                refunderNodeApi
+	wallet             address.Address
+	percentExtra       int
+	dryRun             bool
+	preCommitEnabled   bool
+	proveCommitEnabled bool
 }
 
 func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet) (*MinersRefund, error) {
@@ -331,7 +349,12 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet) (*Mi
 
 		switch m.Method {
 		case builtin.MethodsMiner.ProveCommitSector:
+			if !r.proveCommitEnabled {
+				continue
+			}
+
 			messageMethod = "ProveCommitSector"
+
 			if recps[i].ExitCode != exitcode.Ok {
 				log.Debugw("skipping non-ok exitcode message", "method", messageMethod, "cid", msg.Cid, "miner", m.To, "exitcode", recps[i].ExitCode)
 				continue
@@ -369,6 +392,10 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet) (*Mi
 
 			refundValue = collateral
 		case builtin.MethodsMiner.PreCommitSector:
+			if !r.preCommitEnabled {
+				continue
+			}
+
 			messageMethod = "PreCommitSector"
 
 			if recps[i].ExitCode != exitcode.Ok {

cmd/lotus/daemon.go

@@ -100,7 +100,11 @@ var DaemonCmd = &cli.Command{
 		},
 		&cli.StringFlag{
 			Name:  "import-chain",
-			Usage: "on first run, load chain from given file",
+			Usage: "on first run, load chain from given file and validate",
+		},
+		&cli.StringFlag{
+			Name:  "import-snapshot",
+			Usage: "import chain state from a given chain export file",
 		},
 		&cli.BoolFlag{
 			Name:  "halt-after-import",
@@ -191,13 +195,23 @@ var DaemonCmd = &cli.Command{
 		}
 
 		chainfile := cctx.String("import-chain")
-		if chainfile != "" {
+		snapshot := cctx.String("import-snapshot")
+		if chainfile != "" || snapshot != "" {
+			if chainfile != "" && snapshot != "" {
+				return fmt.Errorf("cannot specify both 'import-snapshot' and 'import-chain'")
+			}
+			var issnapshot bool
+			if chainfile == "" {
+				chainfile = snapshot
+				issnapshot = true
+			}
+
 			chainfile, err := homedir.Expand(chainfile)
 			if err != nil {
 				return err
 			}
 
-			if err := ImportChain(r, chainfile); err != nil {
+			if err := ImportChain(r, chainfile, issnapshot); err != nil {
 				return err
 			}
 			if cctx.Bool("halt-after-import") {
@@ -312,7 +326,7 @@ func importKey(ctx context.Context, api api.FullNode, f string) error {
 	return nil
 }
 
-func ImportChain(r repo.Repo, fname string) error {
+func ImportChain(r repo.Repo, fname string, snapshot bool) error {
 	fi, err := os.Open(fname)
 	if err != nil {
 		return err
@@ -357,9 +371,11 @@ func ImportChain(r repo.Repo, fname string) error {
 
 	stm := stmgr.NewStateManager(cst)
 
-	log.Infof("validating imported chain...")
-	if err := stm.ValidateChain(context.TODO(), ts); err != nil {
-		return xerrors.Errorf("chain validation failed: %w", err)
+	if !snapshot {
+		log.Infof("validating imported chain...")
+		if err := stm.ValidateChain(context.TODO(), ts); err != nil {
+			return xerrors.Errorf("chain validation failed: %w", err)
+		}
 	}
 
 	log.Info("accepting %s as new head", ts.Cids())

cmd/lotus/debug_advance.go

@@ -62,6 +62,9 @@ func init() {
 			}
 
 			mbi, err := api.MinerGetBaseInfo(ctx, addr, head.Height()+1, head.Key())
+			if err != nil {
+				return xerrors.Errorf("getting base info: %w", err)
+			}
 
 			ep := &types.ElectionProof{}
 			ep.WinCount = ep.ComputeWinCount(types.NewInt(1), types.NewInt(1))

conformance/driver.go

@@ -2,7 +2,10 @@ package conformance
 
 import (
 	"context"
+	"fmt"
 
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
@@ -14,7 +17,10 @@ import (
 	"github.com/filecoin-project/test-vectors/chaos"
 	"github.com/filecoin-project/test-vectors/schema"
 
+	"github.com/filecoin-project/go-address"
+
 	"github.com/ipfs/go-cid"
+	ds "github.com/ipfs/go-datastore"
 )
 
 var (
@@ -24,16 +30,92 @@ var (
 )
 
 type Driver struct {
-	ctx    context.Context
-	vector *schema.TestVector
+	ctx      context.Context
+	selector schema.Selector
 }
 
-func NewDriver(ctx context.Context, vector *schema.TestVector) *Driver {
-	return &Driver{ctx: ctx, vector: vector}
+func NewDriver(ctx context.Context, selector schema.Selector) *Driver {
+	return &Driver{ctx: ctx, selector: selector}
+}
+
+type ExecuteTipsetResult struct {
+	ReceiptsRoot  cid.Cid
+	PostStateRoot cid.Cid
+
+	// AppliedMessages stores the messages that were applied, in the order they
+	// were applied. It includes implicit messages (cron, rewards).
+	AppliedMessages []*types.Message
+	// AppliedResults stores the results of AppliedMessages, in the same order.
+	AppliedResults []*vm.ApplyRet
+}
+
+// ExecuteTipset executes the supplied tipset on top of the state represented
+// by the preroot CID.
+//
+// parentEpoch is the last epoch in which an actual tipset was processed. This
+// is used by Lotus for null block counting and cron firing.
+//
+// This method returns the the receipts root, the poststate root, and the VM
+// message results. The latter _include_ implicit messages, such as cron ticks
+// and reward withdrawal per miner.
+func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot cid.Cid, parentEpoch abi.ChainEpoch, tipset *schema.Tipset) (*ExecuteTipsetResult, error) {
+	var (
+		syscalls = mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier))
+		vmRand   = new(testRand)
+
+		cs = store.NewChainStore(bs, ds, syscalls)
+		sm = stmgr.NewStateManager(cs)
+	)
+
+	blocks := make([]store.BlockMessages, 0, len(tipset.Blocks))
+	for _, b := range tipset.Blocks {
+		sb := store.BlockMessages{
+			Miner:    b.MinerAddr,
+			WinCount: b.WinCount,
+		}
+		for _, m := range b.Messages {
+			msg, err := types.DecodeMessage(m)
+			if err != nil {
+				return nil, err
+			}
+			switch msg.From.Protocol() {
+			case address.SECP256K1:
+				sb.SecpkMessages = append(sb.SecpkMessages, msg)
+			case address.BLS:
+				sb.BlsMessages = append(sb.BlsMessages, msg)
+			default:
+				return nil, fmt.Errorf("from account is not secpk nor bls: %s", msg.From)
+			}
+		}
+		blocks = append(blocks, sb)
+	}
+
+	var (
+		messages []*types.Message
+		results  []*vm.ApplyRet
+	)
+
+	postcid, receiptsroot, err := sm.ApplyBlocks(context.Background(), parentEpoch, preroot, blocks, tipset.Epoch, vmRand, func(_ cid.Cid, msg *types.Message, ret *vm.ApplyRet) error {
+		messages = append(messages, msg)
+		results = append(results, ret)
+		return nil
+	}, tipset.BaseFee)
+
+	if err != nil {
+		return nil, err
+	}
+
+	ret := &ExecuteTipsetResult{
+		ReceiptsRoot:    receiptsroot,
+		PostStateRoot:   postcid,
+		AppliedMessages: messages,
+		AppliedResults:  results,
+	}
+	return ret, nil
 }
 
 // ExecuteMessage executes a conformance test vector message in a temporary VM.
-func (d *Driver) ExecuteMessage(msg *types.Message, preroot cid.Cid, bs blockstore.Blockstore, epoch abi.ChainEpoch) (*vm.ApplyRet, cid.Cid, error) {
+func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message) (*vm.ApplyRet, cid.Cid, error) {
 	vmOpts := &vm.VMOpts{
 		StateBase:      preroot,
 		Epoch:          epoch,
@@ -52,10 +134,10 @@ func (d *Driver) ExecuteMessage(msg *types.Message, preroot cid.Cid, bs blocksto
 	invoker := vm.NewInvoker()
 
 	// add support for the puppet and chaos actors.
-	if puppetOn, ok := d.vector.Selector["puppet_actor"]; ok && puppetOn == "true" {
+	if puppetOn, ok := d.selector["puppet_actor"]; ok && puppetOn == "true" {
 		invoker.Register(puppet.PuppetActorCodeID, puppet.Actor{}, puppet.State{})
 	}
-	if chaosOn, ok := d.vector.Selector["chaos_actor"]; ok && chaosOn == "true" {
+	if chaosOn, ok := d.selector["chaos_actor"]; ok && chaosOn == "true" {
 		invoker.Register(chaos.ChaosActorCodeCID, chaos.Actor{}, chaos.State{})
 	}
 

conformance/runner_test.go

@@ -6,12 +6,17 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 
+	"github.com/ipfs/go-cid"
+	ds "github.com/ipfs/go-datastore"
+
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/lib/blockstore"
@@ -135,6 +140,8 @@ func TestConformance(t *testing.T) {
 			switch vector.Class {
 			case "message":
 				executeMessageVector(t, &vector)
+			case "tipset":
+				executeTipsetVector(t, &vector)
 			default:
 				t.Fatalf("test vector class not supported: %s", vector.Class)
 			}
@@ -150,24 +157,11 @@ func executeMessageVector(t *testing.T, vector *schema.TestVector) {
 		root  = vector.Pre.StateTree.RootCID
 	)
 
-	bs := blockstore.NewTemporary()
-
-	// Read the base64-encoded CAR from the vector, and inflate the gzip.
-	buf := bytes.NewReader(vector.CAR)
-	r, err := gzip.NewReader(buf)
-	if err != nil {
-		t.Fatalf("failed to inflate gzipped CAR: %s", err)
-	}
-	defer r.Close() // nolint
-
-	// Load the CAR embedded in the test vector into the Blockstore.
-	_, err = car.LoadCar(bs, r)
-	if err != nil {
-		t.Fatalf("failed to load state tree car from test vector: %s", err)
-	}
+	// Load the CAR into a new temporary Blockstore.
+	bs := loadCAR(t, vector.CAR)
 
 	// Create a new Driver.
-	driver := NewDriver(ctx, vector)
+	driver := NewDriver(ctx, vector.Selector)
 
 	// Apply every message.
 	for i, m := range vector.ApplyMessages {
@@ -183,52 +177,128 @@ func executeMessageVector(t *testing.T, vector *schema.TestVector) {
 
 		// Execute the message.
 		var ret *vm.ApplyRet
-		ret, root, err = driver.ExecuteMessage(msg, root, bs, epoch)
+		ret, root, err = driver.ExecuteMessage(bs, root, epoch, msg)
 		if err != nil {
 			t.Fatalf("fatal failure when executing message: %s", err)
 		}
 
 		// Assert that the receipt matches what the test vector expects.
-		receipt := vector.Post.Receipts[i]
-		if expected, actual := receipt.ExitCode, ret.ExitCode; expected != actual {
-			t.Errorf("exit code of msg %d did not match; expected: %s, got: %s", i, expected, actual)
-		}
-		if expected, actual := receipt.GasUsed, ret.GasUsed; expected != actual {
-			t.Errorf("gas used of msg %d did not match; expected: %d, got: %d", i, expected, actual)
-		}
-		if expected, actual := []byte(receipt.ReturnValue), ret.Return; !bytes.Equal(expected, actual) {
-			t.Errorf("return value of msg %d did not match; expected: %s, got: %s", i, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual))
-		}
+		assertMsgResult(t, vector.Post.Receipts[i], ret, strconv.Itoa(i))
 	}
 
 	// Once all messages are applied, assert that the final state root matches
 	// the expected postcondition root.
 	if root != vector.Post.StateTree.RootCID {
-		color.NoColor = false // enable colouring.
-
-		t.Errorf("wrong post root cid; expected %v, but got %v", vector.Post.StateTree.RootCID, root)
-
-		var (
-			a  = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state")
-			b  = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state")
-			c  = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state")
-			d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]")
-			d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]")
-			d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]")
-		)
-
-		bold := color.New(color.Bold).SprintfFunc()
-
-		// run state diffs.
-		t.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c))
-
-		t.Log(bold("--- %s left: %s; right: %s ---", d1, a, b))
-		t.Log(statediff.Diff(context.Background(), bs, vector.Post.StateTree.RootCID, root))
-
-		t.Log(bold("--- %s left: %s; right: %s ---", d2, c, b))
-		t.Log(statediff.Diff(context.Background(), bs, vector.Pre.StateTree.RootCID, root))
-
-		t.Log(bold("--- %s left: %s; right: %s ---", d3, c, a))
-		t.Log(statediff.Diff(context.Background(), bs, vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID))
+		dumpThreeWayStateDiff(t, vector, bs, root)
 	}
 }
+
+// executeTipsetVector executes a tipset-class test vector.
+func executeTipsetVector(t *testing.T, vector *schema.TestVector) {
+	var (
+		ctx       = context.Background()
+		prevEpoch = vector.Pre.Epoch
+		root      = vector.Pre.StateTree.RootCID
+		tmpds     = ds.NewMapDatastore()
+	)
+
+	// Load the CAR into a new temporary Blockstore.
+	bs := loadCAR(t, vector.CAR)
+
+	// Create a new Driver.
+	driver := NewDriver(ctx, vector.Selector)
+
+	// Apply every tipset.
+	var receiptsIdx int
+	for i, ts := range vector.ApplyTipsets {
+		ts := ts // capture
+		ret, err := driver.ExecuteTipset(bs, tmpds, root, prevEpoch, &ts)
+		if err != nil {
+			t.Fatalf("failed to apply tipset %d message: %s", i, err)
+		}
+
+		for j, v := range ret.AppliedResults {
+			assertMsgResult(t, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i))
+			receiptsIdx++
+		}
+
+		// Compare the receipts root.
+		if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual {
+			t.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual)
+		}
+
+		prevEpoch = ts.Epoch
+		root = ret.PostStateRoot
+	}
+
+	// Once all messages are applied, assert that the final state root matches
+	// the expected postcondition root.
+	if root != vector.Post.StateTree.RootCID {
+		dumpThreeWayStateDiff(t, vector, bs, root)
+	}
+}
+
+// assertMsgResult compares a message result. It takes the expected receipt
+// encoded in the vector, the actual receipt returned by Lotus, and a message
+// label to log in the assertion failure message to facilitate debugging.
+func assertMsgResult(t *testing.T, expected *schema.Receipt, actual *vm.ApplyRet, label string) {
+	t.Helper()
+
+	if expected, actual := expected.ExitCode, actual.ExitCode; expected != actual {
+		t.Errorf("exit code of msg %s did not match; expected: %s, got: %s", label, expected, actual)
+	}
+	if expected, actual := expected.GasUsed, actual.GasUsed; expected != actual {
+		t.Errorf("gas used of msg %s did not match; expected: %d, got: %d", label, expected, actual)
+	}
+	if expected, actual := []byte(expected.ReturnValue), actual.Return; !bytes.Equal(expected, actual) {
+		t.Errorf("return value of msg %s did not match; expected: %s, got: %s", label, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual))
+	}
+}
+
+func dumpThreeWayStateDiff(t *testing.T, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) {
+	color.NoColor = false // enable colouring.
+
+	t.Errorf("wrong post root cid; expected %v, but got %v", vector.Post.StateTree.RootCID, actual)
+
+	var (
+		a  = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state")
+		b  = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state")
+		c  = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state")
+		d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]")
+		d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]")
+		d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]")
+	)
+
+	bold := color.New(color.Bold).SprintfFunc()
+
+	// run state diffs.
+	t.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c))
+
+	t.Log(bold("--- %s left: %s; right: %s ---", d1, a, b))
+	t.Log(statediff.Diff(context.Background(), bs, vector.Post.StateTree.RootCID, actual))
+
+	t.Log(bold("--- %s left: %s; right: %s ---", d2, c, b))
+	t.Log(statediff.Diff(context.Background(), bs, vector.Pre.StateTree.RootCID, actual))
+
+	t.Log(bold("--- %s left: %s; right: %s ---", d3, c, a))
+	t.Log(statediff.Diff(context.Background(), bs, vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID))
+}
+
+func loadCAR(t *testing.T, vectorCAR schema.Base64EncodedBytes) blockstore.Blockstore {
+	bs := blockstore.NewTemporary()
+
+	// Read the base64-encoded CAR from the vector, and inflate the gzip.
+	buf := bytes.NewReader(vectorCAR)
+	r, err := gzip.NewReader(buf)
+	if err != nil {
+		t.Fatalf("failed to inflate gzipped CAR: %s", err)
+	}
+	defer r.Close() // nolint
+
+	// Load the CAR embedded in the test vector into the Blockstore.
+	_, err = car.LoadCar(bs, r)
+	if err != nil {
+		t.Fatalf("failed to load state tree car from test vector: %s", err)
+	}
+	return bs
+}

documentation/en/api-methods.md

@@ -268,6 +268,9 @@ blockchain, but that do not require any form of state computation.
 
 ### ChainExport
 ChainExport returns a stream of bytes with CAR dump of chain data.
+The exported chain data includes the header chain from the given tipset
+back to genesis, the entire genesis state, and the most recent 'nroots'
+state trees.
 
 
 Perms: read
@@ -275,6 +278,7 @@ Perms: read
 Inputs:
 ```json
 [
+  10101,
   [
     {
       "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"

documentation/en/api.md

@@ -26,7 +26,7 @@ The necessary permissions for each are in [api/struct.go](https://github.com/fil
 
 ## How do I make an API request?
 
-To demonstrate making an API request, we will take the method `ChainHead` from [api/api.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go).
+To demonstrate making an API request, we will take the method `ChainHead` from [api/api_full.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go).
 
 ```go
 ChainHead(context.Context) (*types.TipSet, error)

extern/sector-storage/ffiwrapper/sealer_cgo.go

@@ -371,7 +371,11 @@ func (sb *Sealer) ReadPiece(ctx context.Context, writer io.Writer, sector abi.Se
 	maxPieceSize := abi.PaddedPieceSize(sb.ssize)
 
 	pf, err := openPartialFile(maxPieceSize, path.Unsealed)
-	if xerrors.Is(err, os.ErrNotExist) {
+	if err != nil {
+		if xerrors.Is(err, os.ErrNotExist) {
+			return false, nil
+		}
+
 		return false, xerrors.Errorf("opening partial file: %w", err)
 	}
 

extern/sector-storage/sched.go

@@ -563,6 +563,7 @@ func (sh *scheduler) runWorker(wid WorkerID) {
 				return
 			}
 
+			sh.workersLk.RLock()
 			worker.wndLk.Lock()
 
 			windowsRequested -= sh.workerCompactWindows(worker, wid)
@@ -574,8 +575,6 @@ func (sh *scheduler) runWorker(wid WorkerID) {
 
 				// process tasks within a window, preferring tasks at lower indexes
 				for len(firstWindow.todo) > 0 {
-					sh.workersLk.RLock()
-
 					tidx := -1
 
 					worker.lk.Lock()
@@ -589,7 +588,6 @@ func (sh *scheduler) runWorker(wid WorkerID) {
 					worker.lk.Unlock()
 
 					if tidx == -1 {
-						sh.workersLk.RUnlock()
 						break assignLoop
 					}
 
@@ -597,7 +595,6 @@ func (sh *scheduler) runWorker(wid WorkerID) {
 
 					log.Debugf("assign worker sector %d", todo.sector.Number)
 					err := sh.assignWorker(taskDone, wid, worker, todo)
-					sh.workersLk.RUnlock()
 
 					if err != nil {
 						log.Error("assignWorker error: %+v", err)
@@ -618,6 +615,7 @@ func (sh *scheduler) runWorker(wid WorkerID) {
 			}
 
 			worker.wndLk.Unlock()
+			sh.workersLk.RUnlock()
 		}
 	}()
 }
@@ -776,14 +774,19 @@ func (sh *scheduler) dropWorker(wid WorkerID) {
 }
 
 func (sh *scheduler) workerCleanup(wid WorkerID, w *workerHandle) {
-	if !w.cleanupStarted {
+	select {
+	case <-w.closingMgr:
+	default:
 		close(w.closingMgr)
 	}
+
+	sh.workersLk.Unlock()
 	select {
 	case <-w.closedMgr:
 	case <-time.After(time.Second):
 		log.Errorf("timeout closing worker manager goroutine %d", wid)
 	}
+	sh.workersLk.Lock()
 
 	if !w.cleanupStarted {
 		w.cleanupStarted = true

go.mod

@@ -7,14 +7,12 @@ replace github.com/supranational/blst => github.com/supranational/blst v0.1.2-al
 require (
 	contrib.go.opencensus.io/exporter/jaeger v0.1.0
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0
-	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
 	github.com/BurntSushi/toml v0.3.1
 	github.com/GeertJohan/go.rice v1.0.0
 	github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129
 	github.com/coreos/go-systemd/v22 v22.0.0
-	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e
 	github.com/dgraph-io/badger/v2 v2.0.3
 	github.com/docker/go-units v0.4.0
@@ -30,7 +28,7 @@ require (
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
 	github.com/filecoin-project/go-data-transfer v0.6.3
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
-	github.com/filecoin-project/go-fil-markets v0.5.8
+	github.com/filecoin-project/go-fil-markets v0.5.9
 	github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52
 	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6
@@ -41,7 +39,7 @@ require (
 	github.com/filecoin-project/specs-actors v0.9.3
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200730063404-f7db367e9401
 	github.com/filecoin-project/statediff v0.0.1
-	github.com/filecoin-project/test-vectors v0.0.0-20200826113833-9ffe6524729d
+	github.com/filecoin-project/test-vectors v0.0.0-20200902131127-9806d09b005d
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
 	github.com/go-kit/kit v0.10.0
 	github.com/google/uuid v1.1.1
@@ -92,7 +90,7 @@ require (
 	github.com/libp2p/go-libp2p-mplex v0.2.4
 	github.com/libp2p/go-libp2p-noise v0.1.1
 	github.com/libp2p/go-libp2p-peerstore v0.2.6
-	github.com/libp2p/go-libp2p-pubsub v0.3.5
+	github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200901174250-06a12f17b7de
 	github.com/libp2p/go-libp2p-quic-transport v0.8.0
 	github.com/libp2p/go-libp2p-record v0.1.3
 	github.com/libp2p/go-libp2p-routing-helpers v0.2.3
@@ -119,7 +117,6 @@ require (
 	github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d
 	github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542
 	go.opencensus.io v0.22.4
-	go.uber.org/dig v1.10.0 // indirect
 	go.uber.org/fx v1.9.0
 	go.uber.org/multierr v1.5.0
 	go.uber.org/zap v1.15.0

go.sum

@@ -33,9 +33,8 @@ dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
+github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4=
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
-github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
-github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -163,9 +162,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
+github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f h1:BOaYiTvg8p9vBUXpklC22XSK/mifLF7lG9jtmYYi3Tc=
 github.com/davidlazar/go-crypto v0.0.0-20190912175916-7055855a373f/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
-github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU=
-github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U=
 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU=
 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk=
 github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
@@ -246,9 +244,10 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
 github.com/filecoin-project/go-fil-markets v0.5.6-0.20200814234959-80b1788108ac/go.mod h1:umicPCaN99ysHTiYOmwhuLxTFbOwcsI+mdw/t96vvM4=
-github.com/filecoin-project/go-fil-markets v0.5.6/go.mod h1:SJApXAKr5jyGpbzDEOhvemui0pih7hhT8r2MXJxCP1E=
 github.com/filecoin-project/go-fil-markets v0.5.8 h1:uwl0QNUVmmSlUQfxshpj21Dmhh6WKTQNhnb1GMfdp18=
 github.com/filecoin-project/go-fil-markets v0.5.8/go.mod h1:6ZX1vbZbnukbVQ8tCB/MmEizuW/bmRX7SpGAltU3KVg=
+github.com/filecoin-project/go-fil-markets v0.5.9 h1:iIO17UfIjUCiB37TRwgiBwAyfJJwHb8e8uAfu7F37gc=
+github.com/filecoin-project/go-fil-markets v0.5.9/go.mod h1:/cb1IoaiHhwFEWyIAPm9yN6Z+MiPujFZBT8BGH7LwB8=
 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200817153016-2ea5cbaf5ec0/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52 h1:FXtCp0ybqdQL9knb3OGDpkNTaBbPxgkqPeWKotUwkH0=
 github.com/filecoin-project/go-jsonrpc v0.1.2-0.20200822201400-474f4fdccc52/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4=
@@ -268,8 +267,8 @@ github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIi
 github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
-github.com/filecoin-project/lotus v0.4.3-0.20200819134055-b13681df3205/go.mod h1:rooripL/X8ixwUngDPzphAv/RKZXWBprbyxxDW0EJi0=
 github.com/filecoin-project/lotus v0.4.3-0.20200820203717-d1718369a182/go.mod h1:biFZPQ/YyQGfkHUmHMiaNf2hnD6zm1+OAXPQYQ61Zkg=
+github.com/filecoin-project/lotus v0.5.8-0.20200902130912-0962292f920e/go.mod h1:OkZ5aUqs+fFnJOq9243WJDsTa9c3/Ae67NIAwVhAB+0=
 github.com/filecoin-project/sector-storage v0.0.0-20200712023225-1d67dcfa3c15/go.mod h1:salgVdX7qeXFo/xaiEQE29J4pPkjn71T0kt0n+VDBzo=
 github.com/filecoin-project/sector-storage v0.0.0-20200730050024-3ee28c3b6d9a/go.mod h1:oOawOl9Yk+qeytLzzIryjI8iRbqo+qzS6EEeElP4PWA=
 github.com/filecoin-project/sector-storage v0.0.0-20200810171746-eac70842d8e0 h1:E1fZ27fhKK05bhZItfTwqr1i05vXnEZJznQFEYwEEUU=
@@ -691,6 +690,8 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.1.1-0.20190114141812-62fb9bc030d1 h1:qBCV/RLV02TSfQa7tFmxTihnG+u+7JXByOkhlkR5rmQ=
 github.com/jonboulle/clockwork v0.1.1-0.20190114141812-62fb9bc030d1/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
 github.com/jsimonetti/rtnetlink v0.0.0-20190830100107-3784a6c7c552/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw=
@@ -894,10 +895,9 @@ github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1
 github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk=
 github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q=
 github.com/libp2p/go-libp2p-pubsub v0.3.2-0.20200527132641-c0712c6e92cf/go.mod h1:TxPOBuo1FPdsTjFnv+FGZbNbWYsp74Culx+4ViQpato=
-github.com/libp2p/go-libp2p-pubsub v0.3.4/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI=
 github.com/libp2p/go-libp2p-pubsub v0.3.5-0.20200820194335-bfc96c2cd081/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI=
-github.com/libp2p/go-libp2p-pubsub v0.3.5 h1:iF75GWpcxKEUQU8tTkgLy69qIQvfhL+t6U6ndQrB6ho=
-github.com/libp2p/go-libp2p-pubsub v0.3.5/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI=
+github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200901174250-06a12f17b7de h1:Dl0B0x6u+OSKXAa1DeB6xHFsUOBAhjrXJ10zykVSN6Q=
+github.com/libp2p/go-libp2p-pubsub v0.3.6-0.20200901174250-06a12f17b7de/go.mod h1:DTMSVmZZfXodB/pvdTGrY2eHPZ9W2ev7hzTH83OKHrI=
 github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU=
 github.com/libp2p/go-libp2p-quic-transport v0.5.0/go.mod h1:IEcuC5MLxvZ5KuHKjRu+dr3LjCT1Be3rcD/4d8JrX8M=
 github.com/libp2p/go-libp2p-quic-transport v0.7.1/go.mod h1:TD31to4E5exogR/GWHClXCfkktigjAl5rXSt7HoxNvY=
@@ -1497,9 +1497,8 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/dig v1.8.0 h1:1rR6hnL/bu1EVcjnRDN5kx1vbIjEJDTGhSQ2B3ddpcI=
 go.uber.org/dig v1.8.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw=
-go.uber.org/dig v1.10.0 h1:yLmDDj9/zuDjv3gz8GQGviXMs9TfysIUMUilCpgzUJY=
-go.uber.org/dig v1.10.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw=
 go.uber.org/fx v1.9.0 h1:7OAz8ucp35AU8eydejpYG7QrbE8rLKzGhHbZlJi5LYY=
 go.uber.org/fx v1.9.0/go.mod h1:mFdUyAUuJ3w4jAckiKSKbldsxy1ojpAMJ+dVZg5Y0Aw=
 go.uber.org/goleak v1.0.0 h1:qsup4IcBdlmsnGfqyLl4Ntn3C2XCCuKAE7DwHpScyUo=
@@ -1551,7 +1550,6 @@ golang.org/x/crypto v0.0.0-20200317142112-1b76d66859c6/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

node/impl/full/chain.go

@@ -494,7 +494,7 @@ func (a *ChainAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Mess
 	return cm.VMMessage(), nil
 }
 
-func (a *ChainAPI) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan []byte, error) {
+func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) {
 	ts, err := a.Chain.GetTipSetFromKey(tsk)
 	if err != nil {
 		return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
@@ -503,7 +503,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan
 	out := make(chan []byte)
 	go func() {
 		defer w.Close() //nolint:errcheck // it is a pipe
-		if err := a.Chain.Export(ctx, ts, w); err != nil {
+		if err := a.Chain.Export(ctx, ts, nroots, w); err != nil {
 			log.Errorf("chain export call failed: %s", err)
 			return
 		}

node/impl/full/gas.go

@@ -59,6 +59,10 @@ func (a *GasAPI) GasEstimateFeeCap(ctx context.Context, msg *types.Message, maxq
 		out = types.BigDiv(maxAccepted, gasLimitBig)
 	}
 
+	if msg.GasPremium != types.EmptyInt {
+		out = types.BigAdd(out, msg.GasPremium)
+	}
+
 	return out, nil
 }
 
@@ -214,7 +218,7 @@ func (a *GasAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message,
 		if err != nil {
 			return nil, xerrors.Errorf("estimating fee cap: %w", err)
 		}
-		msg.GasFeeCap = big.Add(feeCap, msg.GasPremium)
+		msg.GasFeeCap = feeCap
 	}
 
 	capGasFee(msg, spec.Get().MaxFee)

node/repo/fsrepo_ds.go

@@ -17,7 +17,7 @@ import (
 type dsCtor func(path string) (datastore.Batching, error)
 
 var fsDatastores = map[string]dsCtor{
-	"chain":    badgerDs,
+	"chain":    chainBadgerDs,
 	"metadata": levelDs,
 
 	// Those need to be fast for large writes... but also need a really good GC :c
@@ -26,6 +26,16 @@ var fsDatastores = map[string]dsCtor{
 	"client": badgerDs, // client specific
 }
 
+func chainBadgerDs(path string) (datastore.Batching, error) {
+	opts := badger.DefaultOptions
+	opts.GcInterval = 0 // disable GC for chain datastore
+
+	opts.Options = dgbadger.DefaultOptions("").WithTruncate(true).
+		WithValueThreshold(1 << 10)
+
+	return badger.NewDatastore(path, &opts)
+}
+
 func badgerDs(path string) (datastore.Batching, error) {
 	opts := badger.DefaultOptions
 	opts.Options = dgbadger.DefaultOptions("").WithTruncate(true).