Merge remote-tracking branch 'origin/master' into feat/async-restartable-workers

.github/CODEOWNERS

@@ -13,3 +13,4 @@
 ### Conformance testing.
 conformance/            @raulk
 extern/test-vectors     @raulk
+cmd/tvx                 @raulk

README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <a href="https://lotu.sh/" title="Lotus Docs">
+  <a href="https://docs.filecoin.io/" title="Filecoin Docs">
     <img src="documentation/images/lotus_logo_h.png" alt="Project Lotus Logo" width="244" />
   </a>
 </p>
@@ -18,7 +18,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more
 
 ## Building & Documentation
 
-For instructions on how to build lotus from source, please visit [Lotus build and setup instruction](https://docs.filecoin.io/get-started/lotus/installation/#minimal-requirements) or read the source [here](https://github.com/filecoin-project/lotus/tree/master/documentation).
+For instructions on how to build, install and setup lotus, please visit [https://docs.filecoin.io/get-started/lotus](https://docs.filecoin.io/get-started/lotus/).
 
 ## Reporting a Vulnerability
 

api/api_full.go

@@ -176,6 +176,9 @@ type FullNode interface {
 	// the reason.
 	SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error)
 
+	// SyncValidateTipset indicates whether the provided tipset is valid or not
+	SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error)
+
 	// MethodGroup: Mpool
 	// The Mpool methods are for interacting with the message pool. The message pool
 	// manages all incoming and outgoing 'messages' going over the network.
@@ -244,6 +247,8 @@ type FullNode interface {
 	WalletImport(context.Context, *types.KeyInfo) (address.Address, error)
 	// WalletDelete deletes an address from the wallet.
 	WalletDelete(context.Context, address.Address) error
+	// WalletValidateAddress validates whether a given string can be decoded as a well-formed address
+	WalletValidateAddress(context.Context, string) (address.Address, error)
 
 	// Other
 

api/apistruct/struct.go

@@ -112,6 +112,7 @@ type FullNodeStruct struct {
 		SyncMarkBad        func(ctx context.Context, bcid cid.Cid) error                `perm:"admin"`
 		SyncUnmarkBad      func(ctx context.Context, bcid cid.Cid) error                `perm:"admin"`
 		SyncCheckBad       func(ctx context.Context, bcid cid.Cid) (string, error)      `perm:"read"`
+		SyncValidateTipset func(ctx context.Context, tsk types.TipSetKey) (bool, error) `perm:"read"`
 
 		MpoolGetConfig func(context.Context) (*types.MpoolConfig, error) `perm:"read"`
 		MpoolSetConfig func(context.Context, *types.MpoolConfig) error   `perm:"write"`
@@ -141,6 +142,7 @@ type FullNodeStruct struct {
 		WalletExport          func(context.Context, address.Address) (*types.KeyInfo, error)                       `perm:"admin"`
 		WalletImport          func(context.Context, *types.KeyInfo) (address.Address, error)                       `perm:"admin"`
 		WalletDelete          func(context.Context, address.Address) error                                         `perm:"write"`
+		WalletValidateAddress func(context.Context, string) (address.Address, error)                               `perm:"read"`
 
 		ClientImport                              func(ctx context.Context, ref api.FileRef) (*api.ImportRes, error)                                                `perm:"admin"`
 		ClientListImports                         func(ctx context.Context) ([]api.Import, error)                                                                   `perm:"write"`
@@ -642,6 +644,10 @@ func (c *FullNodeStruct) WalletDelete(ctx context.Context, addr address.Address)
 	return c.Internal.WalletDelete(ctx, addr)
 }
 
+func (c *FullNodeStruct) WalletValidateAddress(ctx context.Context, str string) (address.Address, error) {
+	return c.Internal.WalletValidateAddress(ctx, str)
+}
+
 func (c *FullNodeStruct) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) {
 	return c.Internal.MpoolGetNonce(ctx, addr)
 }
@@ -746,6 +752,10 @@ func (c *FullNodeStruct) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string
 	return c.Internal.SyncCheckBad(ctx, bcid)
 }
 
+func (c *FullNodeStruct) SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) {
+	return c.Internal.SyncValidateTipset(ctx, tsk)
+}
+
 func (c *FullNodeStruct) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) {
 	return c.Internal.StateNetworkName(ctx)
 }

api/test/window_post.go

@@ -153,18 +153,16 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 	require.NoError(t, err)
 
 	fmt.Printf("Running one proving period\n")
+	fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2)
 
 	for {
 		head, err := client.ChainHead(ctx)
 		require.NoError(t, err)
 
 		if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 {
+			fmt.Printf("Now head.Height = %d\n", head.Height())
 			break
 		}
-
-		if head.Height()%100 == 0 {
-			fmt.Printf("@%d\n", head.Height())
-		}
 		build.Clock.Sleep(blocktime)
 	}
 
@@ -186,7 +184,6 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 		require.Greater(t, len(parts), 0)
 
 		secs := parts[0].AllSectors
-		require.NoError(t, err)
 		n, err := secs.Count()
 		require.NoError(t, err)
 		require.Equal(t, uint64(2), n)
@@ -210,7 +207,6 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 		require.Greater(t, len(parts), 0)
 
 		secs := parts[0].AllSectors
-		require.NoError(t, err)
 		n, err := secs.Count()
 		require.NoError(t, err)
 		require.Equal(t, uint64(2), n)
@@ -236,18 +232,17 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 	require.NoError(t, err)
 
 	fmt.Printf("Go through another PP, wait for sectors to become faulty\n")
+	fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2)
 
 	for {
 		head, err := client.ChainHead(ctx)
 		require.NoError(t, err)
 
 		if head.Height() > di.PeriodStart+(di.WPoStProvingPeriod)+2 {
+			fmt.Printf("Now head.Height = %d\n", head.Height())
 			break
 		}
 
-		if head.Height()%100 == 0 {
-			fmt.Printf("@%d\n", head.Height())
-		}
 		build.Clock.Sleep(blocktime)
 	}
 
@@ -267,17 +262,17 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 	di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
 	require.NoError(t, err)
 
+	fmt.Printf("End for head.Height > %d\n", di.PeriodStart+di.WPoStProvingPeriod+2)
+
 	for {
 		head, err := client.ChainHead(ctx)
 		require.NoError(t, err)
 
 		if head.Height() > di.PeriodStart+di.WPoStProvingPeriod+2 {
+			fmt.Printf("Now head.Height = %d\n", head.Height())
 			break
 		}
 
-		if head.Height()%100 == 0 {
-			fmt.Printf("@%d\n", head.Height())
-		}
 		build.Clock.Sleep(blocktime)
 	}
 
@@ -300,12 +295,14 @@ func TestWindowPost(t *testing.T, b APIBuilder, blocktime time.Duration, nSector
 		require.NoError(t, err)
 
 		waitUntil := head.Height() + 10
+		fmt.Printf("End for head.Height > %d\n", waitUntil)
 
 		for {
 			head, err := client.ChainHead(ctx)
 			require.NoError(t, err)
 
 			if head.Height() > waitUntil {
+				fmt.Printf("Now head.Height = %d\n", head.Height())
 				break
 			}
 		}

chain/actors/builtin/miner/miner.go

@@ -20,6 +20,10 @@ import (
 
 // Unchanged between v0 and v1 actors
 var WPoStProvingPeriod = miner0.WPoStProvingPeriod
+var WPoStPeriodDeadlines = miner0.WPoStPeriodDeadlines
+var WPoStChallengeWindow = miner0.WPoStChallengeWindow
+var WPoStChallengeLookback = miner0.WPoStChallengeLookback
+var FaultDeclarationCutoff = miner0.FaultDeclarationCutoff
 
 const MinSectorExpiration = miner0.MinSectorExpiration
 

chain/messagepool/messagepool.go

@@ -75,8 +75,6 @@ var (
 	ErrRBFTooLowPremium       = errors.New("replace by fee has too low GasPremium")
 	ErrTooManyPendingMessages = errors.New("too many pending messages for actor")
 	ErrNonceGap               = errors.New("unfulfilled nonce gap")
-
-	ErrTryAgain = errors.New("state inconsistency while pushing message; please try again")
 )
 
 const (
@@ -795,98 +793,6 @@ func (mp *MessagePool) getStateBalance(addr address.Address, ts *types.TipSet) (
 	return act.Balance, nil
 }
 
-func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address, cb func(address.Address, uint64) (*types.SignedMessage, error)) (*types.SignedMessage, error) {
-	// serialize push access to reduce lock contention
-	mp.addSema <- struct{}{}
-	defer func() {
-		<-mp.addSema
-	}()
-
-	mp.curTsLk.Lock()
-	mp.lk.Lock()
-
-	curTs := mp.curTs
-
-	fromKey := addr
-	if fromKey.Protocol() == address.ID {
-		var err error
-		fromKey, err = mp.api.StateAccountKey(ctx, fromKey, mp.curTs)
-		if err != nil {
-			mp.lk.Unlock()
-			mp.curTsLk.Unlock()
-			return nil, xerrors.Errorf("resolving sender key: %w", err)
-		}
-	}
-
-	nonce, err := mp.getNonceLocked(fromKey, mp.curTs)
-	if err != nil {
-		mp.lk.Unlock()
-		mp.curTsLk.Unlock()
-		return nil, xerrors.Errorf("get nonce locked failed: %w", err)
-	}
-
-	// release the locks for signing
-	mp.lk.Unlock()
-	mp.curTsLk.Unlock()
-
-	msg, err := cb(fromKey, nonce)
-	if err != nil {
-		return nil, err
-	}
-
-	err = mp.checkMessage(msg)
-	if err != nil {
-		return nil, err
-	}
-
-	msgb, err := msg.Serialize()
-	if err != nil {
-		return nil, err
-	}
-
-	// reacquire the locks and check state for consistency
-	mp.curTsLk.Lock()
-	defer mp.curTsLk.Unlock()
-
-	if mp.curTs != curTs {
-		return nil, ErrTryAgain
-	}
-
-	mp.lk.Lock()
-	defer mp.lk.Unlock()
-
-	nonce2, err := mp.getNonceLocked(fromKey, mp.curTs)
-	if err != nil {
-		return nil, xerrors.Errorf("get nonce locked failed: %w", err)
-	}
-
-	if nonce2 != nonce {
-		return nil, ErrTryAgain
-	}
-
-	publish, err := mp.verifyMsgBeforeAdd(msg, curTs, true)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := mp.checkBalance(msg, curTs); err != nil {
-		return nil, err
-	}
-
-	if err := mp.addLocked(msg, false); err != nil {
-		return nil, xerrors.Errorf("add locked failed: %w", err)
-	}
-	if err := mp.addLocal(msg, msgb); err != nil {
-		log.Errorf("addLocal failed: %+v", err)
-	}
-
-	if publish {
-		err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb)
-	}
-
-	return msg, err
-}
-
 func (mp *MessagePool) Remove(from address.Address, nonce uint64, applied bool) {
 	mp.lk.Lock()
 	defer mp.lk.Unlock()

chain/messagesigner/messagesigner.go

@@ -0,0 +1,124 @@
+package messagesigner
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/chain/messagepool"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore/namespace"
+	logging "github.com/ipfs/go-log/v2"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"golang.org/x/xerrors"
+)
+
+const dsKeyActorNonce = "ActorNonce"
+
+var log = logging.Logger("messagesigner")
+
+type mpoolAPI interface {
+	GetNonce(address.Address) (uint64, error)
+}
+
+// MessageSigner keeps track of nonces per address, and increments the nonce
+// when signing a message
+type MessageSigner struct {
+	wallet *wallet.Wallet
+	mpool  mpoolAPI
+	ds     datastore.Batching
+}
+
+func NewMessageSigner(wallet *wallet.Wallet, mpool *messagepool.MessagePool, ds dtypes.MetadataDS) *MessageSigner {
+	return newMessageSigner(wallet, mpool, ds)
+}
+
+func newMessageSigner(wallet *wallet.Wallet, mpool mpoolAPI, ds dtypes.MetadataDS) *MessageSigner {
+	ds = namespace.Wrap(ds, datastore.NewKey("/message-signer/"))
+	return &MessageSigner{
+		wallet: wallet,
+		mpool:  mpool,
+		ds:     ds,
+	}
+}
+
+// SignMessage increments the nonce for the message From address, and signs
+// the message
+func (ms *MessageSigner) SignMessage(ctx context.Context, msg *types.Message) (*types.SignedMessage, error) {
+	nonce, err := ms.nextNonce(msg.From)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to create nonce: %w", err)
+	}
+
+	msg.Nonce = nonce
+	sig, err := ms.wallet.Sign(ctx, msg.From, msg.Cid().Bytes())
+	if err != nil {
+		return nil, xerrors.Errorf("failed to sign message: %w", err)
+	}
+
+	return &types.SignedMessage{
+		Message:   *msg,
+		Signature: *sig,
+	}, nil
+}
+
+// nextNonce increments the nonce.
+// If there is no nonce in the datastore, gets the nonce from the message pool.
+func (ms *MessageSigner) nextNonce(addr address.Address) (uint64, error) {
+	// Nonces used to be created by the mempool and we need to support nodes
+	// that have mempool nonces, so first check the mempool for a nonce for
+	// this address. Note that the mempool returns the actor state's nonce
+	// by default.
+	nonce, err := ms.mpool.GetNonce(addr)
+	if err != nil {
+		return 0, xerrors.Errorf("failed to get nonce from mempool: %w", err)
+	}
+
+	// Get the nonce for this address from the datastore
+	addrNonceKey := datastore.KeyWithNamespaces([]string{dsKeyActorNonce, addr.String()})
+	dsNonceBytes, err := ms.ds.Get(addrNonceKey)
+
+	switch {
+	case xerrors.Is(err, datastore.ErrNotFound):
+		// If a nonce for this address hasn't yet been created in the
+		// datastore, just use the nonce from the mempool
+
+	case err != nil:
+		return 0, xerrors.Errorf("failed to get nonce from datastore: %w", err)
+
+	default:
+		// There is a nonce in the datastore, so unmarshall and increment it
+		maj, val, err := cbg.CborReadHeader(bytes.NewReader(dsNonceBytes))
+		if err != nil {
+			return 0, xerrors.Errorf("failed to parse nonce from datastore: %w", err)
+		}
+		if maj != cbg.MajUnsignedInt {
+			return 0, xerrors.Errorf("bad cbor type parsing nonce from datastore")
+		}
+
+		dsNonce := val + 1
+
+		// The message pool nonce should be <= than the datastore nonce
+		if nonce <= dsNonce {
+			nonce = dsNonce
+		} else {
+			log.Warnf("mempool nonce was larger than datastore nonce (%d > %d)", nonce, dsNonce)
+		}
+	}
+
+	// Write the nonce for this address to the datastore
+	buf := bytes.Buffer{}
+	_, err = buf.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, nonce))
+	if err != nil {
+		return 0, xerrors.Errorf("failed to marshall nonce: %w", err)
+	}
+	err = ms.ds.Put(addrNonceKey, buf.Bytes())
+	if err != nil {
+		return 0, xerrors.Errorf("failed to write nonce to datastore: %w", err)
+	}
+
+	return nonce, nil
+}

chain/messagesigner/messagesigner_test.go

@@ -0,0 +1,158 @@
+package messagesigner
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"github.com/filecoin-project/lotus/chain/wallet"
+
+	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/stretchr/testify/require"
+
+	ds_sync "github.com/ipfs/go-datastore/sync"
+
+	"github.com/filecoin-project/go-address"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/ipfs/go-datastore"
+)
+
+type mockMpool struct {
+	lk     sync.RWMutex
+	nonces map[address.Address]uint64
+}
+
+func newMockMpool() *mockMpool {
+	return &mockMpool{nonces: make(map[address.Address]uint64)}
+}
+
+func (mp *mockMpool) setNonce(addr address.Address, nonce uint64) {
+	mp.lk.Lock()
+	defer mp.lk.Unlock()
+
+	mp.nonces[addr] = nonce
+}
+
+func (mp *mockMpool) GetNonce(addr address.Address) (uint64, error) {
+	mp.lk.RLock()
+	defer mp.lk.RUnlock()
+
+	return mp.nonces[addr], nil
+}
+
+func TestMessageSignerSignMessage(t *testing.T) {
+	ctx := context.Background()
+
+	w, _ := wallet.NewWallet(wallet.NewMemKeyStore())
+	from1, err := w.GenerateKey(crypto.SigTypeSecp256k1)
+	require.NoError(t, err)
+	from2, err := w.GenerateKey(crypto.SigTypeSecp256k1)
+	require.NoError(t, err)
+	to1, err := w.GenerateKey(crypto.SigTypeSecp256k1)
+	require.NoError(t, err)
+	to2, err := w.GenerateKey(crypto.SigTypeSecp256k1)
+	require.NoError(t, err)
+
+	type msgSpec struct {
+		msg        *types.Message
+		mpoolNonce [1]uint64
+		expNonce   uint64
+	}
+	tests := []struct {
+		name string
+		msgs []msgSpec
+	}{{
+		// No nonce yet in datastore
+		name: "no nonce yet",
+		msgs: []msgSpec{{
+			msg: &types.Message{
+				To:   to1,
+				From: from1,
+			},
+			expNonce: 0,
+		}},
+	}, {
+		// Get nonce value of zero from mpool
+		name: "mpool nonce zero",
+		msgs: []msgSpec{{
+			msg: &types.Message{
+				To:   to1,
+				From: from1,
+			},
+			mpoolNonce: [1]uint64{0},
+			expNonce:   0,
+		}},
+	}, {
+		// Get non-zero nonce value from mpool
+		name: "mpool nonce set",
+		msgs: []msgSpec{{
+			msg: &types.Message{
+				To:   to1,
+				From: from1,
+			},
+			mpoolNonce: [1]uint64{5},
+			expNonce:   5,
+		}, {
+			msg: &types.Message{
+				To:   to1,
+				From: from1,
+			},
+			// Should adjust datastore nonce because mpool nonce is higher
+			mpoolNonce: [1]uint64{10},
+			expNonce:   10,
+		}},
+	}, {
+		// Nonce should increment independently for each address
+		name: "nonce increments per address",
+		msgs: []msgSpec{{
+			msg: &types.Message{
+				To:   to1,
+				From: from1,
+			},
+			expNonce: 0,
+		}, {
+			msg: &types.Message{
+				To:   to1,
+				From: from1,
+			},
+			expNonce: 1,
+		}, {
+			msg: &types.Message{
+				To:   to2,
+				From: from2,
+			},
+			mpoolNonce: [1]uint64{5},
+			expNonce:   5,
+		}, {
+			msg: &types.Message{
+				To:   to2,
+				From: from2,
+			},
+			expNonce: 6,
+		}, {
+			msg: &types.Message{
+				To:   to1,
+				From: from1,
+			},
+			expNonce: 2,
+		}},
+	}}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			mpool := newMockMpool()
+			ds := ds_sync.MutexWrap(datastore.NewMapDatastore())
+			ms := newMessageSigner(w, mpool, ds)
+
+			for _, m := range tt.msgs {
+				if len(m.mpoolNonce) == 1 {
+					mpool.setNonce(m.msg.From, m.mpoolNonce[0])
+				}
+				smsg, err := ms.SignMessage(ctx, m.msg)
+				require.NoError(t, err)
+				require.Equal(t, m.expNonce, smsg.Message.Nonce)
+			}
+		})
+	}
+}

chain/store/store.go

@@ -10,6 +10,8 @@ import (
 	"strconv"
 	"sync"
 
+	"golang.org/x/sync/errgroup"
+
 	"github.com/filecoin-project/go-state-types/crypto"
 	"github.com/minio/blake2b-simd"
 
@@ -284,6 +286,16 @@ func (cs *ChainStore) MarkBlockAsValidated(ctx context.Context, blkid cid.Cid) e
 	return nil
 }
 
+func (cs *ChainStore) UnmarkBlockAsValidated(ctx context.Context, blkid cid.Cid) error {
+	key := blockValidationCacheKeyPrefix.Instance(blkid.String())
+
+	if err := cs.ds.Delete(key); err != nil {
+		return xerrors.Errorf("removing from valid block cache: %w", err)
+	}
+
+	return nil
+}
+
 func (cs *ChainStore) SetGenesis(b *types.BlockHeader) error {
 	ts, err := types.NewTipSet([]*types.BlockHeader{b})
 	if err != nil {
@@ -467,14 +479,25 @@ func (cs *ChainStore) LoadTipSet(tsk types.TipSetKey) (*types.TipSet, error) {
 		return v.(*types.TipSet), nil
 	}
 
-	var blks []*types.BlockHeader
+	// Fetch tipset block headers from blockstore in parallel
-	for _, c := range tsk.Cids() {
+	var eg errgroup.Group
+	cids := tsk.Cids()
+	blks := make([]*types.BlockHeader, len(cids))
+	for i, c := range cids {
+		i, c := i, c
+		eg.Go(func() error {
 			b, err := cs.GetBlock(c)
 			if err != nil {
-			return nil, xerrors.Errorf("get block %s: %w", c, err)
+				return xerrors.Errorf("get block %s: %w", c, err)
 			}
 
-		blks = append(blks, b)
+			blks[i] = b
+			return nil
+		})
+	}
+	err := eg.Wait()
+	if err != nil {
+		return nil, err
 	}
 
 	ts, err := types.NewTipSet(blks)

chain/sync.go

@@ -597,7 +597,7 @@ func isPermanent(err error) bool {
 	return !errors.Is(err, ErrTemporal)
 }
 
-func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet) error {
+func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet, useCache bool) error {
 	ctx, span := trace.StartSpan(ctx, "validateTipSet")
 	defer span.End()
 
@@ -613,7 +613,7 @@ func (syncer *Syncer) ValidateTipSet(ctx context.Context, fts *store.FullTipSet)
 		b := b // rebind to a scoped variable
 
 		futures = append(futures, async.Err(func() error {
-			if err := syncer.ValidateBlock(ctx, b); err != nil {
+			if err := syncer.ValidateBlock(ctx, b, useCache); err != nil {
 				if isPermanent(err) {
 					syncer.bad.Add(b.Cid(), NewBadBlockReason([]cid.Cid{b.Cid()}, err.Error()))
 				}
@@ -680,7 +680,7 @@ func blockSanityChecks(h *types.BlockHeader) error {
 }
 
 // ValidateBlock should match up with 'Semantical Validation' in validation.md in the spec
-func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (err error) {
+func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock, useCache bool) (err error) {
 	defer func() {
 		// b.Cid() could panic for empty blocks that are used in tests.
 		if rerr := recover(); rerr != nil {
@@ -689,6 +689,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
 		}
 	}()
 
+	if useCache {
 		isValidated, err := syncer.store.IsBlockValidated(ctx, b.Cid())
 		if err != nil {
 			return xerrors.Errorf("check block validation cache %s: %w", b.Cid(), err)
@@ -697,6 +698,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
 		if isValidated {
 			return nil
 		}
+	}
 
 	validationStart := build.Clock.Now()
 	defer func() {
@@ -782,7 +784,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
 			b.Header.ParentWeight, pweight)
 	}
 
-	// Stuff that needs stateroot / worker address
+	stateRootCheck := async.Err(func() error {
 		stateroot, precp, err := syncer.sm.TipSetState(ctx, baseTs)
 		if err != nil {
 			return xerrors.Errorf("get tipsetstate(%d, %s) failed: %w", h.Height, h.Parents, err)
@@ -807,6 +809,10 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
 			return xerrors.Errorf("parent receipts root did not match computed value (%s != %s)", precp, h.ParentMessageReceipts)
 		}
 
+		return nil
+	})
+
+	// Stuff that needs worker address
 	waddr, err := stmgr.GetMinerWorkerRaw(ctx, syncer.sm, lbst, h.Miner)
 	if err != nil {
 		return xerrors.Errorf("GetMinerWorkerRaw failed: %w", err)
@@ -927,6 +933,7 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
 		winnerCheck,
 		msgsCheck,
 		baseFeeCheck,
+		stateRootCheck,
 	}
 
 	var merr error
@@ -954,9 +961,11 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) (er
 		return mulErr
 	}
 
+	if useCache {
 		if err := syncer.store.MarkBlockAsValidated(ctx, b.Cid()); err != nil {
 			return xerrors.Errorf("caching block validation %s: %w", b.Cid(), err)
 		}
+	}
 
 	return nil
 }
@@ -1457,7 +1466,7 @@ func (syncer *Syncer) syncMessagesAndCheckState(ctx context.Context, headers []*
 
 	return syncer.iterFullTipsets(ctx, headers, func(ctx context.Context, fts *store.FullTipSet) error {
 		log.Debugw("validating tipset", "height", fts.TipSet().Height(), "size", len(fts.TipSet().Cids()))
-		if err := syncer.ValidateTipSet(ctx, fts); err != nil {
+		if err := syncer.ValidateTipSet(ctx, fts, true); err != nil {
 			log.Errorf("failed to validate tipset: %+v", err)
 			return xerrors.Errorf("message processing failed: %w", err)
 		}

chain/sync_test.go

@@ -732,7 +732,7 @@ func TestSyncInputs(t *testing.T) {
 
 	err := s.ValidateBlock(context.TODO(), &types.FullBlock{
 		Header: &types.BlockHeader{},
-	})
+	}, false)
 	if err == nil {
 		t.Fatal("should error on empty block")
 	}
@@ -741,7 +741,7 @@ func TestSyncInputs(t *testing.T) {
 
 	h.ElectionProof = nil
 
-	err = s.ValidateBlock(context.TODO(), &types.FullBlock{Header: h})
+	err = s.ValidateBlock(context.TODO(), &types.FullBlock{Header: h}, false)
 	if err == nil {
 		t.Fatal("should error on block with nil election proof")
 	}

chain/types/tipset.go

@@ -167,12 +167,16 @@ func (ts *TipSet) Equals(ots *TipSet) bool {
 		return false
 	}
 
-	if len(ts.blks) != len(ots.blks) {
+	if ts.height != ots.height {
 		return false
 	}
 
-	for i, b := range ts.blks {
+	if len(ts.cids) != len(ots.cids) {
-		if b.Cid() != ots.blks[i].Cid() {
+		return false
+	}
+
+	for i, cid := range ts.cids {
+		if cid != ots.cids[i] {
 			return false
 		}
 	}

chain/vm/runtime.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"encoding/binary"
 	"fmt"
-	gruntime "runtime"
 	"time"
 
 	"github.com/filecoin-project/go-address"
@@ -460,9 +459,11 @@ func (rt *Runtime) stateCommit(oldh, newh cid.Cid) aerrors.ActorError {
 }
 
 func (rt *Runtime) finilizeGasTracing() {
+	if enableTracing {
 		if rt.lastGasCharge != nil {
 			rt.lastGasCharge.TimeTaken = time.Since(rt.lastGasChargeTime)
 		}
+	}
 }
 
 // ChargeGas is spec actors function
@@ -490,10 +491,14 @@ func (rt *Runtime) chargeGasFunc(skip int) func(GasCharge) {
 
 }
 
+var enableTracing = false
+
 func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError {
 	toUse := gas.Total()
+	if enableTracing {
 		var callers [10]uintptr
-	cout := gruntime.Callers(2+skip, callers[:])
+
+		cout := 0 //gruntime.Callers(2+skip, callers[:])
 
 		now := build.Clock.Now()
 		if rt.lastGasCharge != nil {
@@ -517,6 +522,7 @@ func (rt *Runtime) chargeGasInternal(gas GasCharge, skip int) aerrors.ActorError
 		rt.executionTrace.GasCharges = append(rt.executionTrace.GasCharges, &gasTrace)
 		rt.lastGasChargeTime = now
 		rt.lastGasCharge = &gasTrace
+	}
 
 	// overflow safe
 	if rt.gasUsed > rt.gasAvailable-toUse {

chain/vm/vm.go

@@ -227,16 +227,23 @@ func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime,
 	}
 
 	rt := vm.makeRuntime(ctx, msg, origin, on, gasUsed, nac)
+	if enableTracing {
 		rt.lastGasChargeTime = start
 		if parent != nil {
 			rt.lastGasChargeTime = parent.lastGasChargeTime
 			rt.lastGasCharge = parent.lastGasCharge
 			defer func() {
-			parent.gasUsed = rt.gasUsed
 				parent.lastGasChargeTime = rt.lastGasChargeTime
 				parent.lastGasCharge = rt.lastGasCharge
 			}()
 		}
+	}
+
+	if parent != nil {
+		defer func() {
+			parent.gasUsed = rt.gasUsed
+		}()
+	}
 	if gasCharge != nil {
 		if err := rt.chargeGasSafe(*gasCharge); err != nil {
 			// this should never happen

cli/wallet.go

@@ -9,13 +9,16 @@ import (
 	"os"
 	"strings"
 
-	"github.com/filecoin-project/go-address"
+	"github.com/urfave/cli/v2"
-	"github.com/filecoin-project/go-state-types/crypto"
-	types "github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/wallet"
 	"golang.org/x/xerrors"
 
-	"github.com/urfave/cli/v2"
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/crypto"
+
+	types "github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/wallet"
+	"github.com/filecoin-project/lotus/lib/tablewriter"
 )
 
 var walletCmd = &cli.Command{
@@ -66,6 +69,13 @@ var walletNew = &cli.Command{
 var walletList = &cli.Command{
 	Name:  "list",
 	Usage: "List wallet address",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:    "addr-only",
+			Usage:   "Only print addresses",
+			Aliases: []string{"a"},
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
@@ -79,9 +89,52 @@ var walletList = &cli.Command{
 			return err
 		}
 
+		// Assume an error means no default key is set
+		def, _ := api.WalletDefaultAddress(ctx)
+
+		tw := tablewriter.New(
+			tablewriter.Col("Address"),
+			tablewriter.Col("Balance"),
+			tablewriter.Col("Nonce"),
+			tablewriter.Col("Default"),
+			tablewriter.NewLineCol("Error"))
+
 		for _, addr := range addrs {
+			if cctx.Bool("addr-only") {
 				fmt.Println(addr.String())
+			} else {
+				a, err := api.StateGetActor(ctx, addr, types.EmptyTSK)
+				if err != nil {
+					if !strings.Contains(err.Error(), "actor not found") {
+						tw.Write(map[string]interface{}{
+							"Address": addr,
+							"Error":   err,
+						})
+						continue
 					}
+
+					a = &types.Actor{
+						Balance: big.Zero(),
+					}
+				}
+
+				row := map[string]interface{}{
+					"Address": addr,
+					"Balance": types.FIL(a.Balance),
+					"Nonce":   a.Nonce,
+				}
+				if addr == def {
+					row["Default"] = "X"
+				}
+
+				tw.Write(row)
+			}
+		}
+
+		if !cctx.Bool("addr-only") {
+			return tw.Flush(os.Stdout)
+		}
+
 		return nil
 	},
 }

cmd/lotus-bench/caching_verifier.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
+	"github.com/filecoin-project/specs-actors/actors/runtime/proof"
+	"github.com/ipfs/go-datastore"
+	"github.com/minio/blake2b-simd"
+	cbg "github.com/whyrusleeping/cbor-gen"
+)
+
+type cachingVerifier struct {
+	ds      datastore.Datastore
+	backend ffiwrapper.Verifier
+}
+
+const bufsize = 128
+
+func (cv cachingVerifier) withCache(execute func() (bool, error), param cbg.CBORMarshaler) (bool, error) {
+	hasher := blake2b.New256()
+	wr := bufio.NewWriterSize(hasher, bufsize)
+	err := param.MarshalCBOR(wr)
+	if err != nil {
+		log.Errorf("could not marshal call info: %+v", err)
+		return execute()
+	}
+	err = wr.Flush()
+	if err != nil {
+		log.Errorf("could not flush: %+v", err)
+		return execute()
+	}
+	hash := hasher.Sum(nil)
+	key := datastore.NewKey(string(hash))
+	fromDs, err := cv.ds.Get(key)
+	if err == nil {
+		switch fromDs[0] {
+		case 's':
+			return true, nil
+		case 'f':
+			return false, nil
+		case 'e':
+			return false, errors.New(string(fromDs[1:]))
+		default:
+			log.Errorf("bad cached result in cache %s(%x)", fromDs[0], fromDs[0])
+			return execute()
+		}
+	} else if errors.Is(err, datastore.ErrNotFound) {
+		// recalc
+		ok, err := execute()
+		var save []byte
+		if err != nil {
+			if ok {
+				log.Errorf("success with an error: %+v", err)
+			} else {
+				save = append([]byte{'e'}, []byte(err.Error())...)
+			}
+		} else if ok {
+			save = []byte{'s'}
+		} else {
+			save = []byte{'f'}
+		}
+
+		if len(save) != 0 {
+			errSave := cv.ds.Put(key, save)
+			if errSave != nil {
+				log.Errorf("error saving result: %+v", errSave)
+			}
+		}
+
+		return ok, err
+	} else {
+		log.Errorf("could not get data from cache: %+v", err)
+		return execute()
+	}
+}
+
+func (cv *cachingVerifier) VerifySeal(svi proof.SealVerifyInfo) (bool, error) {
+	return cv.withCache(func() (bool, error) {
+		return cv.backend.VerifySeal(svi)
+	}, &svi)
+}
+func (cv *cachingVerifier) VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error) {
+	return cv.backend.VerifyWinningPoSt(ctx, info)
+}
+func (cv *cachingVerifier) VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error) {
+	return cv.withCache(func() (bool, error) {
+		return cv.backend.VerifyWindowPoSt(ctx, info)
+	}, &info)
+}
+func (cv *cachingVerifier) GenerateWinningPoStSectorChallenge(ctx context.Context, proofType abi.RegisteredPoStProof, a abi.ActorID, rnd abi.PoStRandomness, u uint64) ([]uint64, error) {
+	return cv.backend.GenerateWinningPoStSectorChallenge(ctx, proofType, a, rnd, u)
+}
+
+var _ ffiwrapper.Verifier = (*cachingVerifier)(nil)

cmd/lotus-bench/import.go

@@ -16,6 +16,8 @@ import (
 	"sort"
 	"time"
 
+	"github.com/cockroachdb/pebble"
+	"github.com/cockroachdb/pebble/bloom"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -24,12 +26,16 @@ import (
 	"github.com/filecoin-project/lotus/lib/blockstore"
 	_ "github.com/filecoin-project/lotus/lib/sigs/bls"
 	_ "github.com/filecoin-project/lotus/lib/sigs/secp"
+	"github.com/ipld/go-car"
 
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
 
+	bdg "github.com/dgraph-io/badger/v2"
 	"github.com/ipfs/go-datastore"
 	badger "github.com/ipfs/go-ds-badger2"
+	pebbleds "github.com/ipfs/go-ds-pebble"
+
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 )
@@ -60,6 +66,29 @@ var importBenchCmd = &cli.Command{
 			Name:  "repodir",
 			Usage: "set the repo directory for the lotus bench run (defaults to /tmp)",
 		},
+		&cli.StringFlag{
+			Name:  "syscall-cache",
+			Usage: "read and write syscall results from datastore",
+		},
+		&cli.BoolFlag{
+			Name:  "export-traces",
+			Usage: "should we export execution traces",
+			Value: true,
+		},
+		&cli.BoolFlag{
+			Name:  "no-import",
+			Usage: "should we import the chain? if set to true chain has to be previously imported",
+		},
+		&cli.BoolFlag{
+			Name:  "global-profile",
+			Value: true,
+		},
+		&cli.Int64Flag{
+			Name: "start-at",
+		},
+		&cli.BoolFlag{
+			Name: "only-import",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		vm.BatchSealVerifyParallelism = cctx.Int("batch-seal-verify-threads")
@@ -74,6 +103,10 @@ var importBenchCmd = &cli.Command{
 		}
 		defer cfi.Close() //nolint:errcheck // read only file
 
+		go func() {
+			http.ListenAndServe("localhost:6060", nil) //nolint:errcheck
+		}()
+
 		var tdir string
 		if rdir := cctx.String("repodir"); rdir != "" {
 			tdir = rdir
@@ -85,20 +118,75 @@ var importBenchCmd = &cli.Command{
 			tdir = tmp
 		}
 
-		bds, err := badger.NewDatastore(tdir, nil)
+		bdgOpt := badger.DefaultOptions
+		bdgOpt.GcInterval = 0
+		bdgOpt.Options = bdg.DefaultOptions("")
+		bdgOpt.Options.SyncWrites = false
+		bdgOpt.Options.Truncate = true
+		bdgOpt.Options.DetectConflicts = false
+
+		var bds datastore.Batching
+		if false {
+			cache := 512
+			bds, err = pebbleds.NewDatastore(tdir, &pebble.Options{
+				// Pebble has a single combined cache area and the write
+				// buffers are taken from this too. Assign all available
+				// memory allowance for cache.
+				Cache: pebble.NewCache(int64(cache * 1024 * 1024)),
+				// The size of memory table(as well as the write buffer).
+				// Note, there may have more than two memory tables in the system.
+				// MemTableStopWritesThreshold can be configured to avoid the memory abuse.
+				MemTableSize: cache * 1024 * 1024 / 4,
+				// The default compaction concurrency(1 thread),
+				// Here use all available CPUs for faster compaction.
+				MaxConcurrentCompactions: runtime.NumCPU(),
+				// Per-level options. Options for at least one level must be specified. The
+				// options for the last level are used for all subsequent levels.
+				Levels: []pebble.LevelOptions{
+					{TargetFileSize: 16 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10), Compression: pebble.NoCompression},
+				},
+				Logger: log,
+			})
+		} else {
+			bds, err = badger.NewDatastore(tdir, &bdgOpt)
+		}
 		if err != nil {
 			return err
 		}
+		defer bds.Close() //nolint:errcheck
+
 		bs := blockstore.NewBlockstore(bds)
-		cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, blockstore.DefaultCacheOpts())
+		cacheOpts := blockstore.DefaultCacheOpts()
+		cacheOpts.HasBloomFilterSize = 0
+
+		cbs, err := blockstore.CachedBlockstore(context.TODO(), bs, cacheOpts)
 		if err != nil {
 			return err
 		}
 		bs = cbs
 		ds := datastore.NewMapDatastore()
-		cs := store.NewChainStore(bs, ds, vm.Syscalls(ffiwrapper.ProofVerifier))
+
+		var verifier ffiwrapper.Verifier = ffiwrapper.ProofVerifier
+		if cctx.IsSet("syscall-cache") {
+			scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &bdgOpt)
+			if err != nil {
+				return xerrors.Errorf("opening syscall-cache datastore: %w", err)
+			}
+			defer scds.Close() //nolint:errcheck
+
+			verifier = &cachingVerifier{
+				ds:      scds,
+				backend: verifier,
+			}
+		}
+		if cctx.Bool("only-gc") {
+			return nil
+		}
+
+		cs := store.NewChainStore(bs, ds, vm.Syscalls(verifier))
 		stm := stmgr.NewStateManager(cs)
 
+		if cctx.Bool("global-profile") {
 			prof, err := os.Create("import-bench.prof")
 			if err != nil {
 				return err
@@ -108,11 +196,28 @@ var importBenchCmd = &cli.Command{
 			if err := pprof.StartCPUProfile(prof); err != nil {
 				return err
 			}
+		}
 
-		head, err := cs.Import(cfi)
+		var head *types.TipSet
+		if !cctx.Bool("no-import") {
+			head, err = cs.Import(cfi)
 			if err != nil {
 				return err
 			}
+		} else {
+			cr, err := car.NewCarReader(cfi)
+			if err != nil {
+				return err
+			}
+			head, err = cs.LoadTipSet(types.NewTipSetKey(cr.Header.Roots...))
+			if err != nil {
+				return err
+			}
+		}
+
+		if cctx.Bool("only-import") {
+			return nil
+		}
 
 		gb, err := cs.GetTipsetByHeight(context.TODO(), 0, head, true)
 		if err != nil {
@@ -124,6 +229,20 @@ var importBenchCmd = &cli.Command{
 			return err
 		}
 
+		startEpoch := abi.ChainEpoch(1)
+		if cctx.IsSet("start-at") {
+			startEpoch = abi.ChainEpoch(cctx.Int64("start-at"))
+			start, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(cctx.Int64("start-at")), head, true)
+			if err != nil {
+				return err
+			}
+
+			err = cs.SetHead(start)
+			if err != nil {
+				return err
+			}
+		}
+
 		if h := cctx.Int64("height"); h != 0 {
 			tsh, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(h), head, true)
 			if err != nil {
@@ -134,7 +253,7 @@ var importBenchCmd = &cli.Command{
 
 		ts := head
 		tschain := []*types.TipSet{ts}
-		for ts.Height() != 0 {
+		for ts.Height() > startEpoch {
 			next, err := cs.LoadTipSet(ts.Parents())
 			if err != nil {
 				return err
@@ -144,45 +263,48 @@ var importBenchCmd = &cli.Command{
 			ts = next
 		}
 
+		var enc *json.Encoder
+		if cctx.Bool("export-traces") {
 			ibj, err := os.Create("import-bench.json")
 			if err != nil {
 				return err
 			}
 			defer ibj.Close() //nolint:errcheck
 
-		enc := json.NewEncoder(ibj)
+			enc = json.NewEncoder(ibj)
+		}
 
-		var lastTse *TipSetExec
+		for i := len(tschain) - 1; i >= 1; i-- {
-
-		lastState := tschain[len(tschain)-1].ParentState()
-		for i := len(tschain) - 2; i >= 0; i-- {
 			cur := tschain[i]
+			start := time.Now()
 			log.Infof("computing state (height: %d, ts=%s)", cur.Height(), cur.Cids())
-			if cur.ParentState() != lastState {
+			st, trace, err := stm.ExecutionTrace(context.TODO(), cur)
-				lastTrace := lastTse.Trace
+			if err != nil {
+				return err
+			}
+			tse := &TipSetExec{
+				TipSet:   cur.Key(),
+				Trace:    trace,
+				Duration: time.Since(start),
+			}
+			if enc != nil {
+				stripCallers(tse.Trace)
+
+				if err := enc.Encode(tse); err != nil {
+					return xerrors.Errorf("failed to write out tipsetexec: %w", err)
+				}
+			}
+			if tschain[i-1].ParentState() != st {
+				stripCallers(tse.Trace)
+				lastTrace := tse.Trace
 				d, err := json.MarshalIndent(lastTrace, "", "  ")
 				if err != nil {
 					panic(err)
 				}
 				fmt.Println("TRACE")
 				fmt.Println(string(d))
-				return xerrors.Errorf("tipset chain had state mismatch at height %d (%s != %s)", cur.Height(), cur.ParentState(), lastState)
+				//fmt.Println(statediff.Diff(context.Background(), bs, tschain[i-1].ParentState(), st, statediff.ExpandActors))
-			}
+				return xerrors.Errorf("tipset chain had state mismatch at height %d (%s != %s)", cur.Height(), cur.ParentState(), st)
-			start := time.Now()
-			st, trace, err := stm.ExecutionTrace(context.TODO(), cur)
-			if err != nil {
-				return err
-			}
-			stripCallers(trace)
-
-			lastTse = &TipSetExec{
-				TipSet:   cur.Key(),
-				Trace:    trace,
-				Duration: time.Since(start),
-			}
-			lastState = st
-			if err := enc.Encode(lastTse); err != nil {
-				return xerrors.Errorf("failed to write out tipsetexec: %w", err)
 			}
 		}
 

cmd/lotus-seal-worker/main.go

@@ -111,6 +111,11 @@ var runCmd = &cli.Command{
 			Name:  "no-local-storage",
 			Usage: "don't use storageminer repo for sector storage",
 		},
+		&cli.BoolFlag{
+			Name:  "no-swap",
+			Usage: "don't use swap",
+			Value: false,
+		},
 		&cli.BoolFlag{
 			Name:  "addpiece",
 			Usage: "enable addpiece",
@@ -355,6 +360,7 @@ var runCmd = &cli.Command{
 			LocalWorker: sectorstorage.NewLocalWorker(sectorstorage.WorkerConfig{
 				SealProof: spt,
 				TaskTypes: taskTypes,
+				NoSwap:    cctx.Bool("no-swap"),
 			}, remote, localStore, nodeApi, nodeApi, wsts),
 			localStore: localStore,
 			ls:         lr,

cmd/lotus-shed/dealtracker.go

@@ -5,10 +5,10 @@ import (
 	"encoding/json"
 	"net"
 	"net/http"
-	"os"
+	"sync"
-	"strings"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/lotus/api"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/ipfs/go-cid"
@@ -19,58 +19,54 @@ type dealStatsServer struct {
 	api api.FullNode
 }
 
-var filteredClients map[address.Address]bool
+// Requested by @jbenet
+// How many epochs back to look at for dealstats
+var epochLookback = abi.ChainEpoch(10)
+
+// these lists grow continuously with the network
+// TODO: need to switch this to an LRU of sorts, to ensure refreshes
+var knownFiltered = new(sync.Map)
+var resolvedWallets = new(sync.Map)
 
 func init() {
-	fc := []string{"t0112", "t0113", "t0114", "t010089"}
+	for _, a := range []string{
-
+		"t0100", // client for genesis miner
-	filtered, set := os.LookupEnv("FILTERED_CLIENTS")
+		"t0101", // client for genesis miner
-	if set {
+		"t0102", // client for genesis miner
-		fc = strings.Split(filtered, ":")
+		"t0112", // client for genesis miner
-	}
+		"t0113", // client for genesis miner
-
+		"t0114", // client for genesis miner
-	filteredClients = make(map[address.Address]bool)
+		"t1nslxql4pck5pq7hddlzym3orxlx35wkepzjkm3i", // SR1 dealbot wallet
-	for _, a := range fc {
+		"t1stghxhdp2w53dym2nz2jtbpk6ccd4l2lxgmezlq", // SR1 dealbot wallet
-		addr, err := address.NewFromString(a)
+		"t1mcr5xkgv4jdl3rnz77outn6xbmygb55vdejgbfi", // SR1 dealbot wallet
+		"t1qiqdbbmrdalbntnuapriirduvxu5ltsc5mhy7si", // SR1 dealbot wallet
+	} {
+		a, err := address.NewFromString(a)
 		if err != nil {
 			panic(err)
 		}
-		filteredClients[addr] = true
+		knownFiltered.Store(a, true)
 	}
 }
 
 type dealCountResp struct {
-	Total int64 `json:"total"`
 	Epoch    int64  `json:"epoch"`
+	Endpoint string `json:"endpoint"`
+	Payload  int64  `json:"payload"`
 }
 
 func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *http.Request) {
-	ctx := context.Background()
 
-	head, err := dss.api.ChainHead(ctx)
+	epoch, deals := dss.filteredDealList()
-	if err != nil {
+	if epoch == 0 {
-		log.Warnf("failed to get chain head: %s", err)
 		w.WriteHeader(500)
 		return
 	}
 
-	deals, err := dss.api.StateMarketDeals(ctx, head.Key())
-	if err != nil {
-		log.Warnf("failed to get market deals: %s", err)
-		w.WriteHeader(500)
-		return
-	}
-
-	var count int64
-	for _, d := range deals {
-		if !filteredClients[d.Proposal.Client] {
-			count++
-		}
-	}
-
 	if err := json.NewEncoder(w).Encode(&dealCountResp{
-		Total: count,
+		Endpoint: "COUNT_DEALS",
-		Epoch: int64(head.Height()),
+		Payload:  int64(len(deals)),
+		Epoch:    epoch,
 	}); err != nil {
 		log.Warnf("failed to write back deal count response: %s", err)
 		return
@@ -78,39 +74,28 @@ func (dss *dealStatsServer) handleStorageDealCount(w http.ResponseWriter, r *htt
 }
 
 type dealAverageResp struct {
-	AverageSize int64 `json:"average_size"`
 	Epoch    int64  `json:"epoch"`
+	Endpoint string `json:"endpoint"`
+	Payload  int64  `json:"payload"`
 }
 
 func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter, r *http.Request) {
-	ctx := context.Background()
 
-	head, err := dss.api.ChainHead(ctx)
+	epoch, deals := dss.filteredDealList()
-	if err != nil {
+	if epoch == 0 {
-		log.Warnf("failed to get chain head: %s", err)
 		w.WriteHeader(500)
 		return
 	}
 
-	deals, err := dss.api.StateMarketDeals(ctx, head.Key())
-	if err != nil {
-		log.Warnf("failed to get market deals: %s", err)
-		w.WriteHeader(500)
-		return
-	}
-
-	var count int64
 	var totalBytes int64
 	for _, d := range deals {
-		if !filteredClients[d.Proposal.Client] {
+		totalBytes += int64(d.deal.Proposal.PieceSize.Unpadded())
-			count++
-			totalBytes += int64(d.Proposal.PieceSize.Unpadded())
-		}
 	}
 
 	if err := json.NewEncoder(w).Encode(&dealAverageResp{
-		AverageSize: totalBytes / count,
+		Endpoint: "AVERAGE_DEAL_SIZE",
-		Epoch:       int64(head.Height()),
+		Payload:  totalBytes / int64(len(deals)),
+		Epoch:    epoch,
 	}); err != nil {
 		log.Warnf("failed to write back deal average response: %s", err)
 		return
@@ -118,37 +103,27 @@ func (dss *dealStatsServer) handleStorageDealAverageSize(w http.ResponseWriter,
 }
 
 type dealTotalResp struct {
-	TotalBytes int64 `json:"total_size"`
 	Epoch    int64  `json:"epoch"`
+	Endpoint string `json:"endpoint"`
+	Payload  int64  `json:"payload"`
 }
 
 func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r *http.Request) {
-	ctx := context.Background()
+	epoch, deals := dss.filteredDealList()
-
+	if epoch == 0 {
-	head, err := dss.api.ChainHead(ctx)
-	if err != nil {
-		log.Warnf("failed to get chain head: %s", err)
-		w.WriteHeader(500)
-		return
-	}
-
-	deals, err := dss.api.StateMarketDeals(ctx, head.Key())
-	if err != nil {
-		log.Warnf("failed to get market deals: %s", err)
 		w.WriteHeader(500)
 		return
 	}
 
 	var totalBytes int64
 	for _, d := range deals {
-		if !filteredClients[d.Proposal.Client] {
+		totalBytes += int64(d.deal.Proposal.PieceSize.Unpadded())
-			totalBytes += int64(d.Proposal.PieceSize.Unpadded())
-		}
 	}
 
 	if err := json.NewEncoder(w).Encode(&dealTotalResp{
-		TotalBytes: totalBytes,
+		Endpoint: "DEAL_BYTES",
-		Epoch:      int64(head.Height()),
+		Payload:  totalBytes,
+		Epoch:    epoch,
 	}); err != nil {
 		log.Warnf("failed to write back deal average response: %s", err)
 		return
@@ -157,6 +132,12 @@ func (dss *dealStatsServer) handleStorageDealTotalReal(w http.ResponseWriter, r
 }
 
 type clientStatsOutput struct {
+	Epoch    int64          `json:"epoch"`
+	Endpoint string         `json:"endpoint"`
+	Payload  []*clientStats `json:"payload"`
+}
+
+type clientStats struct {
 	Client    address.Address `json:"client"`
 	DataSize  int64           `json:"data_size"`
 	NumCids   int             `json:"num_cids"`
@@ -168,51 +149,41 @@ type clientStatsOutput struct {
 }
 
 func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *http.Request) {
-	ctx := context.Background()
+	epoch, deals := dss.filteredDealList()
-
+	if epoch == 0 {
-	head, err := dss.api.ChainHead(ctx)
-	if err != nil {
-		log.Warnf("failed to get chain head: %s", err)
 		w.WriteHeader(500)
 		return
 	}
 
-	deals, err := dss.api.StateMarketDeals(ctx, head.Key())
+	stats := make(map[address.Address]*clientStats)
-	if err != nil {
-		log.Warnf("failed to get market deals: %s", err)
-		w.WriteHeader(500)
-		return
-	}
-
-	stats := make(map[address.Address]*clientStatsOutput)
 
 	for _, d := range deals {
-		if filteredClients[d.Proposal.Client] {
-			continue
-		}
 
-		st, ok := stats[d.Proposal.Client]
+		st, ok := stats[d.deal.Proposal.Client]
 		if !ok {
-			st = &clientStatsOutput{
+			st = &clientStats{
-				Client:    d.Proposal.Client,
+				Client:    d.resolvedWallet,
 				cids:      make(map[cid.Cid]bool),
 				providers: make(map[address.Address]bool),
 			}
-			stats[d.Proposal.Client] = st
+			stats[d.deal.Proposal.Client] = st
 		}
 
-		st.DataSize += int64(d.Proposal.PieceSize.Unpadded())
+		st.DataSize += int64(d.deal.Proposal.PieceSize.Unpadded())
-		st.cids[d.Proposal.PieceCID] = true
+		st.cids[d.deal.Proposal.PieceCID] = true
-		st.providers[d.Proposal.Provider] = true
+		st.providers[d.deal.Proposal.Provider] = true
 		st.NumDeals++
 	}
 
-	out := make([]*clientStatsOutput, 0, len(stats))
+	out := clientStatsOutput{
-	for _, cso := range stats {
+		Epoch:    epoch,
-		cso.NumCids = len(cso.cids)
+		Endpoint: "CLIENT_DEAL_STATS",
-		cso.NumMiners = len(cso.providers)
+		Payload:  make([]*clientStats, 0, len(stats)),
-
+	}
-		out = append(out, cso)
+	for _, cs := range stats {
+		cs.NumCids = len(cs.cids)
+		cs.NumMiners = len(cs.providers)
+		out.Payload = append(out.Payload, cs)
 	}
 
 	if err := json.NewEncoder(w).Encode(out); err != nil {
@@ -221,6 +192,93 @@ func (dss *dealStatsServer) handleStorageClientStats(w http.ResponseWriter, r *h
 	}
 }
 
+type dealInfo struct {
+	deal           api.MarketDeal
+	resolvedWallet address.Address
+}
+
+// filteredDealList returns the current epoch and a list of filtered deals
+// on error returns an epoch of 0
+func (dss *dealStatsServer) filteredDealList() (int64, map[string]dealInfo) {
+	ctx := context.Background()
+
+	head, err := dss.api.ChainHead(ctx)
+	if err != nil {
+		log.Warnf("failed to get chain head: %s", err)
+		return 0, nil
+	}
+
+	head, err = dss.api.ChainGetTipSetByHeight(ctx, head.Height()-epochLookback, head.Key())
+	if err != nil {
+		log.Warnf("failed to walk back %s epochs: %s", epochLookback, err)
+		return 0, nil
+	}
+
+	// Disabled as per @pooja's request
+	//
+	// // Exclude any address associated with a miner
+	// miners, err := dss.api.StateListMiners(ctx, head.Key())
+	// if err != nil {
+	// 	log.Warnf("failed to get miner list: %s", err)
+	// 	return 0, nil
+	// }
+	// for _, m := range miners {
+	// 	info, err := dss.api.StateMinerInfo(ctx, m, head.Key())
+	// 	if err != nil {
+	// 		log.Warnf("failed to get info for known miner '%s': %s", m, err)
+	// 		continue
+	// 	}
+
+	// 	knownFiltered.Store(info.Owner, true)
+	// 	knownFiltered.Store(info.Worker, true)
+	// 	for _, a := range info.ControlAddresses {
+	// 		knownFiltered.Store(a, true)
+	// 	}
+	// }
+
+	deals, err := dss.api.StateMarketDeals(ctx, head.Key())
+	if err != nil {
+		log.Warnf("failed to get market deals: %s", err)
+		return 0, nil
+	}
+
+	ret := make(map[string]dealInfo, len(deals))
+	for dealKey, d := range deals {
+
+		// Counting no-longer-active deals as per Pooja's request
+		// // https://github.com/filecoin-project/specs-actors/blob/v0.9.9/actors/builtin/market/deal.go#L81-L85
+		// if d.State.SectorStartEpoch < 0 {
+		// 	continue
+		// }
+
+		if _, isFiltered := knownFiltered.Load(d.Proposal.Client); isFiltered {
+			continue
+		}
+
+		if _, wasSeen := resolvedWallets.Load(d.Proposal.Client); !wasSeen {
+			w, err := dss.api.StateAccountKey(ctx, d.Proposal.Client, head.Key())
+			if err != nil {
+				log.Warnf("failed to resolve id '%s' to wallet address: %s", d.Proposal.Client, err)
+				continue
+			} else {
+				resolvedWallets.Store(d.Proposal.Client, w)
+			}
+		}
+
+		w, _ := resolvedWallets.Load(d.Proposal.Client)
+		if _, isFiltered := knownFiltered.Load(w); isFiltered {
+			continue
+		}
+
+		ret[dealKey] = dealInfo{
+			deal:           d,
+			resolvedWallet: w.(address.Address),
+		}
+	}
+
+	return int64(head.Height()), ret
+}
+
 var serveDealStatsCmd = &cli.Command{
 	Name:  "serve-deal-stats",
 	Flags: []cli.Flag{},
@@ -260,6 +318,8 @@ var serveDealStatsCmd = &cli.Command{
 			panic(err)
 		}
 
+		log.Warnf("deal-stat server listening on %s\n== NOTE: QUERIES ARE EXPENSIVE - YOU MUST FRONT-CACHE THIS SERVICE\n", list.Addr().String())
+
 		return s.Serve(list)
 	},
 }

cmd/lotus-shed/import-car.go

@@ -1,10 +1,13 @@
 package main
 
 import (
+	"encoding/hex"
 	"fmt"
 	"io"
 	"os"
 
+	block "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
 	"github.com/ipld/go-car"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
@@ -81,3 +84,57 @@ var importCarCmd = &cli.Command{
 		}
 	},
 }
+
+var importObjectCmd = &cli.Command{
+	Name:  "import-obj",
+	Usage: "import a raw ipld object into your datastore",
+	Action: func(cctx *cli.Context) error {
+		r, err := repo.NewFS(cctx.String("repo"))
+		if err != nil {
+			return xerrors.Errorf("opening fs repo: %w", err)
+		}
+
+		exists, err := r.Exists()
+		if err != nil {
+			return err
+		}
+		if !exists {
+			return xerrors.Errorf("lotus repo doesn't exist")
+		}
+
+		lr, err := r.Lock(repo.FullNode)
+		if err != nil {
+			return err
+		}
+		defer lr.Close() //nolint:errcheck
+
+		ds, err := lr.Datastore("/chain")
+		if err != nil {
+			return err
+		}
+
+		bs := blockstore.NewBlockstore(ds)
+
+		c, err := cid.Decode(cctx.Args().Get(0))
+		if err != nil {
+			return err
+		}
+
+		data, err := hex.DecodeString(cctx.Args().Get(1))
+		if err != nil {
+			return err
+		}
+
+		blk, err := block.NewBlockWithCid(data, c)
+		if err != nil {
+			return err
+		}
+
+		if err := bs.Put(blk); err != nil {
+			return err
+		}
+
+		return nil
+
+	},
+}

cmd/lotus-shed/main.go

@@ -25,6 +25,7 @@ func main() {
 		staterootCmd,
 		auditsCmd,
 		importCarCmd,
+		importObjectCmd,
 		commpToCidCmd,
 		fetchParamCmd,
 		proofsCmd,
@@ -37,6 +38,7 @@ func main() {
 		exportChainCmd,
 		consensusCmd,
 		serveDealStatsCmd,
+		syncCmd,
 		datastoreCmd,
 	}
 

cmd/lotus-shed/sync.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-cid"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/urfave/cli/v2"
+)
+
+var syncCmd = &cli.Command{
+	Name:  "sync",
+	Usage: "tools for diagnosing sync issues",
+	Flags: []cli.Flag{},
+	Subcommands: []*cli.Command{
+		syncValidateCmd,
+	},
+}
+
+var syncValidateCmd = &cli.Command{
+	Name:  "validate",
+	Usage: "checks whether a provided tipset is valid",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+
+		defer closer()
+		ctx := lcli.ReqContext(cctx)
+
+		if cctx.Args().Len() < 1 {
+			fmt.Println("usage: <blockCid1> <blockCid2>...")
+			fmt.Println("At least one block cid must be provided")
+			return nil
+		}
+
+		args := cctx.Args().Slice()
+
+		var tscids []cid.Cid
+		for _, s := range args {
+			c, err := cid.Decode(s)
+			if err != nil {
+				return fmt.Errorf("block cid was invalid: %s", err)
+			}
+			tscids = append(tscids, c)
+		}
+
+		tsk := types.NewTipSetKey(tscids...)
+
+		valid, err := api.SyncValidateTipset(ctx, tsk)
+		if err != nil {
+			fmt.Println("Tipset is invalid: ", err)
+		}
+
+		if valid {
+			fmt.Println("Tipset is valid")
+		}
+
+		return nil
+	},
+}

cmd/lotus-storage-miner/info.go

@@ -33,6 +33,12 @@ var infoCmd = &cli.Command{
 	Subcommands: []*cli.Command{
 		infoAllCmd,
 	},
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "hide-sectors-info",
+			Usage: "hide sectors info",
+		},
+	},
 	Action: infoCmdAct,
 }
 
@@ -199,11 +205,13 @@ func infoCmdAct(cctx *cli.Context) error {
 
 	fmt.Printf("Expected Seal Duration: %s\n\n", sealdur)
 
+	if !cctx.Bool("hide-sectors-info") {
 		fmt.Println("Sectors:")
 		err = sectorsInfo(ctx, nodeApi)
 		if err != nil {
 			return err
 		}
+	}
 
 	// TODO: grab actr state / info
 	//  * Sealed sectors (count / bytes)

cmd/lotus/daemon.go

@@ -3,11 +3,14 @@
 package main
 
 import (
+	"bufio"
 	"context"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"runtime/pprof"
 	"strings"
@@ -23,6 +26,7 @@ import (
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/tag"
 	"golang.org/x/xerrors"
+	"gopkg.in/cheggaaa/pb.v1"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
@@ -100,11 +104,11 @@ var DaemonCmd = &cli.Command{
 		},
 		&cli.StringFlag{
 			Name:  "import-chain",
-			Usage: "on first run, load chain from given file and validate",
+			Usage: "on first run, load chain from given file or url and validate",
 		},
 		&cli.StringFlag{
 			Name:  "import-snapshot",
-			Usage: "import chain state from a given chain export file",
+			Usage: "import chain state from a given chain export file or url",
 		},
 		&cli.BoolFlag{
 			Name:  "halt-after-import",
@@ -123,6 +127,10 @@ var DaemonCmd = &cli.Command{
 			Usage: "manage open file limit",
 			Value: true,
 		},
+		&cli.StringFlag{
+			Name:  "config",
+			Usage: "specify path of config file to use",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		err := runmetrics.Enable(runmetrics.RunMetricOptions{
@@ -176,6 +184,10 @@ var DaemonCmd = &cli.Command{
 			return xerrors.Errorf("opening fs repo: %w", err)
 		}
 
+		if cctx.String("config") != "" {
+			r.SetConfigPath(cctx.String("config"))
+		}
+
 		if err := r.Init(repo.FullNode); err != nil && err != repo.ErrRepoExists {
 			return xerrors.Errorf("repo init error: %w", err)
 		}
@@ -206,11 +218,6 @@ var DaemonCmd = &cli.Command{
 				issnapshot = true
 			}
 
-			chainfile, err := homedir.Expand(chainfile)
-			if err != nil {
-				return err
-			}
-
 			if err := ImportChain(r, chainfile, issnapshot); err != nil {
 				return err
 			}
@@ -326,13 +333,43 @@ func importKey(ctx context.Context, api api.FullNode, f string) error {
 	return nil
 }
 
-func ImportChain(r repo.Repo, fname string, snapshot bool) error {
+func ImportChain(r repo.Repo, fname string, snapshot bool) (err error) {
+	var rd io.Reader
+	var l int64
+	if strings.HasPrefix(fname, "http://") || strings.HasPrefix(fname, "https://") {
+		resp, err := http.Get(fname) //nolint:gosec
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close() //nolint:errcheck
+
+		if resp.StatusCode != http.StatusOK {
+			return xerrors.Errorf("non-200 response: %d", resp.StatusCode)
+		}
+
+		rd = resp.Body
+		l = resp.ContentLength
+	} else {
+		fname, err = homedir.Expand(fname)
+		if err != nil {
+			return err
+		}
+
 		fi, err := os.Open(fname)
 		if err != nil {
 			return err
 		}
 		defer fi.Close() //nolint:errcheck
 
+		st, err := os.Stat(fname)
+		if err != nil {
+			return err
+		}
+
+		rd = fi
+		l = st.Size()
+	}
+
 	lr, err := r.Lock(repo.FullNode)
 	if err != nil {
 		return err
@@ -353,8 +390,21 @@ func ImportChain(r repo.Repo, fname string, snapshot bool) error {
 
 	cst := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier))
 
-	log.Info("importing chain from file...")
+	log.Infof("importing chain from %s...", fname)
-	ts, err := cst.Import(fi)
+
+	bufr := bufio.NewReaderSize(rd, 1<<20)
+
+	bar := pb.New64(l)
+	br := bar.NewProxyReader(bufr)
+	bar.ShowTimeLeft = true
+	bar.ShowPercent = true
+	bar.ShowSpeed = true
+	bar.Units = pb.U_BYTES
+
+	bar.Start()
+	ts, err := cst.Import(br)
+	bar.Finish()
+
 	if err != nil {
 		return xerrors.Errorf("importing chain failed: %w", err)
 	}

cmd/tvx/actor_mapping.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"reflect"
+
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+)
+
+var ActorMethodTable = make(map[string][]string, 64)
+
+var Actors = map[cid.Cid]interface{}{
+	builtin.InitActorCodeID:             builtin.MethodsInit,
+	builtin.CronActorCodeID:             builtin.MethodsCron,
+	builtin.AccountActorCodeID:          builtin.MethodsAccount,
+	builtin.StoragePowerActorCodeID:     builtin.MethodsPower,
+	builtin.StorageMinerActorCodeID:     builtin.MethodsMiner,
+	builtin.StorageMarketActorCodeID:    builtin.MethodsMarket,
+	builtin.PaymentChannelActorCodeID:   builtin.MethodsPaych,
+	builtin.MultisigActorCodeID:         builtin.MethodsMultisig,
+	builtin.RewardActorCodeID:           builtin.MethodsReward,
+	builtin.VerifiedRegistryActorCodeID: builtin.MethodsVerifiedRegistry,
+}
+
+func init() {
+	for code, methods := range Actors {
+		cmh, err := multihash.Decode(code.Hash()) // identity hash.
+		if err != nil {
+			panic(err)
+		}
+
+		var (
+			aname = string(cmh.Digest)
+			rt    = reflect.TypeOf(methods)
+			nf    = rt.NumField()
+		)
+
+		ActorMethodTable[aname] = append(ActorMethodTable[aname], "Send")
+		for i := 0; i < nf; i++ {
+			ActorMethodTable[aname] = append(ActorMethodTable[aname], rt.Field(i).Name)
+		}
+	}
+}

cmd/tvx/exec.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/urfave/cli/v2"
+
+	"github.com/filecoin-project/lotus/conformance"
+
+	"github.com/filecoin-project/test-vectors/schema"
+)
+
+var execFlags struct {
+	file string
+}
+
+var execCmd = &cli.Command{
+	Name:        "exec",
+	Description: "execute one or many test vectors against Lotus; supplied as a single JSON file, or a ndjson stdin stream",
+	Action:      runExecLotus,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:        "file",
+			Usage:       "input file; if not supplied, the vector will be read from stdin",
+			TakesFile:   true,
+			Destination: &execFlags.file,
+		},
+	},
+}
+
+func runExecLotus(_ *cli.Context) error {
+	if file := execFlags.file; file != "" {
+		// we have a single test vector supplied as a file.
+		file, err := os.Open(file)
+		if err != nil {
+			return fmt.Errorf("failed to open test vector: %w", err)
+		}
+
+		var (
+			dec = json.NewDecoder(file)
+			tv  schema.TestVector
+		)
+
+		if err = dec.Decode(&tv); err != nil {
+			return fmt.Errorf("failed to decode test vector: %w", err)
+		}
+
+		return executeTestVector(tv)
+	}
+
+	for dec := json.NewDecoder(os.Stdin); ; {
+		var tv schema.TestVector
+		switch err := dec.Decode(&tv); err {
+		case nil:
+			if err = executeTestVector(tv); err != nil {
+				return err
+			}
+		case io.EOF:
+			// we're done.
+			return nil
+		default:
+			// something bad happened.
+			return err
+		}
+	}
+}
+
+func executeTestVector(tv schema.TestVector) error {
+	log.Println("executing test vector:", tv.Meta.ID)
+	r := new(conformance.LogReporter)
+	switch class := tv.Class; class {
+	case "message":
+		conformance.ExecuteMessageVector(r, &tv)
+	case "tipset":
+		conformance.ExecuteTipsetVector(r, &tv)
+	default:
+		return fmt.Errorf("test vector class %s not supported", class)
+	}
+
+	if r.Failed() {
+		log.Println(color.HiRedString("❌ test vector failed"))
+	} else {
+		log.Println(color.GreenString("✅ test vector succeeded"))
+	}
+
+	return nil
+}

cmd/tvx/extract.go

@@ -0,0 +1,503 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/fatih/color"
+
+	"github.com/filecoin-project/lotus/api"
+	init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/reward"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/conformance"
+
+	"github.com/filecoin-project/specs-actors/actors/builtin"
+
+	"github.com/filecoin-project/test-vectors/schema"
+
+	"github.com/ipfs/go-cid"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	PrecursorSelectAll    = "all"
+	PrecursorSelectSender = "sender"
+)
+
+type extractOpts struct {
+	id        string
+	block     string
+	class     string
+	cid       string
+	file      string
+	retain    string
+	precursor string
+}
+
+var extractFlags extractOpts
+
+var extractCmd = &cli.Command{
+	Name:        "extract",
+	Description: "generate a test vector by extracting it from a live chain",
+	Action:      runExtract,
+	Flags: []cli.Flag{
+		&repoFlag,
+		&cli.StringFlag{
+			Name:        "class",
+			Usage:       "class of vector to extract; other required flags depend on the; values: 'message'",
+			Value:       "message",
+			Destination: &extractFlags.class,
+		},
+		&cli.StringFlag{
+			Name:        "id",
+			Usage:       "identifier to name this test vector with",
+			Value:       "(undefined)",
+			Destination: &extractFlags.id,
+		},
+		&cli.StringFlag{
+			Name:        "block",
+			Usage:       "optionally, the block CID the message was included in, to avoid expensive chain scanning",
+			Destination: &extractFlags.block,
+		},
+		&cli.StringFlag{
+			Name:        "cid",
+			Usage:       "message CID to generate test vector from",
+			Required:    true,
+			Destination: &extractFlags.cid,
+		},
+		&cli.StringFlag{
+			Name:        "out",
+			Aliases:     []string{"o"},
+			Usage:       "file to write test vector to",
+			Destination: &extractFlags.file,
+		},
+		&cli.StringFlag{
+			Name:        "state-retain",
+			Usage:       "state retention policy; values: 'accessed-cids', 'accessed-actors'",
+			Value:       "accessed-cids",
+			Destination: &extractFlags.retain,
+		},
+		&cli.StringFlag{
+			Name: "precursor-select",
+			Usage: "precursors to apply; values: 'all', 'sender'; 'all' selects all preceding" +
+				"messages in the canonicalised tipset, 'sender' selects only preceding messages from the same" +
+				"sender. Usually, 'sender' is a good tradeoff and gives you sufficient accuracy. If the receipt sanity" +
+				"check fails due to gas reasons, switch to 'all', as previous messages in the tipset may have" +
+				"affected state in a disruptive way",
+			Value:       "sender",
+			Destination: &extractFlags.precursor,
+		},
+	},
+}
+
+func runExtract(c *cli.Context) error {
+	// LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering",
+	// which stashes write operations in a BufferedBlockstore
+	// (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21)
+	// such that they're not written until the VM is actually flushed.
+	//
+	// For some reason, the standard behaviour was not working for me (raulk),
+	// and disabling it (such that the state transformations are written immediately
+	// to the blockstore) worked.
+	_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
+
+	ctx := context.Background()
+
+	// Make the API client.
+	fapi, closer, err := lcli.GetFullNodeAPI(c)
+	if err != nil {
+		return err
+	}
+	defer closer()
+
+	return doExtract(ctx, fapi, extractFlags)
+}
+
+func doExtract(ctx context.Context, fapi api.FullNode, opts extractOpts) error {
+	mcid, err := cid.Decode(opts.cid)
+	if err != nil {
+		return err
+	}
+
+	msg, execTs, incTs, err := resolveFromChain(ctx, fapi, mcid, opts.block)
+	if err != nil {
+		return fmt.Errorf("failed to resolve message and tipsets from chain: %w", err)
+	}
+
+	// get the circulating supply before the message was executed.
+	circSupplyDetail, err := fapi.StateCirculatingSupply(ctx, incTs.Key())
+	if err != nil {
+		return fmt.Errorf("failed while fetching circulating supply: %w", err)
+	}
+
+	circSupply := circSupplyDetail.FilCirculating
+
+	log.Printf("message was executed in tipset: %s", execTs.Key())
+	log.Printf("message was included in tipset: %s", incTs.Key())
+	log.Printf("circulating supply at inclusion tipset: %d", circSupply)
+	log.Printf("finding precursor messages using mode: %s", opts.precursor)
+
+	// Fetch messages in canonical order from inclusion tipset.
+	msgs, err := fapi.ChainGetParentMessages(ctx, execTs.Blocks()[0].Cid())
+	if err != nil {
+		return fmt.Errorf("failed to fetch messages in canonical order from inclusion tipset: %w", err)
+	}
+
+	related, found, err := findMsgAndPrecursors(opts.precursor, msg, msgs)
+	if err != nil {
+		return fmt.Errorf("failed while finding message and precursors: %w", err)
+	}
+
+	if !found {
+		return fmt.Errorf("message not found; precursors found: %d", len(related))
+	}
+
+	var (
+		precursors     = related[:len(related)-1]
+		precursorsCids []cid.Cid
+	)
+
+	for _, p := range precursors {
+		precursorsCids = append(precursorsCids, p.Cid())
+	}
+
+	log.Println(color.GreenString("found message; precursors (count: %d): %v", len(precursors), precursorsCids))
+
+	var (
+		// create a read-through store that uses ChainGetObject to fetch unknown CIDs.
+		pst = NewProxyingStores(ctx, fapi)
+		g   = NewSurgeon(ctx, fapi, pst)
+	)
+
+	driver := conformance.NewDriver(ctx, schema.Selector{}, conformance.DriverOpts{
+		DisableVMFlush: true,
+	})
+
+	// this is the root of the state tree we start with.
+	root := incTs.ParentState()
+	log.Printf("base state tree root CID: %s", root)
+
+	basefee := incTs.Blocks()[0].ParentBaseFee
+	log.Printf("basefee: %s", basefee)
+
+	// on top of that state tree, we apply all precursors.
+	log.Printf("number of precursors to apply: %d", len(precursors))
+	for i, m := range precursors {
+		log.Printf("applying precursor %d, cid: %s", i, m.Cid())
+		_, root, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
+			Preroot:    root,
+			Epoch:      execTs.Height(),
+			Message:    m,
+			CircSupply: &circSupplyDetail.FilCirculating,
+			BaseFee:    &basefee,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to execute precursor message: %w", err)
+		}
+	}
+
+	var (
+		preroot   cid.Cid
+		postroot  cid.Cid
+		applyret  *vm.ApplyRet
+		carWriter func(w io.Writer) error
+		retention = opts.retain
+	)
+
+	log.Printf("using state retention strategy: %s", retention)
+	switch retention {
+	case "accessed-cids":
+		tbs, ok := pst.Blockstore.(TracingBlockstore)
+		if !ok {
+			return fmt.Errorf("requested 'accessed-cids' state retention, but no tracing blockstore was present")
+		}
+
+		tbs.StartTracing()
+
+		preroot = root
+		applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
+			Preroot:    preroot,
+			Epoch:      execTs.Height(),
+			Message:    msg,
+			CircSupply: &circSupplyDetail.FilCirculating,
+			BaseFee:    &basefee,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to execute message: %w", err)
+		}
+		accessed := tbs.FinishTracing()
+		carWriter = func(w io.Writer) error {
+			return g.WriteCARIncluding(w, accessed, preroot, postroot)
+		}
+
+	case "accessed-actors":
+		log.Printf("calculating accessed actors")
+		// get actors accessed by message.
+		retain, err := g.GetAccessedActors(ctx, fapi, mcid)
+		if err != nil {
+			return fmt.Errorf("failed to calculate accessed actors: %w", err)
+		}
+		// also append the reward actor and the burnt funds actor.
+		retain = append(retain, reward.Address, builtin.BurntFundsActorAddr, init_.Address)
+		log.Printf("calculated accessed actors: %v", retain)
+
+		// get the masked state tree from the root,
+		preroot, err = g.GetMaskedStateTree(root, retain)
+		if err != nil {
+			return err
+		}
+		applyret, postroot, err = driver.ExecuteMessage(pst.Blockstore, conformance.ExecuteMessageParams{
+			Preroot:    preroot,
+			Epoch:      execTs.Height(),
+			Message:    msg,
+			CircSupply: &circSupplyDetail.FilCirculating,
+			BaseFee:    &basefee,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to execute message: %w", err)
+		}
+		carWriter = func(w io.Writer) error {
+			return g.WriteCAR(w, preroot, postroot)
+		}
+
+	default:
+		return fmt.Errorf("unknown state retention option: %s", retention)
+	}
+
+	log.Printf("message applied; preroot: %s, postroot: %s", preroot, postroot)
+	log.Println("performing sanity check on receipt")
+
+	// TODO sometimes this returns a nil receipt and no error ¯\_(ツ)_/¯
+	//  ex: https://filfox.info/en/message/bafy2bzacebpxw3yiaxzy2bako62akig46x3imji7fewszen6fryiz6nymu2b2
+	//  This code is lenient and skips receipt comparison in case of a nil receipt.
+	rec, err := fapi.StateGetReceipt(ctx, mcid, execTs.Key())
+	if err != nil {
+		return fmt.Errorf("failed to find receipt on chain: %w", err)
+	}
+	log.Printf("found receipt: %+v", rec)
+
+	// generate the schema receipt; if we got
+	var receipt *schema.Receipt
+	if rec != nil {
+		receipt = &schema.Receipt{
+			ExitCode:    int64(rec.ExitCode),
+			ReturnValue: rec.Return,
+			GasUsed:     rec.GasUsed,
+		}
+		reporter := new(conformance.LogReporter)
+		conformance.AssertMsgResult(reporter, receipt, applyret, "as locally executed")
+		if reporter.Failed() {
+			log.Println(color.RedString("receipt sanity check failed; aborting"))
+			return fmt.Errorf("vector generation aborted")
+		}
+		log.Println(color.GreenString("receipt sanity check succeeded"))
+	} else {
+		receipt = &schema.Receipt{
+			ExitCode:    int64(applyret.ExitCode),
+			ReturnValue: applyret.Return,
+			GasUsed:     applyret.GasUsed,
+		}
+		log.Println(color.YellowString("skipping receipts comparison; we got back a nil receipt from lotus"))
+	}
+
+	log.Println("generating vector")
+	msgBytes, err := msg.Serialize()
+	if err != nil {
+		return err
+	}
+
+	var (
+		out = new(bytes.Buffer)
+		gw  = gzip.NewWriter(out)
+	)
+	if err := carWriter(gw); err != nil {
+		return err
+	}
+	if err = gw.Flush(); err != nil {
+		return err
+	}
+	if err = gw.Close(); err != nil {
+		return err
+	}
+
+	version, err := fapi.Version(ctx)
+	if err != nil {
+		return err
+	}
+
+	ntwkName, err := fapi.StateNetworkName(ctx)
+	if err != nil {
+		return err
+	}
+
+	// Write out the test vector.
+	vector := schema.TestVector{
+		Class: schema.ClassMessage,
+		Meta: &schema.Metadata{
+			ID: opts.id,
+			// TODO need to replace schema.GenerationData with a more flexible
+			//  data structure that makes no assumption about the traceability
+			//  data that's being recorded; a flexible map[string]string
+			//  would do.
+			Gen: []schema.GenerationData{
+				{Source: fmt.Sprintf("network:%s", ntwkName)},
+				{Source: fmt.Sprintf("message:%s", msg.Cid().String())},
+				{Source: fmt.Sprintf("inclusion_tipset:%s", incTs.Key().String())},
+				{Source: fmt.Sprintf("execution_tipset:%s", execTs.Key().String())},
+				{Source: "github.com/filecoin-project/lotus", Version: version.String()}},
+		},
+		CAR: out.Bytes(),
+		Pre: &schema.Preconditions{
+			Epoch:      int64(execTs.Height()),
+			CircSupply: circSupply.Int,
+			BaseFee:    basefee.Int,
+			StateTree: &schema.StateTree{
+				RootCID: preroot,
+			},
+		},
+		ApplyMessages: []schema.Message{{Bytes: msgBytes}},
+		Post: &schema.Postconditions{
+			StateTree: &schema.StateTree{
+				RootCID: postroot,
+			},
+			Receipts: []*schema.Receipt{
+				{
+					ExitCode:    int64(applyret.ExitCode),
+					ReturnValue: applyret.Return,
+					GasUsed:     applyret.GasUsed,
+				},
+			},
+		},
+	}
+
+	output := io.WriteCloser(os.Stdout)
+	if file := opts.file; file != "" {
+		dir := filepath.Dir(file)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return fmt.Errorf("unable to create directory %s: %w", dir, err)
+		}
+		output, err = os.Create(file)
+		if err != nil {
+			return err
+		}
+		defer output.Close() //nolint:errcheck
+		defer log.Printf("wrote test vector to file: %s", file)
+	}
+
+	enc := json.NewEncoder(output)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(&vector); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// resolveFromChain queries the chain for the provided message, using the block CID to
+// speed up the query, if provided
+func resolveFromChain(ctx context.Context, api api.FullNode, mcid cid.Cid, block string) (msg *types.Message, execTs *types.TipSet, incTs *types.TipSet, err error) {
+	// Extract the full message.
+	msg, err = api.ChainGetMessage(ctx, mcid)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	log.Printf("found message with CID %s: %+v", mcid, msg)
+
+	if block == "" {
+		log.Printf("locating message in blockchain")
+
+		// Locate the message.
+		msgInfo, err := api.StateSearchMsg(ctx, mcid)
+		if err != nil {
+			return nil, nil, nil, fmt.Errorf("failed to locate message: %w", err)
+		}
+
+		log.Printf("located message at tipset %s (height: %d) with exit code: %s", msgInfo.TipSet, msgInfo.Height, msgInfo.Receipt.ExitCode)
+
+		execTs, incTs, err = fetchThisAndPrevTipset(ctx, api, msgInfo.TipSet)
+		return msg, execTs, incTs, err
+	}
+
+	bcid, err := cid.Decode(block)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	log.Printf("message inclusion block CID was provided; scanning around it: %s", bcid)
+
+	blk, err := api.ChainGetBlock(ctx, bcid)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to get block: %w", err)
+	}
+
+	// types.EmptyTSK hints to use the HEAD.
+	execTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height+1, types.EmptyTSK)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to get message execution tipset: %w", err)
+	}
+
+	// walk back from the execTs instead of HEAD, to save time.
+	incTs, err = api.ChainGetTipSetByHeight(ctx, blk.Height, execTs.Key())
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to get message inclusion tipset: %w", err)
+	}
+
+	return msg, execTs, incTs, nil
+}
+
+// fetchThisAndPrevTipset returns the full tipset identified by the key, as well
+// as the previous tipset. In the context of vector generation, the target
+// tipset is the one where a message was executed, and the previous tipset is
+// the one where the message was included.
+func fetchThisAndPrevTipset(ctx context.Context, api api.FullNode, target types.TipSetKey) (targetTs *types.TipSet, prevTs *types.TipSet, err error) {
+	// get the tipset on which this message was "executed" on.
+	// https://github.com/filecoin-project/lotus/issues/2847
+	targetTs, err = api.ChainGetTipSet(ctx, target)
+	if err != nil {
+		return nil, nil, err
+	}
+	// get the previous tipset, on which this message was mined,
+	// i.e. included on-chain.
+	prevTs, err = api.ChainGetTipSet(ctx, targetTs.Parents())
+	if err != nil {
+		return nil, nil, err
+	}
+	return targetTs, prevTs, nil
+}
+
+// findMsgAndPrecursors ranges through the canonical messages slice, locating
+// the target message and returning precursors in accordance to the supplied
+// mode.
+func findMsgAndPrecursors(mode string, target *types.Message, msgs []api.Message) (related []*types.Message, found bool, err error) {
+	// Range through canonicalised messages, selecting only the precursors based
+	// on selection mode.
+	for _, other := range msgs {
+		switch {
+		case mode == PrecursorSelectAll:
+			fallthrough
+		case mode == PrecursorSelectSender && other.Message.From == target.From:
+			related = append(related, other.Message)
+		}
+
+		// this message is the target; we're done.
+		if other.Cid == target.Cid() {
+			return related, true, nil
+		}
+	}
+
+	// this could happen because a block contained related messages, but not
+	// the target (that is, messages with a lower nonce, but ultimately not the
+	// target).
+	return related, false, nil
+}

cmd/tvx/extract_many.go

@@ -0,0 +1,232 @@
+package main
+
+import (
+	"context"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/filecoin-project/go-state-types/exitcode"
+	"github.com/hashicorp/go-multierror"
+	"github.com/urfave/cli/v2"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+)
+
+var extractManyFlags struct {
+	in      string
+	outdir  string
+	batchId string
+}
+
+var extractManyCmd = &cli.Command{
+	Name: "extract-many",
+	Description: `generate many test vectors by repeatedly calling tvx extract, using a csv file as input.
+
+   The CSV file must have a format just like the following:
+
+   message_cid,receiver_code,method_num,exit_code,height,block_cid,seq
+   bafy2bzacedvuvgpsnwq7i7kltfap6hnp7fdmzf6lr4w34zycjrthb3v7k6zi6,fil/1/account,0,0,67972,bafy2bzacebthpxzlk7zhlkz3jfzl4qw7mdoswcxlf3rkof3b4mbxfj3qzfk7w,1
+   bafy2bzacedwicofymn4imgny2hhbmcm4o5bikwnv3qqgohyx73fbtopiqlro6,fil/1/account,0,0,67860,bafy2bzacebj7beoxyzll522o6o76mt7von4psn3tlvunokhv4zhpwmfpipgti,2
+   ...
+
+   The first row MUST be a header row. At the bare minimum, those seven fields
+   must appear, in the order specified. Extra fields are accepted, but always
+   after these compulsory seven.
+`,
+	Action: runExtractMany,
+	Flags: []cli.Flag{
+		&repoFlag,
+		&cli.StringFlag{
+			Name:        "batch-id",
+			Usage:       "batch id; a four-digit left-zero-padded sequential number (e.g. 0041)",
+			Required:    true,
+			Destination: &extractManyFlags.batchId,
+		},
+		&cli.StringFlag{
+			Name:        "in",
+			Usage:       "path to input file (csv)",
+			Destination: &extractManyFlags.in,
+		},
+		&cli.StringFlag{
+			Name:        "outdir",
+			Usage:       "output directory",
+			Destination: &extractManyFlags.outdir,
+		},
+	},
+}
+
+func runExtractMany(c *cli.Context) error {
+	// LOTUS_DISABLE_VM_BUF disables what's called "VM state tree buffering",
+	// which stashes write operations in a BufferedBlockstore
+	// (https://github.com/filecoin-project/lotus/blob/b7a4dbb07fd8332b4492313a617e3458f8003b2a/lib/bufbstore/buf_bstore.go#L21)
+	// such that they're not written until the VM is actually flushed.
+	//
+	// For some reason, the standard behaviour was not working for me (raulk),
+	// and disabling it (such that the state transformations are written immediately
+	// to the blockstore) worked.
+	_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
+
+	ctx := context.Background()
+
+	// Make the API client.
+	fapi, closer, err := lcli.GetFullNodeAPI(c)
+	if err != nil {
+		return err
+	}
+	defer closer()
+
+	var (
+		in     = extractManyFlags.in
+		outdir = extractManyFlags.outdir
+	)
+
+	if in == "" {
+		return fmt.Errorf("input file not provided")
+	}
+
+	if outdir == "" {
+		return fmt.Errorf("output dir not provided")
+	}
+
+	// Open the CSV file for reading.
+	f, err := os.Open(in)
+	if err != nil {
+		return fmt.Errorf("could not open file %s: %w", in, err)
+	}
+
+	// Ensure the output directory exists.
+	if err := os.MkdirAll(outdir, 0755); err != nil {
+		return fmt.Errorf("could not create output dir %s: %w", outdir, err)
+	}
+
+	// Create a CSV reader and validate the header row.
+	reader := csv.NewReader(f)
+	if header, err := reader.Read(); err != nil {
+		return fmt.Errorf("failed to read header from csv: %w", err)
+	} else if l := len(header); l < 7 {
+		return fmt.Errorf("insufficient number of fields: %d", l)
+	} else if f := header[0]; f != "message_cid" {
+		return fmt.Errorf("csv sanity check failed: expected first field in header to be 'message_cid'; was: %s", f)
+	} else {
+		log.Println(color.GreenString("csv sanity check succeeded; header contains fields: %v", header))
+	}
+
+	var (
+		generated []string
+		merr      = new(multierror.Error)
+		retry     []extractOpts // to retry with 'canonical' precursor selection mode
+	)
+
+	// Read each row and extract the requested message.
+	for {
+		row, err := reader.Read()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return fmt.Errorf("failed to read row: %w", err)
+		}
+		var (
+			cid          = row[0]
+			actorcode    = row[1]
+			methodnumstr = row[2]
+			exitcodestr  = row[3]
+			_            = row[4]
+			block        = row[5]
+			seq          = row[6]
+
+			exit       int
+			methodnum  int
+			methodname string
+		)
+
+		// Parse the exit code.
+		if exit, err = strconv.Atoi(exitcodestr); err != nil {
+			return fmt.Errorf("invalid exitcode number: %d", exit)
+		}
+		// Parse the method number.
+		if methodnum, err = strconv.Atoi(methodnumstr); err != nil {
+			return fmt.Errorf("invalid method number: %s", methodnumstr)
+		}
+
+		// Lookup the method in actor method table.
+		if m, ok := ActorMethodTable[actorcode]; !ok {
+			return fmt.Errorf("unrecognized actor: %s", actorcode)
+		} else if methodnum >= len(m) {
+			return fmt.Errorf("unrecognized method number for actor %s: %d", actorcode, methodnum)
+		} else {
+			methodname = m[methodnum]
+		}
+
+		// exitcode string representations are of kind ErrType(0); strip out
+		// the number portion.
+		exitcodename := strings.Split(exitcode.ExitCode(exit).String(), "(")[0]
+		// replace the slashes in the actor code name with underscores.
+		actorcodename := strings.ReplaceAll(actorcode, "/", "_")
+
+		// Compute the ID of the vector.
+		id := fmt.Sprintf("ext-%s-%s-%s-%s-%s", extractManyFlags.batchId, actorcodename, methodname, exitcodename, seq)
+		// Vector filename, using a base of outdir.
+		file := filepath.Join(outdir, actorcodename, methodname, exitcodename, id) + ".json"
+
+		log.Println(color.YellowString("processing message cid with 'sender' precursor mode: %s", id))
+
+		opts := extractOpts{
+			id:        id,
+			block:     block,
+			class:     "message",
+			cid:       cid,
+			file:      file,
+			retain:    "accessed-cids",
+			precursor: PrecursorSelectSender,
+		}
+
+		if err := doExtract(ctx, fapi, opts); err != nil {
+			log.Println(color.RedString("failed to extract vector for message %s: %s; queuing for 'canonical' precursor selection", cid, err))
+			retry = append(retry, opts)
+			continue
+		}
+
+		log.Println(color.MagentaString("generated file: %s", file))
+
+		generated = append(generated, file)
+	}
+
+	log.Printf("extractions to try with canonical precursor selection mode: %d", len(retry))
+
+	for _, r := range retry {
+		log.Printf("retrying %s: %s", r.cid, r.id)
+
+		r.precursor = PrecursorSelectAll
+		if err := doExtract(ctx, fapi, r); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("failed to extract vector for message %s: %w", r.cid, err))
+			continue
+		}
+
+		log.Println(color.MagentaString("generated file: %s", r.file))
+		generated = append(generated, r.file)
+	}
+
+	if len(generated) == 0 {
+		log.Println("no files generated")
+	} else {
+		log.Println("files generated:")
+		for _, g := range generated {
+			log.Println(g)
+		}
+	}
+
+	if merr.ErrorOrNil() != nil {
+		log.Println(color.YellowString("done processing with errors: %v", merr))
+	} else {
+		log.Println(color.GreenString("done processing with no errors"))
+	}
+
+	return merr.ErrorOrNil()
+}

cmd/tvx/main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"log"
+	"os"
+	"sort"
+
+	"github.com/urfave/cli/v2"
+)
+
+// DefaultLotusRepoPath is where the fallback path where to look for a Lotus
+// client repo. It is expanded with mitchellh/go-homedir, so it'll work with all
+// OSes despite the Unix twiddle notation.
+const DefaultLotusRepoPath = "~/.lotus"
+
+var repoFlag = cli.StringFlag{
+	Name:      "repo",
+	EnvVars:   []string{"LOTUS_PATH"},
+	Value:     DefaultLotusRepoPath,
+	TakesFile: true,
+}
+
+func main() {
+	app := &cli.App{
+		Name: "tvx",
+		Description: `tvx is a tool for extracting and executing test vectors. It has three subcommands.
+
+   tvx extract extracts a test vector from a live network. It requires access to
+   a Filecoin client that exposes the standard JSON-RPC API endpoint. Only
+   message class test vectors are supported at this time.
+
+   tvx exec executes test vectors against Lotus. Either you can supply one in a
+   file, or many as an ndjson stdin stream.
+
+   tvx extract-many performs a batch extraction of many messages, supplied in a
+   CSV file. Refer to the help of that subcommand for more info.
+
+   SETTING THE JSON-RPC API ENDPOINT
+
+   You can set the JSON-RPC API endpoint through one of the following methods.
+
+   1. Directly set the API endpoint on the FULLNODE_API_INFO env variable.
+      The format is [token]:multiaddr, where token is optional for commands not
+      accessing privileged operations.
+
+   2. If you're running tvx against a local Lotus client, you can set the REPO
+      env variable to have the API endpoint and token extracted from the repo.
+      Alternatively, you can pass the --repo CLI flag.
+
+   3. Rely on the default fallback, which inspects ~/.lotus and extracts the
+      API endpoint string if the location is a Lotus repo.
+
+   tvx will apply these methods in the same order of precedence they're listed.
+`,
+		Usage: "tvx is a tool for extracting and executing test vectors",
+		Commands: []*cli.Command{
+			extractCmd,
+			execCmd,
+			extractManyCmd,
+		},
+	}
+
+	sort.Sort(cli.CommandsByName(app.Commands))
+	for _, c := range app.Commands {
+		sort.Sort(cli.FlagsByName(c.Flags))
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Fatal(err)
+	}
+}

cmd/tvx/state.go

@@ -0,0 +1,293 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/ipfs/go-cid"
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/ipld/go-car"
+	cbg "github.com/whyrusleeping/cbor-gen"
+
+	"github.com/filecoin-project/lotus/api"
+	init_ "github.com/filecoin-project/lotus/chain/actors/builtin/init"
+	"github.com/filecoin-project/lotus/chain/state"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+)
+
+// StateSurgeon is an object used to fetch and manipulate state.
+type StateSurgeon struct {
+	ctx    context.Context
+	api    api.FullNode
+	stores *Stores
+}
+
+// NewSurgeon returns a state surgeon, an object used to fetch and manipulate
+// state.
+func NewSurgeon(ctx context.Context, api api.FullNode, stores *Stores) *StateSurgeon {
+	return &StateSurgeon{
+		ctx:    ctx,
+		api:    api,
+		stores: stores,
+	}
+}
+
+// GetMaskedStateTree trims the state tree at the supplied tipset to contain
+// only the state of the actors in the retain set. It also "dives" into some
+// singleton system actors, like the init actor, to trim the state so as to
+// compute a minimal state tree. In the future, thid method will dive into
+// other system actors like the power actor and the market actor.
+func (sg *StateSurgeon) GetMaskedStateTree(previousRoot cid.Cid, retain []address.Address) (cid.Cid, error) {
+	// TODO: this will need to be parameterized on network version.
+	st, err := state.LoadStateTree(sg.stores.CBORStore, previousRoot)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	initActor, initState, err := sg.loadInitActor(st)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	err = sg.retainInitEntries(initState, retain)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	err = sg.saveInitActor(initActor, initState, st)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	// resolve all addresses to ID addresses.
+	resolved, err := sg.resolveAddresses(retain, initState)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	st, err = sg.transplantActors(st, resolved)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	root, err := st.Flush(sg.ctx)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	return root, nil
+}
+
+// GetAccessedActors identifies the actors that were accessed during the
+// execution of a message.
+func (sg *StateSurgeon) GetAccessedActors(ctx context.Context, a api.FullNode, mid cid.Cid) ([]address.Address, error) {
+	log.Printf("calculating accessed actors during execution of message: %s", mid)
+	msgInfo, err := a.StateSearchMsg(ctx, mid)
+	if err != nil {
+		return nil, err
+	}
+	if msgInfo == nil {
+		return nil, fmt.Errorf("message info is nil")
+	}
+
+	msgObj, err := a.ChainGetMessage(ctx, mid)
+	if err != nil {
+		return nil, err
+	}
+
+	ts, err := a.ChainGetTipSet(ctx, msgInfo.TipSet)
+	if err != nil {
+		return nil, err
+	}
+
+	trace, err := a.StateCall(ctx, msgObj, ts.Parents())
+	if err != nil {
+		return nil, fmt.Errorf("could not replay msg: %w", err)
+	}
+
+	accessed := make(map[address.Address]struct{})
+
+	var recur func(trace *types.ExecutionTrace)
+	recur = func(trace *types.ExecutionTrace) {
+		accessed[trace.Msg.To] = struct{}{}
+		accessed[trace.Msg.From] = struct{}{}
+		for i := range trace.Subcalls {
+			recur(&trace.Subcalls[i])
+		}
+	}
+	recur(&trace.ExecutionTrace)
+
+	ret := make([]address.Address, 0, len(accessed))
+	for k := range accessed {
+		ret = append(ret, k)
+	}
+
+	return ret, nil
+}
+
+// WriteCAR recursively writes the tree referenced by the root as a CAR into the
+// supplied io.Writer.
+func (sg *StateSurgeon) WriteCAR(w io.Writer, roots ...cid.Cid) error {
+	carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
+		for _, link := range nd.Links() {
+			if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
+				continue
+			}
+			out = append(out, link)
+		}
+		return out, nil
+	}
+	return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn)
+}
+
+// WriteCARIncluding writes a CAR including only the CIDs that are listed in
+// the include set. This leads to an intentially sparse tree with dangling links.
+func (sg *StateSurgeon) WriteCARIncluding(w io.Writer, include map[cid.Cid]struct{}, roots ...cid.Cid) error {
+	carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
+		for _, link := range nd.Links() {
+			if _, ok := include[link.Cid]; !ok {
+				continue
+			}
+			if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
+				continue
+			}
+			out = append(out, link)
+		}
+		return out, nil
+	}
+	return car.WriteCarWithWalker(sg.ctx, sg.stores.DAGService, roots, w, carWalkFn)
+}
+
+// transplantActors plucks the state from the supplied actors at the given
+// tipset, and places it into the supplied state map.
+func (sg *StateSurgeon) transplantActors(src *state.StateTree, pluck []address.Address) (*state.StateTree, error) {
+	log.Printf("transplanting actor states: %v", pluck)
+
+	dst, err := state.NewStateTree(sg.stores.CBORStore, src.Version())
+	if err != nil {
+		return nil, err
+	}
+
+	for _, a := range pluck {
+		actor, err := src.GetActor(a)
+		if err != nil {
+			return nil, fmt.Errorf("get actor %s failed: %w", a, err)
+		}
+
+		err = dst.SetActor(a, actor)
+		if err != nil {
+			return nil, err
+		}
+
+		// recursive copy of the actor state.
+		err = vm.Copy(context.TODO(), sg.stores.Blockstore, sg.stores.Blockstore, actor.Head)
+		if err != nil {
+			return nil, err
+		}
+
+		actorState, err := sg.api.ChainReadObj(sg.ctx, actor.Head)
+		if err != nil {
+			return nil, err
+		}
+
+		cid, err := sg.stores.CBORStore.Put(sg.ctx, &cbg.Deferred{Raw: actorState})
+		if err != nil {
+			return nil, err
+		}
+
+		if cid != actor.Head {
+			panic("mismatched cids")
+		}
+	}
+
+	return dst, nil
+}
+
+// saveInitActor saves the state of the init actor to the provided state map.
+func (sg *StateSurgeon) saveInitActor(initActor *types.Actor, initState init_.State, st *state.StateTree) error {
+	log.Printf("saving init actor into state tree")
+
+	// Store the state of the init actor.
+	cid, err := sg.stores.CBORStore.Put(sg.ctx, initState)
+	if err != nil {
+		return err
+	}
+	actor := *initActor
+	actor.Head = cid
+
+	err = st.SetActor(init_.Address, &actor)
+	if err != nil {
+		return err
+	}
+
+	cid, _ = st.Flush(sg.ctx)
+	log.Printf("saved init actor into state tree; new root: %s", cid)
+	return nil
+}
+
+// retainInitEntries takes an old init actor state, and retains only the
+// entries in the retain set, returning a new init actor state.
+func (sg *StateSurgeon) retainInitEntries(state init_.State, retain []address.Address) error {
+	log.Printf("retaining init actor entries for addresses: %v", retain)
+
+	m := make(map[address.Address]struct{}, len(retain))
+	for _, a := range retain {
+		m[a] = struct{}{}
+	}
+
+	var remove []address.Address
+	_ = state.ForEachActor(func(id abi.ActorID, address address.Address) error {
+		if _, ok := m[address]; !ok {
+			remove = append(remove, address)
+		}
+		return nil
+	})
+
+	err := state.Remove(remove...)
+	log.Printf("new init actor state: %+v", state)
+	return err
+}
+
+// resolveAddresses resolved the requested addresses from the provided
+// InitActor state, returning a slice of length len(orig), where each index
+// contains the resolved address.
+func (sg *StateSurgeon) resolveAddresses(orig []address.Address, ist init_.State) (ret []address.Address, err error) {
+	log.Printf("resolving addresses: %v", orig)
+
+	ret = make([]address.Address, len(orig))
+	for i, addr := range orig {
+		resolved, found, err := ist.ResolveAddress(addr)
+		if err != nil {
+			return nil, err
+		}
+		if !found {
+			return nil, fmt.Errorf("address not found: %s", addr)
+		}
+		ret[i] = resolved
+	}
+
+	log.Printf("resolved addresses: %v", ret)
+	return ret, nil
+}
+
+// loadInitActor loads the init actor state from a given tipset.
+func (sg *StateSurgeon) loadInitActor(st *state.StateTree) (*types.Actor, init_.State, error) {
+	actor, err := st.GetActor(init_.Address)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	initState, err := init_.Load(sg.stores.ADTStore, actor)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	log.Printf("loaded init actor state: %+v", initState)
+
+	return actor, initState, nil
+}

cmd/tvx/stores.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"context"
+	"log"
+	"sync"
+
+	"github.com/fatih/color"
+	dssync "github.com/ipfs/go-datastore/sync"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/lib/blockstore"
+
+	"github.com/filecoin-project/lotus/chain/actors/adt"
+
+	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	ds "github.com/ipfs/go-datastore"
+	exchange "github.com/ipfs/go-ipfs-exchange-interface"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-merkledag"
+)
+
+// Stores is a collection of the different stores and services that are needed
+// to deal with the data layer of Filecoin, conveniently interlinked with one
+// another.
+type Stores struct {
+	CBORStore    cbor.IpldStore
+	ADTStore     adt.Store
+	Datastore    ds.Batching
+	Blockstore   blockstore.Blockstore
+	BlockService blockservice.BlockService
+	Exchange     exchange.Interface
+	DAGService   format.DAGService
+}
+
+// NewProxyingStores is a set of Stores backed by a proxying Blockstore that
+// proxies Get requests for unknown CIDs to a Filecoin node, via the
+// ChainReadObj RPC.
+func NewProxyingStores(ctx context.Context, api api.FullNode) *Stores {
+	ds := dssync.MutexWrap(ds.NewMapDatastore())
+	bs := &proxyingBlockstore{
+		ctx:        ctx,
+		api:        api,
+		Blockstore: blockstore.NewBlockstore(ds),
+	}
+	return NewStores(ctx, ds, bs)
+}
+
+// NewStores creates a non-proxying set of Stores.
+func NewStores(ctx context.Context, ds ds.Batching, bs blockstore.Blockstore) *Stores {
+	var (
+		cborstore = cbor.NewCborStore(bs)
+		offl      = offline.Exchange(bs)
+		blkserv   = blockservice.New(bs, offl)
+		dserv     = merkledag.NewDAGService(blkserv)
+	)
+
+	return &Stores{
+		CBORStore:    cborstore,
+		ADTStore:     adt.WrapStore(ctx, cborstore),
+		Datastore:    ds,
+		Blockstore:   bs,
+		Exchange:     offl,
+		BlockService: blkserv,
+		DAGService:   dserv,
+	}
+}
+
+// TracingBlockstore is a Blockstore trait that records CIDs that were accessed
+// through Get.
+type TracingBlockstore interface {
+	// StartTracing starts tracing CIDs accessed through the this Blockstore.
+	StartTracing()
+
+	// FinishTracing finishes tracing accessed CIDs, and returns a map of the
+	// CIDs that were traced.
+	FinishTracing() map[cid.Cid]struct{}
+}
+
+// proxyingBlockstore is a Blockstore wrapper that fetches unknown CIDs from
+// a Filecoin node via JSON-RPC.
+type proxyingBlockstore struct {
+	ctx context.Context
+	api api.FullNode
+
+	lk      sync.RWMutex
+	tracing bool
+	traced  map[cid.Cid]struct{}
+
+	blockstore.Blockstore
+}
+
+var _ TracingBlockstore = (*proxyingBlockstore)(nil)
+
+func (pb *proxyingBlockstore) StartTracing() {
+	pb.lk.Lock()
+	pb.tracing = true
+	pb.traced = map[cid.Cid]struct{}{}
+	pb.lk.Unlock()
+}
+
+func (pb *proxyingBlockstore) FinishTracing() map[cid.Cid]struct{} {
+	pb.lk.Lock()
+	ret := pb.traced
+	pb.tracing = false
+	pb.traced = map[cid.Cid]struct{}{}
+	pb.lk.Unlock()
+	return ret
+}
+
+func (pb *proxyingBlockstore) Get(cid cid.Cid) (blocks.Block, error) {
+	pb.lk.RLock()
+	if pb.tracing {
+		pb.traced[cid] = struct{}{}
+	}
+	pb.lk.RUnlock()
+
+	if block, err := pb.Blockstore.Get(cid); err == nil {
+		return block, err
+	}
+
+	log.Println(color.CyanString("fetching cid via rpc: %v", cid))
+	item, err := pb.api.ChainReadObj(pb.ctx, cid)
+	if err != nil {
+		return nil, err
+	}
+	block, err := blocks.NewBlockWithCid(item, cid)
+	if err != nil {
+		return nil, err
+	}
+
+	err = pb.Blockstore.Put(block)
+	if err != nil {
+		return nil, err
+	}
+
+	return block, nil
+}

conformance/corpus_test.go

@@ -0,0 +1,133 @@
+package conformance
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/filecoin-project/test-vectors/schema"
+)
+
+const (
+	// EnvSkipConformance, if 1, skips the conformance test suite.
+	EnvSkipConformance = "SKIP_CONFORMANCE"
+
+	// EnvCorpusRootDir is the name of the environment variable where the path
+	// to an alternative corpus location can be provided.
+	//
+	// The default is defaultCorpusRoot.
+	EnvCorpusRootDir = "CORPUS_DIR"
+
+	// defaultCorpusRoot is the directory where the test vector corpus is hosted.
+	// It is mounted on the Lotus repo as a git submodule.
+	//
+	// When running this test, the corpus root can be overridden through the
+	// -conformance.corpus CLI flag to run an alternate corpus.
+	defaultCorpusRoot = "../extern/test-vectors/corpus"
+)
+
+// ignore is a set of paths relative to root to skip.
+var ignore = map[string]struct{}{
+	".git":        {},
+	"schema.json": {},
+}
+
+// TestConformance is the entrypoint test that runs all test vectors found
+// in the corpus root directory.
+//
+// It locates all json files via a recursive walk, skipping over the ignore set,
+// as well as files beginning with _. It parses each file as a test vector, and
+// runs it via the Driver.
+func TestConformance(t *testing.T) {
+	if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" {
+		t.SkipNow()
+	}
+	// corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag,
+	// falling back to defaultCorpusRoot if not provided.
+	corpusRoot := defaultCorpusRoot
+	if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" {
+		corpusRoot = dir
+	}
+
+	var vectors []string
+	err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		filename := filepath.Base(path)
+		rel, err := filepath.Rel(corpusRoot, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if _, ok := ignore[rel]; ok {
+			// skip over using the right error.
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if info.IsDir() {
+			// dive into directories.
+			return nil
+		}
+		if filepath.Ext(path) != ".json" {
+			// skip if not .json.
+			return nil
+		}
+		if ignored := strings.HasPrefix(filename, "_"); ignored {
+			// ignore files starting with _.
+			t.Logf("ignoring: %s", rel)
+			return nil
+		}
+		vectors = append(vectors, rel)
+		return nil
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(vectors) == 0 {
+		t.Fatalf("no test vectors found")
+	}
+
+	// Run a test for each vector.
+	for _, v := range vectors {
+		path := filepath.Join(corpusRoot, v)
+		raw, err := ioutil.ReadFile(path)
+		if err != nil {
+			t.Fatalf("failed to read test raw file: %s", path)
+		}
+
+		var vector schema.TestVector
+		err = json.Unmarshal(raw, &vector)
+		if err != nil {
+			t.Errorf("failed to parse test vector %s: %s; skipping", path, err)
+			continue
+		}
+
+		t.Run(v, func(t *testing.T) {
+			for _, h := range vector.Hints {
+				if h == schema.HintIncorrect {
+					t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID)
+					t.SkipNow()
+				}
+			}
+
+			// dispatch the execution depending on the vector class.
+			switch vector.Class {
+			case "message":
+				ExecuteMessageVector(t, &vector)
+			case "tipset":
+				ExecuteTipsetVector(t, &vector)
+			default:
+				t.Fatalf("test vector class not supported: %s", vector.Class)
+			}
+		})
+	}
+}

conformance/driver.go

@@ -2,9 +2,9 @@ package conformance
 
 import (
 	"context"
+	"os"
 
-	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/lotus/chain/state"
-
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -14,6 +14,7 @@ import (
 	"github.com/filecoin-project/lotus/lib/blockstore"
 
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
 
 	"github.com/filecoin-project/test-vectors/schema"
 
@@ -24,18 +25,36 @@ import (
 )
 
 var (
-	// BaseFee to use in the VM.
+	// DefaultCirculatingSupply is the fallback circulating supply returned by
-	// TODO make parametrisable through vector.
+	// the driver's CircSupplyCalculator function, used if the vector specifies
-	BaseFee = abi.NewTokenAmount(100)
+	// no circulating supply.
+	DefaultCirculatingSupply = types.TotalFilecoinInt
+
+	// DefaultBaseFee to use in the VM, if one is not supplied in the vector.
+	DefaultBaseFee = abi.NewTokenAmount(100)
 )
 
 type Driver struct {
 	ctx      context.Context
 	selector schema.Selector
+	vmFlush  bool
 }
 
-func NewDriver(ctx context.Context, selector schema.Selector) *Driver {
+type DriverOpts struct {
-	return &Driver{ctx: ctx, selector: selector}
+	// DisableVMFlush, when true, avoids calling VM.Flush(), forces a blockstore
+	// recursive copy, from the temporary buffer blockstore, to the real
+	// system's blockstore. Disabling VM flushing is useful when extracting test
+	// vectors and trimming state, as we don't want to force an accidental
+	// deep copy of the state tree.
+	//
+	// Disabling VM flushing almost always should go hand-in-hand with
+	// LOTUS_DISABLE_VM_BUF=iknowitsabadidea. That way, state tree writes are
+	// immediately committed to the blockstore.
+	DisableVMFlush bool
+}
+
+func NewDriver(ctx context.Context, selector schema.Selector, opts DriverOpts) *Driver {
+	return &Driver{ctx: ctx, selector: selector, vmFlush: !opts.DisableVMFlush}
 }
 
 type ExecuteTipsetResult struct {
@@ -120,18 +139,46 @@ func (d *Driver) ExecuteTipset(bs blockstore.Blockstore, ds ds.Batching, preroot
 	return ret, nil
 }
 
+type ExecuteMessageParams struct {
+	Preroot    cid.Cid
+	Epoch      abi.ChainEpoch
+	Message    *types.Message
+	CircSupply *abi.TokenAmount
+	BaseFee    *abi.TokenAmount
+}
+
 // ExecuteMessage executes a conformance test vector message in a temporary VM.
-func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch abi.ChainEpoch, msg *types.Message) (*vm.ApplyRet, cid.Cid, error) {
+func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, params ExecuteMessageParams) (*vm.ApplyRet, cid.Cid, error) {
-	// dummy state manager; only to reference the GetNetworkVersion method, which does not depend on state.
+	if !d.vmFlush {
+		// do not flush the VM, just the state tree; this should be used with
+		// LOTUS_DISABLE_VM_BUF enabled, so writes will anyway be visible.
+		_ = os.Setenv("LOTUS_DISABLE_VM_BUF", "iknowitsabadidea")
+	}
+
+	basefee := DefaultBaseFee
+	if params.BaseFee != nil {
+		basefee = *params.BaseFee
+	}
+
+	circSupply := DefaultCirculatingSupply
+	if params.CircSupply != nil {
+		circSupply = *params.CircSupply
+	}
+
+	// dummy state manager; only to reference the GetNetworkVersion method,
+	// which does not depend on state.
 	sm := new(stmgr.StateManager)
+
 	vmOpts := &vm.VMOpts{
-		StateBase:      preroot,
+		StateBase: params.Preroot,
-		Epoch:          epoch,
+		Epoch:     params.Epoch,
 		Rand:      &testRand{}, // TODO always succeeds; need more flexibility.
 		Bstore:    bs,
 		Syscalls:  mkFakedSigSyscalls(vm.Syscalls(ffiwrapper.ProofVerifier)), // TODO always succeeds; need more flexibility.
-		CircSupplyCalc: nil,
+		CircSupplyCalc: func(_ context.Context, _ abi.ChainEpoch, _ *state.StateTree) (abi.TokenAmount, error) {
-		BaseFee:        BaseFee,
+			return circSupply, nil
+		},
+		BaseFee:     basefee,
 		NtwkVersion: sm.GetNtwkVersion,
 	}
 
@@ -149,12 +196,20 @@ func (d *Driver) ExecuteMessage(bs blockstore.Blockstore, preroot cid.Cid, epoch
 
 	lvm.SetInvoker(invoker)
 
-	ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(msg))
+	ret, err := lvm.ApplyMessage(d.ctx, toChainMsg(params.Message))
 	if err != nil {
 		return nil, cid.Undef, err
 	}
 
-	root, err := lvm.Flush(d.ctx)
+	var root cid.Cid
+	if d.vmFlush {
+		// flush the VM, committing the state tree changes and forcing a
+		// recursive copoy from the temporary blcokstore to the real blockstore.
+		root, err = lvm.Flush(d.ctx)
+	} else {
+		root, err = lvm.StateTree().(*state.StateTree).Flush(d.ctx)
+	}
+
 	return ret, root, err
 }
 

conformance/reporter.go

@@ -0,0 +1,62 @@
+package conformance
+
+import (
+	"log"
+	"os"
+	"sync/atomic"
+	"testing"
+
+	"github.com/fatih/color"
+)
+
+// Reporter is a contains a subset of the testing.T methods, so that the
+// Execute* functions in this package can be used inside or outside of
+// go test runs.
+type Reporter interface {
+	Helper()
+
+	Log(args ...interface{})
+	Errorf(format string, args ...interface{})
+	Fatalf(format string, args ...interface{})
+	Logf(format string, args ...interface{})
+	FailNow()
+	Failed() bool
+}
+
+var _ Reporter = (*testing.T)(nil)
+
+// LogReporter wires the Reporter methods to the log package. It is appropriate
+// to use when calling the Execute* functions from a standalone CLI program.
+type LogReporter struct {
+	failed int32
+}
+
+var _ Reporter = (*LogReporter)(nil)
+
+func (*LogReporter) Helper() {}
+
+func (*LogReporter) Log(args ...interface{}) {
+	log.Println(args...)
+}
+
+func (*LogReporter) Logf(format string, args ...interface{}) {
+	log.Printf(format, args...)
+}
+
+func (*LogReporter) FailNow() {
+	os.Exit(1)
+}
+
+func (l *LogReporter) Failed() bool {
+	return atomic.LoadInt32(&l.failed) == 1
+}
+
+func (l *LogReporter) Errorf(format string, args ...interface{}) {
+	atomic.StoreInt32(&l.failed, 1)
+	log.Println(color.HiRedString("❌ "+format, args...))
+}
+
+func (l *LogReporter) Fatalf(format string, args ...interface{}) {
+	atomic.StoreInt32(&l.failed, 1)
+	log.Fatal(color.HiRedString("❌ "+format, args...))
+}

conformance/runner.go

@@ -0,0 +1,272 @@
+package conformance
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+
+	"github.com/fatih/color"
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/big"
+	"github.com/filecoin-project/go-state-types/exitcode"
+	"github.com/filecoin-project/test-vectors/schema"
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	ds "github.com/ipfs/go-datastore"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-merkledag"
+	"github.com/ipld/go-car"
+
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
+	"github.com/filecoin-project/lotus/lib/blockstore"
+)
+
+// ExecuteMessageVector executes a message-class test vector.
+func ExecuteMessageVector(r Reporter, vector *schema.TestVector) {
+	var (
+		ctx   = context.Background()
+		epoch = vector.Pre.Epoch
+		root  = vector.Pre.StateTree.RootCID
+	)
+
+	// Load the CAR into a new temporary Blockstore.
+	bs, err := LoadVectorCAR(vector.CAR)
+	if err != nil {
+		r.Fatalf("failed to load the vector CAR: %w", err)
+	}
+
+	// Create a new Driver.
+	driver := NewDriver(ctx, vector.Selector, DriverOpts{DisableVMFlush: true})
+
+	var circSupply *abi.TokenAmount
+	if cs := vector.Pre.CircSupply; cs != nil {
+		ta := big.NewFromGo(cs)
+		circSupply = &ta
+	}
+
+	var basefee *abi.TokenAmount
+	if bf := vector.Pre.BaseFee; bf != nil {
+		ta := big.NewFromGo(bf)
+		basefee = &ta
+	}
+
+	// Apply every message.
+	for i, m := range vector.ApplyMessages {
+		msg, err := types.DecodeMessage(m.Bytes)
+		if err != nil {
+			r.Fatalf("failed to deserialize message: %s", err)
+		}
+
+		// add an epoch if one's set.
+		if m.Epoch != nil {
+			epoch = *m.Epoch
+		}
+
+		// Execute the message.
+		var ret *vm.ApplyRet
+		ret, root, err = driver.ExecuteMessage(bs, ExecuteMessageParams{
+			Preroot:    root,
+			Epoch:      abi.ChainEpoch(epoch),
+			Message:    msg,
+			CircSupply: circSupply,
+			BaseFee:    basefee,
+		})
+		if err != nil {
+			r.Fatalf("fatal failure when executing message: %s", err)
+		}
+
+		// Assert that the receipt matches what the test vector expects.
+		AssertMsgResult(r, vector.Post.Receipts[i], ret, strconv.Itoa(i))
+	}
+
+	// Once all messages are applied, assert that the final state root matches
+	// the expected postcondition root.
+	if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
+		r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual)
+		dumpThreeWayStateDiff(r, vector, bs, root)
+		r.FailNow()
+	}
+}
+
+// ExecuteTipsetVector executes a tipset-class test vector.
+func ExecuteTipsetVector(r Reporter, vector *schema.TestVector) {
+	var (
+		ctx       = context.Background()
+		prevEpoch = vector.Pre.Epoch
+		root      = vector.Pre.StateTree.RootCID
+		tmpds     = ds.NewMapDatastore()
+	)
+
+	// Load the vector CAR into a new temporary Blockstore.
+	bs, err := LoadVectorCAR(vector.CAR)
+	if err != nil {
+		r.Fatalf("failed to load the vector CAR: %w", err)
+	}
+
+	// Create a new Driver.
+	driver := NewDriver(ctx, vector.Selector, DriverOpts{})
+
+	// Apply every tipset.
+	var receiptsIdx int
+	for i, ts := range vector.ApplyTipsets {
+		ts := ts // capture
+		ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts)
+		if err != nil {
+			r.Fatalf("failed to apply tipset %d message: %s", i, err)
+		}
+
+		for j, v := range ret.AppliedResults {
+			AssertMsgResult(r, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i))
+			receiptsIdx++
+		}
+
+		// Compare the receipts root.
+		if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual {
+			r.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual)
+		}
+
+		prevEpoch = ts.Epoch
+		root = ret.PostStateRoot
+	}
+
+	// Once all messages are applied, assert that the final state root matches
+	// the expected postcondition root.
+	if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
+		r.Errorf("wrong post root cid; expected %v, but got %v", expected, actual)
+		dumpThreeWayStateDiff(r, vector, bs, root)
+		r.FailNow()
+	}
+}
+
+// AssertMsgResult compares a message result. It takes the expected receipt
+// encoded in the vector, the actual receipt returned by Lotus, and a message
+// label to log in the assertion failure message to facilitate debugging.
+func AssertMsgResult(r Reporter, expected *schema.Receipt, actual *vm.ApplyRet, label string) {
+	r.Helper()
+
+	if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual {
+		r.Errorf("exit code of msg %s did not match; expected: %s, got: %s", label, expected, actual)
+	}
+	if expected, actual := expected.GasUsed, actual.GasUsed; expected != actual {
+		r.Errorf("gas used of msg %s did not match; expected: %d, got: %d", label, expected, actual)
+	}
+	if expected, actual := []byte(expected.ReturnValue), actual.Return; !bytes.Equal(expected, actual) {
+		r.Errorf("return value of msg %s did not match; expected: %s, got: %s", label, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual))
+	}
+}
+
+func dumpThreeWayStateDiff(r Reporter, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) {
+	// check if statediff exists; if not, skip.
+	if err := exec.Command("statediff", "--help").Run(); err != nil {
+		r.Log("could not dump 3-way state tree diff upon test failure: statediff command not found")
+		r.Log("install statediff with:")
+		r.Log("$ git clone https://github.com/filecoin-project/statediff.git")
+		r.Log("$ cd statediff")
+		r.Log("$ go generate ./...")
+		r.Log("$ go install ./cmd/statediff")
+		return
+	}
+
+	tmpCar, err := writeStateToTempCAR(bs,
+		vector.Pre.StateTree.RootCID,
+		vector.Post.StateTree.RootCID,
+		actual,
+	)
+	if err != nil {
+		r.Fatalf("failed to write temporary state CAR: %s", err)
+	}
+	defer os.RemoveAll(tmpCar) //nolint:errcheck
+
+	color.NoColor = false // enable colouring.
+
+	var (
+		a  = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state")
+		b  = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state")
+		c  = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state")
+		d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]")
+		d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]")
+		d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]")
+	)
+
+	printDiff := func(left, right cid.Cid) {
+		cmd := exec.Command("statediff", "car", "--file", tmpCar, left.String(), right.String())
+		b, err := cmd.CombinedOutput()
+		if err != nil {
+			r.Fatalf("statediff failed: %s", err)
+		}
+		r.Log(string(b))
+	}
+
+	bold := color.New(color.Bold).SprintfFunc()
+
+	// run state diffs.
+	r.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c))
+
+	r.Log(bold("--- %s left: %s; right: %s ---", d1, a, b))
+	printDiff(vector.Post.StateTree.RootCID, actual)
+
+	r.Log(bold("--- %s left: %s; right: %s ---", d2, c, b))
+	printDiff(vector.Pre.StateTree.RootCID, actual)
+
+	r.Log(bold("--- %s left: %s; right: %s ---", d3, c, a))
+	printDiff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID)
+}
+
+// writeStateToTempCAR writes the provided roots to a temporary CAR that'll be
+// cleaned up via t.Cleanup(). It returns the full path of the temp file.
+func writeStateToTempCAR(bs blockstore.Blockstore, roots ...cid.Cid) (string, error) {
+	tmp, err := ioutil.TempFile("", "lotus-tests-*.car")
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp file to dump CAR for diffing: %w", err)
+	}
+
+	carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
+		for _, link := range nd.Links() {
+			if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
+				continue
+			}
+			out = append(out, link)
+		}
+		return out, nil
+	}
+
+	var (
+		offl    = offline.Exchange(bs)
+		blkserv = blockservice.New(bs, offl)
+		dserv   = merkledag.NewDAGService(blkserv)
+	)
+
+	err = car.WriteCarWithWalker(context.Background(), dserv, roots, tmp, carWalkFn)
+	if err != nil {
+		return "", fmt.Errorf("failed to dump CAR for diffing: %w", err)
+	}
+	_ = tmp.Close()
+	return tmp.Name(), nil
+}
+
+func LoadVectorCAR(vectorCAR schema.Base64EncodedBytes) (blockstore.Blockstore, error) {
+	bs := blockstore.NewTemporary()
+
+	// Read the base64-encoded CAR from the vector, and inflate the gzip.
+	buf := bytes.NewReader(vectorCAR)
+	r, err := gzip.NewReader(buf)
+	if err != nil {
+		return nil, fmt.Errorf("failed to inflate gzipped CAR: %s", err)
+	}
+	defer r.Close() // nolint
+
+	// Load the CAR embedded in the test vector into the Blockstore.
+	_, err = car.LoadCar(bs, r)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load state tree car from test vector: %s", err)
+	}
+	return bs, nil
+}

conformance/runner_test.go

@@ -1,376 +0,0 @@
-package conformance
-
-import (
-	"bytes"
-	"compress/gzip"
-	"context"
-	"encoding/base64"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"testing"
-
-	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/filecoin-project/go-state-types/exitcode"
-	"github.com/ipfs/go-blockservice"
-	"github.com/ipfs/go-cid"
-	ds "github.com/ipfs/go-datastore"
-	offline "github.com/ipfs/go-ipfs-exchange-offline"
-	format "github.com/ipfs/go-ipld-format"
-	"github.com/ipfs/go-merkledag"
-
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/vm"
-	"github.com/filecoin-project/lotus/lib/blockstore"
-
-	"github.com/filecoin-project/test-vectors/schema"
-
-	"github.com/fatih/color"
-	"github.com/ipld/go-car"
-)
-
-const (
-	// EnvSkipConformance, if 1, skips the conformance test suite.
-	EnvSkipConformance = "SKIP_CONFORMANCE"
-
-	// EnvCorpusRootDir is the name of the environment variable where the path
-	// to an alternative corpus location can be provided.
-	//
-	// The default is defaultCorpusRoot.
-	EnvCorpusRootDir = "CORPUS_DIR"
-
-	// defaultCorpusRoot is the directory where the test vector corpus is hosted.
-	// It is mounted on the Lotus repo as a git submodule.
-	//
-	// When running this test, the corpus root can be overridden through the
-	// -conformance.corpus CLI flag to run an alternate corpus.
-	defaultCorpusRoot = "../extern/test-vectors/corpus"
-)
-
-// ignore is a set of paths relative to root to skip.
-var ignore = map[string]struct{}{
-	".git":        {},
-	"schema.json": {},
-}
-
-// TestConformance is the entrypoint test that runs all test vectors found
-// in the corpus root directory.
-//
-// It locates all json files via a recursive walk, skipping over the ignore set,
-// as well as files beginning with _. It parses each file as a test vector, and
-// runs it via the Driver.
-func TestConformance(t *testing.T) {
-	if skip := strings.TrimSpace(os.Getenv(EnvSkipConformance)); skip == "1" {
-		t.SkipNow()
-	}
-	// corpusRoot is the effective corpus root path, taken from the `-conformance.corpus` CLI flag,
-	// falling back to defaultCorpusRoot if not provided.
-	corpusRoot := defaultCorpusRoot
-	if dir := strings.TrimSpace(os.Getenv(EnvCorpusRootDir)); dir != "" {
-		corpusRoot = dir
-	}
-
-	var vectors []string
-	err := filepath.Walk(corpusRoot+"/", func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		filename := filepath.Base(path)
-		rel, err := filepath.Rel(corpusRoot, path)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if _, ok := ignore[rel]; ok {
-			// skip over using the right error.
-			if info.IsDir() {
-				return filepath.SkipDir
-			}
-			return nil
-		}
-		if info.IsDir() {
-			// dive into directories.
-			return nil
-		}
-		if filepath.Ext(path) != ".json" {
-			// skip if not .json.
-			return nil
-		}
-		if ignored := strings.HasPrefix(filename, "_"); ignored {
-			// ignore files starting with _.
-			t.Logf("ignoring: %s", rel)
-			return nil
-		}
-		vectors = append(vectors, rel)
-		return nil
-	})
-
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if len(vectors) == 0 {
-		t.Fatalf("no test vectors found")
-	}
-
-	// Run a test for each vector.
-	for _, v := range vectors {
-		path := filepath.Join(corpusRoot, v)
-		raw, err := ioutil.ReadFile(path)
-		if err != nil {
-			t.Fatalf("failed to read test raw file: %s", path)
-		}
-
-		var vector schema.TestVector
-		err = json.Unmarshal(raw, &vector)
-		if err != nil {
-			t.Errorf("failed to parse test vector %s: %s; skipping", path, err)
-			continue
-		}
-
-		t.Run(v, func(t *testing.T) {
-			for _, h := range vector.Hints {
-				if h == schema.HintIncorrect {
-					t.Logf("skipping vector marked as incorrect: %s", vector.Meta.ID)
-					t.SkipNow()
-				}
-			}
-
-			// dispatch the execution depending on the vector class.
-			switch vector.Class {
-			case "message":
-				executeMessageVector(t, &vector)
-			case "tipset":
-				executeTipsetVector(t, &vector)
-			default:
-				t.Fatalf("test vector class not supported: %s", vector.Class)
-			}
-		})
-	}
-}
-
-// executeMessageVector executes a message-class test vector.
-func executeMessageVector(t *testing.T, vector *schema.TestVector) {
-	var (
-		ctx   = context.Background()
-		epoch = vector.Pre.Epoch
-		root  = vector.Pre.StateTree.RootCID
-	)
-
-	// Load the CAR into a new temporary Blockstore.
-	bs := loadCAR(t, vector.CAR)
-
-	// Create a new Driver.
-	driver := NewDriver(ctx, vector.Selector)
-
-	// Apply every message.
-	for i, m := range vector.ApplyMessages {
-		msg, err := types.DecodeMessage(m.Bytes)
-		if err != nil {
-			t.Fatalf("failed to deserialize message: %s", err)
-		}
-
-		// add an epoch if one's set.
-		if m.Epoch != nil {
-			epoch = *m.Epoch
-		}
-
-		// Execute the message.
-		var ret *vm.ApplyRet
-		ret, root, err = driver.ExecuteMessage(bs, root, abi.ChainEpoch(epoch), msg)
-		if err != nil {
-			t.Fatalf("fatal failure when executing message: %s", err)
-		}
-
-		// Assert that the receipt matches what the test vector expects.
-		assertMsgResult(t, vector.Post.Receipts[i], ret, strconv.Itoa(i))
-	}
-
-	// Once all messages are applied, assert that the final state root matches
-	// the expected postcondition root.
-	if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
-		t.Logf("actual state root CID doesn't match expected one; expected: %s, actual: %s", expected, actual)
-		dumpThreeWayStateDiff(t, vector, bs, root)
-		t.FailNow()
-	}
-}
-
-// executeTipsetVector executes a tipset-class test vector.
-func executeTipsetVector(t *testing.T, vector *schema.TestVector) {
-	var (
-		ctx       = context.Background()
-		prevEpoch = vector.Pre.Epoch
-		root      = vector.Pre.StateTree.RootCID
-		tmpds     = ds.NewMapDatastore()
-	)
-
-	// Load the CAR into a new temporary Blockstore.
-	bs := loadCAR(t, vector.CAR)
-
-	// Create a new Driver.
-	driver := NewDriver(ctx, vector.Selector)
-
-	// Apply every tipset.
-	var receiptsIdx int
-	for i, ts := range vector.ApplyTipsets {
-		ts := ts // capture
-		ret, err := driver.ExecuteTipset(bs, tmpds, root, abi.ChainEpoch(prevEpoch), &ts)
-		if err != nil {
-			t.Fatalf("failed to apply tipset %d message: %s", i, err)
-		}
-
-		for j, v := range ret.AppliedResults {
-			assertMsgResult(t, vector.Post.Receipts[receiptsIdx], v, fmt.Sprintf("%d of tipset %d", j, i))
-			receiptsIdx++
-		}
-
-		// Compare the receipts root.
-		if expected, actual := vector.Post.ReceiptsRoots[i], ret.ReceiptsRoot; expected != actual {
-			t.Errorf("post receipts root doesn't match; expected: %s, was: %s", expected, actual)
-		}
-
-		prevEpoch = ts.Epoch
-		root = ret.PostStateRoot
-	}
-
-	// Once all messages are applied, assert that the final state root matches
-	// the expected postcondition root.
-	if expected, actual := vector.Post.StateTree.RootCID, root; expected != actual {
-		t.Logf("actual state root CID doesn't match expected one; expected: %s, actual: %s", expected, actual)
-		dumpThreeWayStateDiff(t, vector, bs, root)
-		t.FailNow()
-	}
-}
-
-// assertMsgResult compares a message result. It takes the expected receipt
-// encoded in the vector, the actual receipt returned by Lotus, and a message
-// label to log in the assertion failure message to facilitate debugging.
-func assertMsgResult(t *testing.T, expected *schema.Receipt, actual *vm.ApplyRet, label string) {
-	t.Helper()
-
-	if expected, actual := exitcode.ExitCode(expected.ExitCode), actual.ExitCode; expected != actual {
-		t.Errorf("exit code of msg %s did not match; expected: %s, got: %s", label, expected, actual)
-	}
-	if expected, actual := expected.GasUsed, actual.GasUsed; expected != actual {
-		t.Errorf("gas used of msg %s did not match; expected: %d, got: %d", label, expected, actual)
-	}
-	if expected, actual := []byte(expected.ReturnValue), actual.Return; !bytes.Equal(expected, actual) {
-		t.Errorf("return value of msg %s did not match; expected: %s, got: %s", label, base64.StdEncoding.EncodeToString(expected), base64.StdEncoding.EncodeToString(actual))
-	}
-}
-
-func dumpThreeWayStateDiff(t *testing.T, vector *schema.TestVector, bs blockstore.Blockstore, actual cid.Cid) {
-	// check if statediff exists; if not, skip.
-	if err := exec.Command("statediff", "--help").Run(); err != nil {
-		t.Log("could not dump 3-way state tree diff upon test failure: statediff command not found")
-		t.Log("install statediff with:")
-		t.Log("$ git clone https://github.com/filecoin-project/statediff.git")
-		t.Log("$ cd statediff")
-		t.Log("$ go generate ./...")
-		t.Log("$ go install ./cmd/statediff")
-		return
-	}
-
-	tmpCar := writeStateToTempCAR(t, bs,
-		vector.Pre.StateTree.RootCID,
-		vector.Post.StateTree.RootCID,
-		actual,
-	)
-
-	color.NoColor = false // enable colouring.
-
-	t.Errorf("wrong post root cid; expected %v, but got %v", vector.Post.StateTree.RootCID, actual)
-
-	var (
-		a  = color.New(color.FgMagenta, color.Bold).Sprint("(A) expected final state")
-		b  = color.New(color.FgYellow, color.Bold).Sprint("(B) actual final state")
-		c  = color.New(color.FgCyan, color.Bold).Sprint("(C) initial state")
-		d1 = color.New(color.FgGreen, color.Bold).Sprint("[Δ1]")
-		d2 = color.New(color.FgGreen, color.Bold).Sprint("[Δ2]")
-		d3 = color.New(color.FgGreen, color.Bold).Sprint("[Δ3]")
-	)
-
-	printDiff := func(left, right cid.Cid) {
-		cmd := exec.Command("statediff", "car", "--file", tmpCar, left.String(), right.String())
-		b, err := cmd.CombinedOutput()
-		if err != nil {
-			t.Fatalf("statediff failed: %s", err)
-		}
-		t.Log(string(b))
-	}
-
-	bold := color.New(color.Bold).SprintfFunc()
-
-	// run state diffs.
-	t.Log(bold("=== dumping 3-way diffs between %s, %s, %s ===", a, b, c))
-
-	t.Log(bold("--- %s left: %s; right: %s ---", d1, a, b))
-	printDiff(vector.Post.StateTree.RootCID, actual)
-
-	t.Log(bold("--- %s left: %s; right: %s ---", d2, c, b))
-	printDiff(vector.Pre.StateTree.RootCID, actual)
-
-	t.Log(bold("--- %s left: %s; right: %s ---", d3, c, a))
-	printDiff(vector.Pre.StateTree.RootCID, vector.Post.StateTree.RootCID)
-}
-
-// writeStateToTempCAR writes the provided roots to a temporary CAR that'll be
-// cleaned up via t.Cleanup(). It returns the full path of the temp file.
-func writeStateToTempCAR(t *testing.T, bs blockstore.Blockstore, roots ...cid.Cid) string {
-	tmp, err := ioutil.TempFile("", "lotus-tests-*.car")
-	if err != nil {
-		t.Fatalf("failed to create temp file to dump CAR for diffing: %s", err)
-	}
-	// register a cleanup function to delete the CAR.
-	t.Cleanup(func() {
-		_ = os.Remove(tmp.Name())
-	})
-
-	carWalkFn := func(nd format.Node) (out []*format.Link, err error) {
-		for _, link := range nd.Links() {
-			if link.Cid.Prefix().Codec == cid.FilCommitmentSealed || link.Cid.Prefix().Codec == cid.FilCommitmentUnsealed {
-				continue
-			}
-			out = append(out, link)
-		}
-		return out, nil
-	}
-
-	var (
-		offl    = offline.Exchange(bs)
-		blkserv = blockservice.New(bs, offl)
-		dserv   = merkledag.NewDAGService(blkserv)
-	)
-
-	err = car.WriteCarWithWalker(context.Background(), dserv, roots, tmp, carWalkFn)
-	if err != nil {
-		t.Fatalf("failed to dump CAR for diffing: %s", err)
-	}
-	_ = tmp.Close()
-	return tmp.Name()
-}
-
-func loadCAR(t *testing.T, vectorCAR schema.Base64EncodedBytes) blockstore.Blockstore {
-	bs := blockstore.NewTemporary()
-
-	// Read the base64-encoded CAR from the vector, and inflate the gzip.
-	buf := bytes.NewReader(vectorCAR)
-	r, err := gzip.NewReader(buf)
-	if err != nil {
-		t.Fatalf("failed to inflate gzipped CAR: %s", err)
-	}
-	defer r.Close() // nolint
-
-	// Load the CAR embedded in the test vector into the Blockstore.
-	_, err = car.LoadCar(bs, r)
-	if err != nil {
-		t.Fatalf("failed to load state tree car from test vector: %s", err)
-	}
-	return bs
-}

documentation/en/.glossary.json

@@ -1,146 +1 @@
-{
+{}
-  "bellman": {
-    "title": "Bellman",
-    "value": "Bellman is a rust crate for building zk-SNARK circuits. It provides circuit traits and primitive structures, as well as basic gadget implementations such as booleans and number abstractions."
-  },
-  "nvme": {
-    "title": "NVMe",
-    "value": "(non-volatile memory express) is a host controller interface and storage protocol created to accelerate the transfer of data between enterprise and client systems and solid-state drives (SSDs) over a computer's high-speed Peripheral Component Interconnect Express (PCIe) bus."
-  },
-  "multiaddr": {
-    "title": "Multiaddr",
-    "value": "Multiaddr is a format for encoding addresses from various well-established network protocols. It is useful to write applications that future-proof their use of addresses, and allow multiple transport protocols and addresses to coexist."
-  },
-  "attofil": {
-    "title": "attoFIL",
-    "value": "AttoFIL is a word used to describe 10^-18 FIL. The word atto comes from the Norwegian and Danish term: atten eighteen."
-  },
-  "fil": {
-    "title": "FIL",
-    "value": "A ticker symbol is an abbreviation used to uniquely identify Filecoin when it is used in a wallet exchange or a cryptocurrency exchange."
-  },
-  "epost": {
-    "title": "Election Proof-of-Spacetime",
-    "value": "Election Proof-of-Spacetime couples the Proof-of-Spacetime process with block production, meaning that in order to produce a block, the miner must produce a valid Proof-of-Spacetime proof (snark output)."
-  },
-  "jwt": {
-    "title": "JWT",
-    "value": "JSON Web Tokens are an open, industry standard RFC 7519 method for representing claims securely between two parties."
-  },
-  "json-rpc": {
-    "title": "JSON-RPC",
-    "value": "JSON-RPC is a remote procedure call protocol encoded in JSON. It is a very simple protocol (and very similar to XML-RPC), defining only a few data types and commands."
-  },
-  "bls-address": {
-    "title": "BLS Signature (Address)",
-    "value": "A Boneh–Lynn–Shacham (BLS) signature is a digital signature scheme that allows a user to determine the authenticity of a signer, and is a commonly used signature scheme in the Filecoin Distributed Storage Network."
-  },
-  "faucet": {
-    "title": "Filecoin Test Faucet",
-    "value": "A webpage where you can get free test Filecoin to participate in the Testnet."
-  },
-  "chain": {
-    "title": "Chain",
-    "value": "The Filecoin Blockchain is a distributed virtual machine that achieves consensus, processes messages, accounts for storage, and maintains security in the Filecoin Protocol. It is the main interface linking various actors in the Filecoin system."
-  },
-  "miner-power": {
-    "title": "Miner Power",
-    "value": "Miner storage in relation to network storage, tracked in the power table."
-  },
-  "sector": {
-    "title": "Sector",
-    "value": "A fixed-size block of data of SECTOR_SIZE bytes which generally contains client's data."
-  },
-  "sealing": {
-    "title": "Sealing",
-    "value": "A slow encoding process that returns commitments and proofs for data being stored in a sector."
-  },
-  "seal": {
-    "title": "Seal",
-    "value": "A slow encoding process that returns commitments and proofs for data being stored in a sector."
-  },
-  "posts": {
-    "title": "Proof-of-Spacetime(s)",
-    "value": "Filecoin is a protocol token whose blockchain runs on a novel proof, called Proof-of-Spacetime, where blocks are created by miners that are storing data."
-  },
-  "filecoin-testnet": {
-    "title": "Filecoin Testnet",
-    "value": "Until we launch, we are making lots of changes to Lotus. The Testnet is expected to bring a few significant fixes/improvements. During Testnet, you can retrieve test filecoin from our network faucet to use as collateral to start mining. Test filecoin do not have any value – the official filecoin tokens will not be released until Mainnet launch."
-  },
-  "filecoin-decentralized-storage-market": {
-    "title": "Filecoin Decentralized Storage Market",
-    "value": "Storage Market subsystem is the data entry point into the network. Miners only earn power from data stored in a storage deal and all deals live on the Filecoin network."
-  },
-  "filecoin-proof-parameters": {
-    "title": "Filecoin Proof Parameters",
-    "value": "The proving algorithms rely on a large binary parameter file."
-  },
-  "lotus-devnet": {
-    "title": "DevNet",
-    "value": "On the DevNets, you can store data as a storage client and also try how Filecoin mining works. The devnets are an important development tool for those who anticipate building applications on top of the Filecoin protocol or storing data on the decentralized storage market. "
-  },
-  "filecoin-distributed-storage-network": {
-    "title": "Filecoin Distributed Storage Network",
-    "value": "Filecoin is a distributed storage network based on a blockchain mechanism. Filecoin miners can elect to provide storage capacity for the network, and thereby earn units of the Filecoin cryptocurrency (FIL) by periodically producing cryptographic proofs that certify that they are providing the capacity specified."
-  },
-  "lotus-node": {
-    "title": "Lotus Node",
-    "value": "The Lotus Node is full of capabilities. It runs the Blockchain system, makes retrieval deals, does data transfer, supports block producer logic, and syncs and validates the chain."
-  },
-  "block-rewards": {
-    "title": "Block Reward",
-    "value": "Over the entire lifetime of the protocol, 1,400,000,000 FIL (TotalIssuance) will be given out to miners. The rate at which the funds are given out is set to halve every six years, smoothly (not a fixed jump like in Bitcoin)."
-  },
-  "block-producer-miner": {
-    "title": "Miner (Block Producer)",
-    "value": "The Block Producer Miner's logic. It currently shares an interface and process with the Lotus Node. A Block Producer chooses which messages to include in a block and is rewarded according to each message’s gas price and consumption, forming a market."
-  },
-  "lotus-miner": {
-    "title": "Miner (lotus-miner)",
-    "value": "The Miner's logic. It has its own dedicated process. Contributes to the network through Sector commitments and Proofs of Spacetime to prove that it is storing the sectors it has commited to."
-  },
-  "swarm-port": {
-    "title": "Swarm Port (Libp2p)",
-    "value": "The LibP2P Swarm manages groups of connections to peers, handles incoming and outgoing streams, and is part of the miners implementation. The port value is part of the Host interface."
-  },
-  "daemon": {
-    "title": "Lotus Daemon",
-    "value": "A Daemon is a program that runs as a background process. A Daemon in the context of the Filecoin Distributed Storage Network may enable applications to communicate with peers, handle protocols, participate in pubsub, and interact with a distributed hash table (DHT)."
-  },
-  "storage-deal": {
-    "title": "Storage deal",
-    "value": "One of the two types of deals in Filecoin markets. Storage deals are recorded on the blockchain and enforced by the protocol."
-  },
-  "retrieval-deal": {
-    "title": "Retrieval deal",
-    "value": "One of the two types of deals in Filecoin markets. Retrieval deals are off chain and enabled by micropayment channel by transacting parties."
-  },
-  "deal-cid": {
-    "title": "Deal CID",
-    "value": "CID is a format for referencing content in distributed information systems, it is a way to store information so it can be retrieved based on its content, not its location. DealCID specifically is used in storage deals."
-  },
-  "data-cid": {
-    "title": "Data CID",
-    "value": "CID is a format for referencing content in distributed information systems, it is a way to store information so it can be retrieved based on its content, not its location. DataCID specifically is used to represent the file that is stored in the Filecoin Distributed Storage Network."
-  },
-  "cid": {
-    "title": "CID",
-    "value": "A CID is a self-describing content-addressed identifier. It uses cryptographic hashes to achieve content addressing. It uses several multiformats to achieve flexible self-description, namely multihash for hashes, multicodec for data content types, and multibase to encode the CID itself into strings."
-  },
-  "total-network-power": {
-    "title": "Total Network Power",
-    "value": "A reference to all the Power Tables for every subchain, accounting for each Lotus Miner on chain."
-  },
-  "chain-block-height": {
-    "title": "Chain Block Height",
-    "value": "Chain block height is defined as the number of blocks in the chain between any given block and the very first block in the blockchain."
-  },
-  "block-height": {
-    "title": "Block Height",
-    "value": "Height of the Merkle Tree of a sector. A sector is a contiguous array of bytes that a miner puts together, seals, and performs Proofs of Spacetime on."
-  },
-  "blocktime": {
-    "title": "Blocktime",
-    "value": "The time it takes for a Block to propagate to the whole network."
-  }
-}

documentation/en/.library.json

@@ -1,232 +1,25 @@
 {
   "posts": [
     {
-      "title": "Hardware Requirements",
+      "title": "About Lotus",
-      "slug": "en+hardware",
+      "slug": "",
-      "github": "en/hardware.md",
+      "github": "en/about.md",
       "value": null,
-      "posts": [
+      "posts": []
-        {
-          "title": "Testing Configuration",
-          "slug": "en+hardware-mining",
-          "github": "en/hardware-mining.md",
-          "value": null
-        }
-      ]
     },
     {
-      "title": "Setup",
+      "title": "Lotus Architecture (WIP)",
-      "slug": "en+getting-started",
-      "github": "en/getting-started.md",
-      "value": null,
-      "posts": [
-        {
-          "title": "Arch Linux Installation",
-          "slug": "en+install-lotus-arch",
-          "github": "en/install-lotus-arch.md",
-          "value": null
-        },
-        {
-          "title": "Ubuntu Installation",
-          "slug": "en+install-lotus-ubuntu",
-          "github": "en/install-lotus-ubuntu.md",
-          "value": null
-        },
-        {
-          "title": "Fedora Installation",
-          "slug": "en+install-lotus-fedora",
-          "github": "en/install-lotus-fedora.md",
-          "value": null
-        },
-        {
-          "title": "MacOS Installation",
-          "slug": "en+install-lotus-macos",
-          "github": "en/install-lotus-macos.md",
-          "value": null
-        },
-        {
-          "title": "Updating Lotus",
-          "slug": "en+updating-lotus",
-          "github": "en/updating-lotus.md",
-          "value": null
-        },
-        {
-          "title": "Join Testnet",
-          "slug": "en+join-testnet",
-          "github": "en/join-testnet.md",
-          "value": null
-        },
-        {
-          "title": "Use Lotus with systemd",
-          "slug": "en+install-systemd-services",
-          "github": "en/install-systemd-services.md",
-          "value": null
-        },
-        {
-          "title": "Setup Troubleshooting",
-          "slug": "en+setup-troubleshooting",
-          "github": "en/setup-troubleshooting.md",
-          "value": null
-        },
-        {
-          "title": "Environment Variables",
-          "slug": "en+env-vars",
-          "github": "en/environment-vars.md",
-          "value": null
-        }
-      ]
-    },
-    {
-      "title": "Architecture",
       "slug": "en+arch",
-      "github": "en/architecture.md",
+      "github": "en/architecture/architecture.md",
       "value": null,
       "posts": [
 	{
 	  "title": "The Message Pool",
 	  "slug": "en+mpool",
-          "github": "en/mpool.md",
+	  "github": "en/architecture/mpool.md",
 	  "value": null
 	}
       ]
-    },
-    {
-      "title": "Storage Mining",
-      "slug": "en+mining",
-      "github": "en/mining.md",
-      "value": null,
-      "posts": [
-        {
-          "title": "Lotus Worker",
-          "slug": "en+lotus-worker",
-          "github": "en/mining-lotus-worker.md",
-          "value": null
-        },
-        {
-          "title": "Static Ports",
-          "slug": "en+setting-a-static-port",
-          "github": "en/setting-a-static-port.md",
-          "value": null
-        },
-        {
-          "title": "Mining Troubleshooting",
-          "slug": "en+mining-troubleshooting",
-          "github": "en/mining-troubleshooting.md",
-          "value": null
-        }
-      ]
-    },
-    {
-      "title": "Storing Data",
-      "slug": "en+storing-data",
-      "github": "en/storing-data.md",
-      "value": null,
-      "posts": [
-        {
-          "title": "Storage Troubleshooting",
-          "slug": "en+storing-data-troubleshooting",
-          "github": "en/storing-data-troubleshooting.md",
-          "value": null
-        },
-        {
-          "title": "Information for Miners",
-          "slug": "en+info-for-miners",
-          "github": "en/miner-deals.md",
-          "value": null
-        },
-        {
-          "title": "IPFS Integration",
-          "slug": "en+ipfs-client-integration",
-          "github": "en/storing-ipfs-integration.md",
-          "value": null
-        }
-      ]
-    },
-    {
-      "title": "Retrieving Data",
-      "slug": "en+retrieving-data",
-      "github": "en/retrieving-data.md",
-      "value": null,
-      "posts": []
-    },
-    {
-      "title": "Payment Channels",
-      "slug": "en+payment-channels",
-      "github": "en/payment-channels.md",
-      "value": null,
-      "posts": []
-    },
-    {
-      "title": "Command Line Interface",
-      "slug": "en+cli",
-      "github": "en/cli.md",
-      "value": null,
-      "posts": []
-    },
-    {
-      "title": "API",
-      "slug": "en+api",
-      "github": "en/api.md",
-      "value": null,
-      "posts": [
-        {
-          "title": "Remote API Support",
-          "slug": "en+api-scripting-support",
-          "github": "en/api-scripting-support.md",
-          "value": null
-        },
-        {
-          "title": "API Methods",
-          "slug": "en+api-methods",
-          "github": "en/api-methods.md",
-          "value": null
-        },
-        {
-          "title": "API Troubleshooting",
-          "slug": "en+api-troubleshooting",
-          "github": "en/api-troubleshooting.md",
-          "value": null
-        }
-      ]
-    },
-    {
-      "title": "Developer Tools",
-      "slug": "en+dev-tools",
-      "github": "en/dev-tools.md",
-      "value": null,
-      "posts": [
-        {
-          "title": "Setup Local Devnet",
-          "slug": "en+setup-local-dev-net",
-          "github": "en/local-dev-net.md",
-          "value": null,
-          "posts": []
-        },
-        {
-          "title": "Jaeger Tracing",
-          "slug": "en+dev-tools-jaeger-tracing",
-          "github": "en/dev-tools-jaeger-tracing.md",
-          "value": null,
-          "posts": []
-        }
-      ]
-    },
-    {
-      "title": "FAQs",
-      "slug": "en+faqs",
-      "github": "en/faqs.md",
-      "value": null,
-      "posts": []
-    },
-    {
-      "title": "Glossary",
-      "slug": "en+glossary",
-      "github": "en/.glossary.json",
-      "value": null,
-      "custom": {
-        "glossary": true
-      },
-      "posts": []
     }
   ]
 }

documentation/en/README.md

@@ -0,0 +1,16 @@
+# Lotus documentation
+
+This folder contains some Lotus documentation mostly intended for Lotus developers.
+
+User documentation (including documentation for miners) has been moved to specific Lotus sections in https://docs.filecoin.io:
+
+- https://docs.filecoin.io/get-started/lotus
+- https://docs.filecoin.io/store/lotus
+- https://docs.filecoin.io/mine/lotus
+- https://docs.filecoin.io/build/lotus
+
+## The Lotu.sh site
+
+The https://lotu.sh and https://docs.lotu.sh sites are generated from this folder based on the index provided by [.library.json](.library.json). This is done at the [lotus-docs repository](https://github.com/filecoin-project/lotus-docs), which contains Lotus as a git submodule.
+
+To update the site, the lotus-docs repository should be updated with the desired version for the lotus git submodule. Once pushed to master, it will be auto-deployed.

documentation/en/about.md

@@ -0,0 +1,19 @@
+# Lotus
+
+Lotus is an implementation of the **Filecoin Distributed Storage Network**.
+
+It is written in Go and provides a suite of command-line applications:
+
+- Lotus Node (`lotus`): a Filecoin Node: validates network transactions, manages a FIL wallet, can perform storage and retrieval deals.
+- Lotus Miner (`lotus-miner`): a Filecoin miner. See the the respective Lotus Miner section in the Mine documentation.
+- Lotus Worker (`lotus-worker`): a worker that assists miners to perform mining-related tasks. See its respective guide for more information.
+
+The [Lotus user documentation](https://docs.filecoin.io/get-started/lotus) is part of the [Filecoin documentation site](https://docs.filecoin.io):
+
+* To install and get started with Lotus, visit the [Get Started section](https://docs.filecoin.io/get-started/lotus).
+* Information about how to perform deals on the Filecoin network using Lotus can be found  in the [Store section](https://docs.filecoin.io/store/lotus).
+* Miners looking to provide storage to the Network can find the latest guides in the [Mine section](https://docs.filecoin.io/mine/lotus).
+* Developers and integrators that wish to use the Lotus APIs can start in the [Build section](https://docs.filecoin.io/mine/lotus).
+
+For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://spec.filecoin.io/).
+

documentation/en/api-methods.md

@@ -169,6 +169,7 @@
   * [SyncState](#SyncState)
   * [SyncSubmitBlock](#SyncSubmitBlock)
   * [SyncUnmarkBad](#SyncUnmarkBad)
+  * [SyncValidateTipset](#SyncValidateTipset)
 * [Wallet](#Wallet)
   * [WalletBalance](#WalletBalance)
   * [WalletDefaultAddress](#WalletDefaultAddress)
@@ -181,6 +182,7 @@
   * [WalletSetDefault](#WalletSetDefault)
   * [WalletSign](#WalletSign)
   * [WalletSignMessage](#WalletSignMessage)
+  * [WalletValidateAddress](#WalletValidateAddress)
   * [WalletVerify](#WalletVerify)
 ## 
 
@@ -4379,6 +4381,28 @@ Inputs:
 
 Response: `{}`
 
+### SyncValidateTipset
+SyncValidateTipset indicates whether the provided tipset is valid or not
+
+
+Perms: read
+
+Inputs:
+```json
+[
+  [
+    {
+      "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
+    },
+    {
+      "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve"
+    }
+  ]
+]
+```
+
+Response: `true`
+
 ## Wallet
 
 
@@ -4585,6 +4609,21 @@ Response:
 }
 ```
 
+### WalletValidateAddress
+WalletValidateAddress validates whether a given string can be decoded as a well-formed address
+
+
+Perms: read
+
+Inputs:
+```json
+[
+  "string value"
+]
+```
+
+Response: `"t01234"`
+
 ### WalletVerify
 WalletVerify takes an address, a signature, and some bytes, and indicates whether the signature is valid.
 The address does not have to be in the wallet.

documentation/en/api-scripting-support.md

@@ -1,25 +0,0 @@
-# Remote API Support
-
-You may want to delegate the work **Lotus Miner** or **Lotus Node** performs to other machines. 
-Here is how to setup the necessary authorization and environment variables.
-
-## Environment variables
-
-Environmental variables are variables that are defined for the current shell and are inherited by any child shells or processes. Environmental variables are used to pass information into processes that are spawned from the shell.
-
-Using the [JWT you generated](https://lotu.sh/en+api#how-do-i-generate-a-token-18865), you can assign it and the **multiaddr** to the appropriate environment variable.
-
-```sh
-# Lotus Node
-FULLNODE_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/1234/http"
-
-# Lotus Miner
-MINER_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/2345/http"
-```
-
-You can also use `lotus auth api-info --perm admin` to quickly create _API_INFO env vars
-
-- The **Lotus Node**'s `mutliaddr` is in `~/.lotus/api`.
-- The default token is in `~/.lotus/token`.
-- The **Lotus Miner**'s `multiaddr` is in `~/.lotusminer/config`.
-- The default token is in `~/.lotusminer/token`.

documentation/en/api-troubleshooting.md

@@ -1,36 +0,0 @@
-# API Troubleshooting
-
-## Types: params
-
-`params` must be an array. If there are no `params` you should still pass an empty array.
-
-## Types: TipSet
-
-For methods such as `Filecoin.StateMinerPower`, where the method accepts the argument of the type `TipSet`, you can pass `null` to use the current chain head.
-
-```sh
-curl -X POST \
-     -H "Content-Type: application/json" \
-     --data '{ "jsonrpc": "2.0", "method": "Filecoin.StateMinerPower", "params": ["t0101", null], "id": 3 }' \
-     'http://127.0.0.1:1234/rpc/v0'
-```
-
-## Types: Sending a CID
-
-If you do not serialize the CID as a [JSON IPLD link](https://did-ipid.github.io/ipid-did-method/#txref), you will receive an error. Here is an example of a broken CURL request:
-
-```sh
-curl -X POST \
-     -H "Content-Type: application/json" \
-     --data '{ "jsonrpc": "2.0", "method":"Filecoin.ClientGetDealInfo", "params": ["bafyreiaxl446wlnu6t6dpq4ivrjf4gda4gvsoi4rr6mpxau7z25xvk5pl4"], "id": 0 }' \
-     'http://127.0.0.1:1234/rpc/v0'
-```
-
-To fix it, change the `params` property to:
-
-```sh
-curl -X POST \
-     -H "Content-Type: application/json" \
-     --data '{ "jsonrpc": "2.0", "method":"Filecoin.ClientGetDealInfo", "params": [{"/": "bafyreiaxl446wlnu6t6dpq4ivrjf4gda4gvsoi4rr6mpxau7z25xvk5pl4"}], "id": 0 }' \
-     'http://127.0.0.1:1234/rpc/v0'
-```

documentation/en/api.md

@@ -1,85 +0,0 @@
-# API
-
-Here is an early overview of how to make API calls.
-
-Implementation details for the **JSON-RPC** package are [here](https://github.com/filecoin-project/go-jsonrpc).
-
-## Overview: How do you modify the config.toml to change the API endpoint?
-
-API requests are made against `127.0.0.1:1234` unless you modify `.lotus/config.toml`.
-
-Options:
-
-- `http://[api:port]/rpc/v0` - HTTP endpoint
-- `ws://[api:port]/rpc/v0` - Websocket endpoint
-- `PUT http://[api:port]/rest/v0/import` - File import, it requires write permissions.
-
-## What methods can I use?
-
-For now, you can look into different files to find methods available to you based on your needs:
-
-- [Both Lotus node + miner APIs](https://github.com/filecoin-project/lotus/blob/master/api/api_common.go)
-- [Lotus node API](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go)
-- [Lotus miner API](https://github.com/filecoin-project/lotus/blob/master/api/api_storage.go)
-
-The necessary permissions for each are in [api/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/struct.go).
-
-## How do I make an API request?
-
-To demonstrate making an API request, we will take the method `ChainHead` from [api/api_full.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go).
-
-```go
-ChainHead(context.Context) (*types.TipSet, error)
-```
-
-And create a CURL command. In this command, `ChainHead` is included as `{ "method": "Filecoin.ChainHead" }`:
-
-```sh
-curl -X POST \
-     -H "Content-Type: application/json" \
-     --data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \
-     'http://127.0.0.1:1234/rpc/v0'
-```
-
-If the request requires authorization, add an authorization header:
-
-```sh
-curl -X POST \
-     -H "Content-Type: application/json" \
-     -H "Authorization: Bearer $(cat ~/.lotusminer/token)" \
-     --data '{ "jsonrpc": "2.0", "method": "Filecoin.ChainHead", "params": [], "id": 3 }' \
-     'http://127.0.0.1:1234/rpc/v0'
-```
-
-> In the future we will add a playground to make it easier to build and experiment with API requests.
-
-## CURL authorization
-
-To authorize your request, you will need to include the **JWT** in a HTTP header, for example:
-
-```sh
--H "Authorization: Bearer $(cat ~/.lotusminer/token)"
-```
-
-Admin token is stored in `~/.lotus/token` for the **Lotus Node** or `~/.lotusminer/token` for the **Lotus Miner**.
-
-## How do I generate a token?
-
-To generate a JWT with custom permissions, use this command:
-
-```sh
-# Lotus Node
-lotus auth create-token --perm admin
-
-# Lotus Miner
-lotus-miner auth create-token --perm admin
-```
-
-## What authorization level should I use?
-
-When viewing [api/apistruct/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/apistruct/struct.go), you will encounter these types:
-
-- `read` - Read node state, no private data.
-- `write` - Write to local store / chain, and `read` permissions.
-- `sign` - Use private keys stored in wallet for signing, `read` and `write` permissions.
-- `admin` - Manage permissions, `read`, `write`, and `sign` permissions.

documentation/en/architecture/architecture.md

@@ -6,7 +6,7 @@ Filecoin protocol, validating the blocks and state transitions.
 The specification for the Filecoin protocol can be found [here](https://filecoin-project.github.io/specs/).
 
 For information on how to setup and operate a Lotus node,
-please follow the instructions [here](https://lotu.sh/en+getting-started).
+please follow the instructions [here](en+getting-started).
 
 # Components
 

documentation/en/cli.md

@@ -1,108 +0,0 @@
-# Lotus Command Line Interface
-
-The Command Line Interface (CLI) is a convenient way to interact with
-a Lotus node. You can use the CLI to operate your node,
-get information about the blockchain,
-manage your accounts and transfer funds,
-create storage deals, and much more! 
-
-The CLI is intended to be self-documenting, so when in doubt, simply add `--help`
-to whatever command you're trying to run! This will also display all of the 
-input parameters that can be provided to a command.
-
-We highlight some of the commonly
-used features of the CLI below.
-All CLI commands should be run from the home directory of the Lotus project.
-
-## Operating a Lotus node
-
-### Starting up a node
-
-```sh
-lotus daemon
-```
-This command will start up your Lotus node, with its API port open at 1234. 
-You can pass `--api=<number>` to use a different port.
-
-### Checking your sync progress
-
-```sh
-lotus sync status
-```
-This command will print your current tipset height under `Height`, and the target tipset height
-under `Taregt`. 
-
-You can also run `lotus sync wait` to get constant updates on your sync progress.
-
-### Getting the head tipset
-
-```sh
-lotus chain head
-```
-
-### Control the logging level
-
-```sh
-lotus log set-level
-```
-This command can be used to toggle the logging levels of the different
-systems of a Lotus node. In decreasing order
-of logging detail, the levels are `debug`, `info`, `warn`, and `error`. 
-
-As an example,
-to set the `chain` and `blocksync` to log at the `debug` level, run 
-`lotus log set-level --system chain --system blocksync debug`. 
-
-To see the various logging system, run `lotus log list`.
-
-### Find out what version of Lotus you're running
-
-```sh
-lotus version
-```
-
-## Managing your accounts
-
-### Listing accounts in your wallet
-
-```sh
-lotus wallet list
-``` 
-
-### Creating a new account
-
-```sh
-lotus wallet new bls
-```
-This command will create a new BLS account in your wallet; these
-addresses start with the prefix `t3`. Running `lotus wallet new secp256k1` 
-(or just `lotus wallet new`) will create
-a new Secp256k1 account, which begins with the prefix `t1`.
-
-### Getting an account's balance
-
-```sh
-lotus wallet balance <address>
-``` 
-
-### Transferring funds 
-
-```sh
-lotus send --source=<source address> <destination address> <amount>
-``` 
-This command will transfer `amount` (in attoFIL) from `source address` to `destination address`.
-
-### Importing an account into your wallet
-
-```sh
-lotus wallet import <path to private key>
-``` 
-This command will import an account whose private key is saved at the specified file.
-
-### Exporting an account from your wallet
-
-```sh
-lotus wallet export <address>
-``` 
-This command will print out the private key of the specified address
-if it is in your wallet. Always be careful with your private key!

documentation/en/dev-tools.md

@@ -1,3 +0,0 @@
-# Developer Tools
-
-> Running a local network can be a great way to understand how Lotus works and test your setup. 

documentation/en/environment-vars.md

@@ -1,65 +0,0 @@
-# Lotus Environment Variables
-
-## Building
-
-## Common
-
-The environment variables are common across most lotus binaries.
-
-### `LOTUS_FD_MAX`
-
-Sets the file descriptor limit for the process. This should be set high (8192
-or higher) if you ever notice 'too many open file descriptor' errors.
-
-### `LOTUS_JAEGER`
-
-This can be set to enable jaeger trace reporting. The value should be the url
-of the jaeger trace collector, the default for most jaeger setups should be
-`localhost:6831`.
-
-### `LOTUS_DEV`
-
-If set to a non-empty value, certain parts of the application will print more
-verbose information to aid in development of the software. Not recommended for
-end users.
-
-## Lotus Daemon
-
-### `LOTUS_PATH`
-
-Sets the location for the lotus daemon on-disk repo. If left empty, this defaults to `~/.lotus`.
-
-### `LOTUS_SKIP_GENESIS_CHECK`
-
-Can be set to `_yes_` if you wish to run a lotus network with a different
-genesis than the default one built into your lotus binary.
-
-### `LOTUS_CHAIN_TIPSET_CACHE`
-
-Sets the cache size for the chainstore tipset cache. The default value is 8192,
-but if your usage of the lotus API involves frequent arbitrary tipset lookups,
-you may want to increase this.
-
-### `LOTUS_CHAIN_INDEX_CACHE`
-
-Sets the cache size for the chainstore epoch index cache. The default value is 32768,
-but if your usage of the lotus API involves frequent deep chain lookups for
-block heights that are very far from the current chain height, you may want to
-increase this.
-
-
-### `LOTUS_BSYNC_MSG_WINDOW`
-
-Set the initial maximum window size for message fetching blocksync requests. If
-you have a slower internet connection and are having trouble syncing, you might
-try lowering this down to 10-20 for a 'poor' internet connection.
-
-## Lotus Miner
-
-A number of environment variables are respected for configuring the behavior of the filecoin proving subsystem. For more details on those [see here](https://github.com/filecoin-project/rust-fil-proofs/#settings).
-
-### `LOTUS_MINER_PATH`
-
-Sets the location for the lotus miners on-disk repo. If left empty, this defaults to `~/.lotusminer`.
-
-

documentation/en/faqs.md

@@ -1,138 +0,0 @@
-# Frequently Asked Questions
-
-Here are some FAQs concerning the Lotus implementation and participation in 
-Testnet.
-For questions concerning the broader Filecoin project, please 
-go [here](https://filecoin.io/faqs/).
-
-## Introduction to Lotus
-
-### What is Lotus?
-
-Lotus is an implementation of the **Filecoin Distributed Storage Network**, written in Go. 
-It is designed to be modular and interoperable with any other implementation of the Filecoin Protocol.
-More information about Lotus can be found [here](https://lotu.sh/).
-
-### What are the components of Lotus?
-
-Lotus is composed of two separate pieces that can talk to each other:
-
-The Lotus Node can sync the blockchain, validating all blocks, transfers, and deals
-along the way. It can also facilitate the creation of new storage deals. If you are not 
-interested in providing your own storage to the network, and do not want to produce blocks
-yourself, then the Lotus Node is all you need!
-
-The Lotus Miner does everything you need for the registration of storage, and the
-production of new blocks. The Lotus Miner communicates with the network by talking 
-to a Lotus Node over the JSON-RPC API.
-
-## Setting up a Lotus Node
-
-### How do I set up a Lotus Node?
-
-Follow the instructions found [here](https://lotu.sh/en+getting-started).
-
-### Where can I get the latest version of Lotus?
-
-Download the binary tagged as the `Latest Release` from the
- [Lotus Github repo](https://github.com/filecoin-project/lotus/releases).
- 
-### What operating systems can Lotus run on?
-
-Lotus can build and run on most Linux and MacOS systems with at least 
-8GB of RAM. Windows is not yet supported.
-
-### How can I update to the latest version of Lotus?
-
-To update Lotus, follow the instructions [here](https://lotu.sh/en+updating-lotus).
-
-### How do I prepare a fresh installation of Lotus?
-
-Stop the Lotus daemon, and delete all related files, including sealed and chain data by 
-running `rm ~/.lotus ~/.lotusminer`.
-
-Then, install Lotus afresh by following the instructions 
-found [here](https://lotu.sh/en+getting-started).
-
-### Can I configure where the node's config and data goes?
-
-Yes! The `LOTUS_PATH` variable sets the path for where the Lotus node's data is written.
-The `LOTUS_MINER_PATH` variable does the same for miner-specific information.
-
-## Interacting with a Lotus Node
-
-### How can I communicate with a Lotus Node?
-
-Lotus Nodes have a command-line interface, as well as a JSON-RPC API.
-
-### What are the commands I can send using the command-line interface? 
-
-The command-line interface is self-documenting, try running `lotus --help` from the `lotus` home 
-directory for more.
-
-### How can I send a request over the JSON-RPC API?
-
-Information on how to send a `cURL` request to the JSON-RPC API can be found
-[here](https://lotu.sh/en+api). A JavaScript client is under development.
-
-### What are the requests I can send over the JSON-RPC API?
-
-Please have a look at the 
-[source code](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go) 
-for a list of methods supported by the JSON-RPC API.
-## The Test Network
-
-### What is Testnet?
-
-Testnet is a live network of Lotus Nodes run by the 
-community for testing purposes.
- It has 2 PiB of storage (and growing!) dedicated to it.
-
-### Is FIL on the Testnet worth anything?
-
-Nothing at all! Real-world incentives may be provided in a future phase of Testnet, but this is 
-yet to be confirmed.
-
-### How can I see the status of Testnet?
-
-The [dashboard](https://stats.testnet.filecoin.io/) displays the status of the network as 
-well as a ton
-of other metrics you might find interesting.
-
-## Mining with a Lotus Node on Testnet
-
-### How do I get started mining with Lotus?
-
-Follow the instructions found [here](https://lotu.sh/en+mining).
-
-### What are the minimum hardware requirements?
-
-An example test configuration, and minimum hardware requirements can be found 
-[here](https://lotu.sh/en+hardware-mining). 
-
-Note that these might NOT be the minimum requirements for mining on Mainnet.
-
-### What are some GPUs that have been tested?
-
-A list of benchmarked GPUs can be found [here](https://lotu.sh/en+hardware-mining#benchmarked-gpus-7393).
-
-### Why is my GPU not being used when sealing a sector?
-
-Sealing a sector does not involve constant GPU operations. It's possible
-that your GPU simply isn't necessary at the moment you checked.
-
-## Advanced questions
-
-### Is there a Docker image for lotus?
-
-Community-contributed Docker and Docker Compose examples are available 
-[here](https://github.com/filecoin-project/lotus/tree/master/tools/dockers/docker-examples).
-
-### How can I run two miners on the same machine?
-
-You can do so by changing the storage path variable for the second miner, e.g.,
-`LOTUS_MINER_PATH=~/.lotusminer2`. You will also need to make sure that no ports collide.
-
-### How do I setup my own local devnet?     
-
-Follow the instructions found [here](https://lotu.sh/en+setup-local-dev-net).

documentation/en/getting-started.md

@@ -1,23 +0,0 @@
-# Lotus
-
-Lotus is an implementation of the **Filecoin Distributed Storage Network**. You can run the Lotus software client to join the **Filecoin Testnet**.
-
-For more details about Filecoin, check out the [Filecoin Docs](https://docs.filecoin.io) and [Filecoin Spec](https://filecoin-project.github.io/specs/).
-
-## What can I learn here?
-
-- How to install Lotus on [Arch Linux](https://lotu.sh/en+install-lotus-arch), [Ubuntu](https://lotu.sh/en+install-lotus-ubuntu), or [MacOS](https://lotu.sh/en+install-lotus-macos).
-- Joining the [Lotus Testnet](https://lotu.sh/en+join-testnet).
-- [Storing](https://lotu.sh/en+storing-data) or [retrieving](https://lotu.sh/en+retrieving-data) data.
-- Mining Filecoin using the **Lotus Miner** in your [CLI](https://lotu.sh/en+mining).
-
-## How is Lotus designed?
-
-Lotus is architected modularly to keep clean API boundaries while using the same process. Installing Lotus will include two separate programs:
-
-- The **Lotus Node**
-- The **Lotus Miner**
-
-The **Lotus Miner** is intended to be run on the machine that manages a single miner instance, and is meant to communicate with the **Lotus Node** via the websocket **JSON-RPC** API for all of the chain interaction needs.
-
-This way, a mining operation may easily run a **Lotus Miner** or many of them, connected to one or many **Lotus Node** instances.

documentation/en/hardware-mining.md

@@ -1,54 +0,0 @@
-# Protocol Labs Standard Testing Configuration
-
-> This documentation page describes the standard testing configuration the Protocol Labs team has used to test **Lotus Miner**s on Lotus. There is no guarantee this testing configuration will be suitable for Filecoin storage mining at MainNet launch. If you need to buy new hardware to join the Filecoin Testnet, we recommend to buy no more hardware than you require for testing. To learn more please read this [Protocol Labs Standard Testing Configuration post](https://filecoin.io/blog/filecoin-testnet-mining/).
-
-**Sector sizes** and **minimum pledged storage** required to mine blocks are two very important Filecoin Testnet parameters that impact hardware decisions. We will continue to refine all parameters during Testnet.
-
-BECAUSE OF THIS, OUR STANDARD TESTING CONFIGURATION FOR FILECOIN MAINNET CAN AND WILL CHANGE. YOU HAVE BEEN WARNED.
-
-## Example configuration
-
-The setup below is a minimal example for sealing 32 GiB sectors on Lotus:
-
-- 2 TB of hard drive space.
-- 8 core CPU
-- 128 GiB of RAM
-
-Note that 1GB sectors don't require as high of specs, but are likely to be removed as we improve the performance of 32GB sector sealing.
-
-For the first part of the sealing process, AMD CPU's are __highly recommended__, because of the `Intel SHA Extensions` instruction set that is available there ever since the `Zen` microarchitecture. Hence, AMD CPU's seem to perform much better on the testnet than other CPU's. Contrary to what the name implies, this extended instruction set is not available on recent Intel desktop/server chips. 
-
-## Testnet discoveries
-
-- If you only have 128GiB of ram, enabling 256GB of **NVMe** swap on an SSD will help you avoid out-of-memory issues while mining.
-
-## Benchmarked GPUs
-
-GPUs are a must for getting **block rewards**. Here are a few that have been confirmed to generate **SNARKs** quickly enough to successfully mine blocks on the Lotus Testnet.
-
-- GeForce RTX 2080 Ti
-- GeForce RTX 2080 SUPER
-- GeForce RTX 2080
-- GeForce GTX 1080 Ti
-- GeForce GTX 1080
-- GeForce GTX 1060
-
-## Testing other GPUs
-
-If you want to test a GPU that is not explicitly supported, use the following global **environment variable**:
-
-```sh
-BELLMAN_CUSTOM_GPU="<NAME>:<NUMBER_OF_CORES>"
-```
-
-Here is an example of trying a GeForce GTX 1660 Ti with 1536 cores.
-
-```sh
-BELLMAN_CUSTOM_GPU="GeForce GTX 1660 Ti:1536"
-```
-
-To get the number of cores for your GPU, you will need to check your card’s specifications.
-
-## Benchmarking
-
-Here is a [benchmarking tool](https://github.com/filecoin-project/lotus/tree/master/cmd/lotus-bench) and a [GitHub issue thread](https://github.com/filecoin-project/lotus/issues/694) for those who wish to experiment with and contribute hardware setups for the **Filecoin Testnet**.

documentation/en/hardware.md

@@ -1,7 +0,0 @@
-# Hardware
-
-> This page is a work in progress. Exact mining requirements are still in the works.
-
-Lotus can build and run on most [Linux](https://ubuntu.com/) and [MacOS](https://www.apple.com/macos) systems with at least 8GiB of RAM.
-
-Windows is not yet supported.

documentation/en/install-lotus-arch.md

@@ -1,51 +0,0 @@
-# Arch Linux Instructions
-
-These steps will install the following dependencies:
-
-- go (1.14 or higher)
-- gcc (7.4.0 or higher)
-- git (version 2 or higher)
-- bzr (some go dependency needs this)
-- jq
-- pkg-config
-- opencl-icd-loader
-- opencl driver (like nvidia-opencl on arch) (for GPU acceleration)
-- opencl-headers (build)
-- rustup (proofs build)
-- llvm (proofs build)
-- clang (proofs build)
-
-### Install dependencies
-
-```sh
-sudo pacman -Syu opencl-icd-loader gcc git bzr jq pkg-config opencl-icd-loader opencl-headers
-```
-
-### Install Go 1.14
-
-Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install).
-
-### Clone the Lotus repository
-
-```sh
-git clone https://github.com/filecoin-project/lotus.git
-cd lotus/
-```
-
-### Build the Lotus binaries from source and install
-
-! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively**
-
-```sh
-make clean && make all
-sudo make install
-```
-
-#### Native Filecoin FFI building
-
-```sh
-env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all
-sudo make install
-```
-
-After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet).

documentation/en/install-lotus-fedora.md

@@ -1,54 +0,0 @@
-# Fedora Instructions
-
-> tested on 30
-
-**NOTE:** If you have an AMD GPU the opencl instructions may be incorrect...
-
-These steps will install the following dependencies:
-
-- go (1.14 or higher)
-- gcc (7.4.0 or higher)
-- git (version 2 or higher)
-- bzr (some go dependency needs this)
-- jq
-- pkg-config
-- rustup (proofs build)
-- llvm (proofs build)
-- clang (proofs build)
-
-### Install dependencies
-
-```sh
-$ sudo dnf -y update
-$ sudo dnf -y install gcc git bzr jq pkgconfig mesa-libOpenCL mesa-libOpenCL-devel opencl-headers ocl-icd ocl-icd-devel clang llvm
-$ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
-```
-
-### Install Go 1.14
-
-Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install).
-
-### Clone the Lotus repository
-
-```sh
-git clone https://github.com/filecoin-project/lotus.git
-cd lotus/
-```
-
-### Build the Lotus binaries from source and install
-
-! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively**
-
-```sh
-$ make clean && make all
-$ sudo make install
-```
-
-#### Native Filecoin FFI building
-
-```sh
-env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all
-sudo make install
-```
-
-After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus TestNet](https://lotu.sh/en+join-testnet).

documentation/en/install-lotus-macos.md

@@ -1,62 +0,0 @@
-# MacOS Instructions
-
-## Get XCode Command Line Tools
-
-To check if you already have the XCode Command Line Tools installed via the CLI, run:
-
-```sh
-xcode-select -p
-```
-
-If this command returns a path, you can move on to the next step. Otherwise, to install via the CLI, run:
-
-```sh
-xcode-select --install
-```
-
-To update, run:
-
-```sh
-sudo rm -rf /Library/Developer/CommandLineTools
-xcode-select --install
-```
-
-## Get HomeBrew
-
-We recommend that MacOS users use [HomeBrew](https://brew.sh) to install each the necessary packages.
-
-Check if you have HomeBrew:
-
-```sh
-brew -v
-```
-
-This command returns a version number if you have HomeBrew installed and nothing otherwise.
-
-In your terminal, enter this command to install Homebrew:
-
-```sh
-/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-```
-
-Use the command `brew install` to install the following packages:
-
-```sh
-brew install go bzr jq pkg-config rustup
-```
-
-Clone
-
-```sh
-git clone https://github.com/filecoin-project/lotus.git
-cd lotus/
-```
-
-Build
-
-```sh
-make clean && make all
-sudo make install
-```
-
-After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet).

documentation/en/install-lotus-ubuntu.md

@@ -1,54 +0,0 @@
-# Ubuntu Instructions
-
-These steps will install the following dependencies:
-
-- go (1.14 or higher)
-- gcc (7.4.0 or higher)
-- git (version 2 or higher)
-- bzr (some go dependency needs this)
-- jq
-- pkg-config
-- opencl-icd-loader
-- opencl driver (like nvidia-opencl on arch) (for GPU acceleration)
-- opencl-headers (build)
-- rustup (proofs build)
-- llvm (proofs build)
-- clang (proofs build)
-
-### Install dependencies
-
-```sh
-sudo apt update
-sudo apt install mesa-opencl-icd ocl-icd-opencl-dev gcc git bzr jq pkg-config curl
-sudo apt upgrade
-```
-
-### Install Go 1.14
-
-Install the latest version of Go by following [the docs on their website](https://golang.org/doc/install).
-
-### Clone the Lotus repository
-
-```sh
-git clone https://github.com/filecoin-project/lotus.git
-cd lotus/
-```
-
-### Build the Lotus binaries from source and install
-
-! **If you are running an AMD platform or if your CPU supports SHA extensions you will want to build the Filecoin proofs natively**
-
-```sh
-make clean && make all
-sudo make install
-```
-
-#### Native Filecoin FFI building
-
-```sh
-env env RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make clean deps all
-sudo make install
-```
-
-
-After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus Testnet](https://lotu.sh/en+join-testnet).

documentation/en/install-systemd-services.md

@@ -1,145 +0,0 @@
-# Use Lotus with systemd
-
-Lotus is capable of running as a systemd service daemon. You can find installable service files for systemd in the [lotus repo scripts directory](https://github.com/filecoin-project/lotus/tree/master/scripts) as files with `.service` extension. In order to install these service files, you can copy these `.service` files to the default systemd unit load path.
-
-The services expect their binaries to be present in `/usr/local/bin/`. You can use `make` to install them by running:
-
-```sh
-$ sudo make install
-```
-
-for `lotus` and `lotus-miner` and 
-
-```sh
-$ sudo make install-chainwatch
-```
-
-for the `chainwatch` tool.
-
-## Installing services via `make`
-
-If your host uses the default systemd unit load path, the `lotus-daemon` and `lotus-miner` services can be installed by running:
-
-```sh
-$ sudo make install-services
-```
-
-To install the the `lotus-chainwatch` service run:
-
-```sh
-$ sudo make install-chainwatch-service
-```
-
-You can install all services together by running:
-
-```sh
-$ sudo make install-all-services
-```
-
-The `lotus-daemon` and the `lotus-miner` services can be installed individually too by running:
-
-```sh
-$ sudo make install-daemon-service
-```
-
-and 
-
-```sh
-$ sudo make install-miner-service
-```
-
-### Notes
-
-When installing the `lotus-miner` and/or `lotus-chainwatch` service the `lotus-daemon` service gets automatically installed since the other two services depend on it being installed to run.
-
-All `install-*-service*` commands will install the latest binaries in the lotus build folders to `/usr/local/bin/`. If you do not want to use the latest build binaries please copy the `*.service` files by hand.
-
-## Removing via `make`
-
-All services can beremoved via `make`. To remove all services together run:
-
-```sh
-$ sudo make clean-all-services
-```
-
-Individual services can be removed by running:
-
-```sh
-$ sudo make clean-chainwatch-services
-$ sudo make clean-miner-services
-$ sudo make clean-daemon-services
-```
-
-### Notes
-
-The services will be stoppend and disabled when removed.
-
-Removing the `lotus-daemon` service will automatically remove the depending services `lotus-miner` and `lotus-chainwatch`.
-
-
-## Controlling services
-
-All service can be controlled with the `systemctl`. A few basic control commands are listed below. To get detailed infos about the capabilities of the `systemctl` command please consult your distributions man pages by running: 
-
-```sh
-$ man systemctl
-```
-
-### Start/Stop services
-
-You can start the services by running:
-
-```sh
-$ sudo systemctl start lotus-daemon
-$ sudo systemctl start lotus-miner
-$ sudo systemctl start lotus-chainwatch
-```
-
-and can be stopped by running:
-
-```sh
-$ sudo systemctl stop lotus-daemon
-$ sudo systemctl stop lotus-miner
-$ sudo systemctl stop lotus-chainwatch
-```
-
-### Enabling services on startup
-
-To enable the services to run automatically on startup execute:
-
-```sh
-$ sudo systemctl enable lotus-daemon
-$ sudo systemctl enable lotus-miner
-$ sudo systemctl enable lotus-chainwatch
-```
-
-To disable the services on startup run:
-
-```sh
-$ sudo systemctl disable lotus-daemon
-$ sudo systemctl disable lotus-miner
-$ sudo systemctl disable lotus-chainwatch
-```
-### Notes
-
-Systemd will not let services be enabled or started without their requirements. Starting the `lotus-chainwatch` and/or `lotus-miner` service with automatically start the `lotus-daemon` service (if installed!). Stopping the `lotus-daemon` service will stop the other two services. The same pattern is executed for enabling and disabling the services. 
-
-## Interacting with service logs
-
-Logs from the services can be reviewed using `journalctl`.
-
-### Follow logs from a specific service unit
-
-```sh
-$ sudo journalctl -u lotus-daemon -f
-```
-
-### View logs in reverse order
-
-```sh
-$ sudo journalctl -u lotus-miner -r
-```
-
-### Log files
-
-Besides the systemd service logs all services save their own log files in `/var/log/lotus/`.

documentation/en/join-testnet.md

@@ -1,93 +0,0 @@
-# Join Testnet
-
-## Introduction
-
-Anyone can set up a **Lotus Node** and connect to the **Lotus Testnet**. This is the best way to explore the current CLI and the **Filecoin Decentralized Storage Market**.
-
-## Note: Using the Lotus Node from China
-
-If you are trying to use `lotus` from China. You should set this **environment variable** on your machine:
-
-```sh
-export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/"
-```
-
-## Get started
-
-Start the **daemon** using the default configuration in `./build`:
-
-```sh
-lotus daemon
-```
-
-In another terminal window, check your connection with peers:
-
-```sh
-lotus net peers | wc -l
-```
-
-In order to connect to the network, you need to be connected to at least 1 peer. If you’re seeing 0 peers, read our [troubleshooting notes](https://lotu.sh/en+setup-troubleshooting).
-
-Make sure that you have a reasonable "open files limit" set on your machine, such as 10000. If you're seeing a lower value, such as 256 (default on macOS), read our [troubleshooting notes](https://lotu.sh/en+setup-troubleshooting) on how to update it prior to starting the Lotus daemon.
-
-## Chain sync
-
-While the daemon is running, the next requirement is to sync the chain. Run the command below to view the chain sync progress. To see current chain height, visit the [network stats page](https://stats.testnet.filecoin.io/).
-
-```sh
-lotus sync wait
-```
-
-- This step will take anywhere between a few hours to a couple of days.
-- You will be able to perform **Lotus Testnet** operations after it is finished.
-
-## Create your first address
-
-Initialize a new wallet:
-
-```sh
-lotus wallet new
-```
-
-Sometimes your operating system may limit file name length to under 150 characters. You need to use a file system that supports long filenames.
-
-Here is an example of the response:
-
-```sh
-t1aswwvjsae63tcrniz6x5ykvsuotlgkvlulnqpsi
-```
-
-- Visit the [faucet](http://spacerace.faucet.glif.io/) to add funds.
-- Paste the address you created under REQUEST.
-- Press the Request button.
-
-## Check wallet address balance
-
-Wallet balances in the Lotus Testnet are in **FIL**, the smallest denomination of FIL is an **attoFil**, where 1 attoFil = 10^-18 FIL.
-
-```sh
-lotus wallet balance <YOUR_NEW_ADDRESS>
-```
-
-You will not see any attoFIL in your wallet if your **chain** is not fully synced.
-
-## Send FIL to another wallet
-
-To send FIL to another wallet from your default account, use this command:
-
-```
-lotus send <target> <amount>
-```
-
-## Configure your node's connectivity
-
-To effectively accept incoming storage & retrieval deals, your Lotus node needs to be accessible to other nodes on the network. To improve your connectivity, be sure to: 
-
-- [Set the multiaddresses for you miner to listen on](https://docs.filecoin.io/mine/connectivity/#setting-multiaddresses)
-- [Maintain a healthy peer count](https://docs.filecoin.io/mine/connectivity/#checking-peer-count)
-- [Enable port forwarding](https://docs.filecoin.io/mine/connectivity/#port-forwarding)
-- [Configure your public IP address and port](https://docs.filecoin.io/mine/connectivity/#setting-a-public-ip-address)
-
-## Monitor the dashboard
-
-To see the latest network activity, including **chain block height**, **block height**, **blocktime**, **total network power**, largest **block producer miner**, check out the [monitoring dashboard](https://stats.testnet.filecoin.io).

documentation/en/local-dev-net.md

@@ -1,54 +0,0 @@
-# Setup Local Devnet
-
-Build the Lotus Binaries in debug mode, This enables the use of 2048 byte sectors.
-
-```sh
-make 2k
-```
-
-Set the `LOTUS_SKIP_GENESIS_CHECK` environment variable to `_yes_`. This tells your
-Lotus node that it's okay if the genesis being used doesn't match any baked-in
-genesis.
-
-```sh
-export LOTUS_SKIP_GENESIS_CHECK=_yes_
-```
-
-Download the 2048 byte parameters:
-```sh
-./lotus fetch-params 2048
-```
-
-Pre-seal some sectors:
-
-```sh
-./lotus-seed pre-seal --sector-size 2KiB --num-sectors 2
-```
-
-Create the genesis block and start up the first node:
-
-```sh
-./lotus-seed genesis new localnet.json
-./lotus-seed genesis add-miner localnet.json ~/.genesis-sectors/pre-seal-t01000.json
-./lotus daemon --lotus-make-genesis=devgen.car --genesis-template=localnet.json --bootstrap=false
-```
-
-Then, in another console, import the genesis miner key:
-
-```sh
-./lotus wallet import --as-default ~/.genesis-sectors/pre-seal-t01000.key
-```
-
-Set up the genesis miner:
-
-```sh
-./lotus-miner init --genesis-miner --actor=t01000 --sector-size=2KiB --pre-sealed-sectors=~/.genesis-sectors --pre-sealed-metadata=~/.genesis-sectors/pre-seal-t01000.json --nosync
-```
-
-Now, finally, start up the miner:
-
-```sh
-./lotus-miner run --nosync
-```
-
-If all went well, you will have your own local Lotus Devnet running.

documentation/en/miner-deals.md

@@ -1,39 +0,0 @@
-# Information for Miners 
-
-Here is how a miner can get set up to accept storage deals. The first step is
-to install a Lotus node and sync to the top of the chain.
-
-## Set up an ask
-
-```
-lotus-miner set-price <price>
-```
-
-This command will set up your miner to accept deal proposals that meet the input price.
-The price is inputted in FIL per GiB per epoch, and the default is 0.0000000005. 
-
-## Ensure you can be discovered
-
-Clients need to be able to find you in order to make storage deals with you. 
-While there isn't necessarily anything you need to do to become discoverable, here are some things you can
-try to check that people can connect to you. 
-
-To start off, make sure you are connected to at least some peers, and your port is 
-open and working.
-
-### Connect to your own node
-
-If you are in contact with someone else running Lotus, you can ask them to try connecting
-to your node. To do so, provide them your peer ID, which you can get by running `lotus net id` on
-your node.
-
-They can then try running `lotus net findpeer <peerID>` to get your address(es), and can then
-run `lotus net connect <address>` to connect to you. If successful, your node will now
-appear on their peers list (run `lotus net peers` to check).
-
-You can also check this by running a second instance of Lotus yourself.
-
-### Query your own ask
-
-A client should be able to find your ask by running `lotus client query-ask <minerID>`. If 
-someone is not able to retrieve your ask by doing so, then there is an issue with your node.

documentation/en/mining-lotus-worker.md

@@ -1,81 +0,0 @@
-# Lotus Worker
-
-The **Lotus Worker** is an extra process that can offload heavy processing tasks from your **Lotus Miner**. The sealing process automatically runs in the **Lotus Miner** process, but you can use the Worker on another machine communicating over a fast network to free up resources on the machine running the mining process.
-
-## Note: Using the Lotus Worker from China
-
-If you are trying to use `lotus-worker` from China. You should set this **environment variable** on your machine:
-
-```sh
-export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/"
-```
-
-## Get Started
-
-Make sure that the `lotus-worker` is compiled and installed by running:
-
-```sh
-make lotus-worker
-```
-
-## Setting up the Miner
-
-First, you will need to ensure your `lotus-miner`'s API is accessible over the network.
-
-To do this, open up `~/.lotusminer/config.toml` (Or if you manually set `LOTUS_MINER_PATH`, look under that directory) and look for the API field.
-
-Default config:
-
-```toml
-[API]
-ListenAddress = "/ip4/127.0.0.1/tcp/2345/http"
-RemoteListenAddress = "127.0.0.1:2345"
-```
-
-To make your node accessible over the local area network, you will need to determine your machines IP on the LAN, and change the `127.0.0.1` in the file to that address.
-
-A more permissive and less secure option is to change it to `0.0.0.0`. This will allow anyone who can connect to your computer on that port to access the [API](https://lotu.sh/en+api). They will still need an auth token.
-
-`RemoteListenAddress` must be set to an address which other nodes on your network will be able to reach.
-
-Next, you will need to [create an authentication token](https://lotu.sh/en+api-scripting-support#generate-a-jwt-46). All Lotus APIs require authentication tokens to ensure your processes are as secure against attackers attempting to make unauthenticated requests to them.
-
-### Connect the Lotus Worker
-
-On the machine that will run `lotus-worker`, set the `MINER_API_INFO` environment variable to `TOKEN:MINER_NODE_MULTIADDR`. Where `TOKEN` is the token we created above, and `NIMER_NODE_MULTIADDR` is the `multiaddr` of the **Lotus Miner** API that was set in `config.toml`.
-
-Once this is set, run:
-
-```sh
-lotus-worker run
-```
-
-If you are running multiple workers on the same host, you will need to specify the `--listen` flag and ensure each worker is on a different port.
-
-To check that the **Lotus Worker** is connected to your **Lotus Miner**, run `lotus-miner sealing workers` and check that the remote worker count has increased.
-
-```sh
-why@computer ~/lotus> lotus-miner sealing workers
-Worker 0, host computer
-        CPU:  [                                                                ] 0 core(s) in use
-        RAM:  [||||||||||||||||||                                              ] 28% 18.1 GiB/62.7 GiB
-        VMEM: [||||||||||||||||||                                              ] 28% 18.1 GiB/62.7 GiB
-        GPU: GeForce RTX 2080, not used
-
-Worker 1, host othercomputer
-        CPU:  [                                                                ] 0 core(s) in use
-        RAM:  [||||||||||||||                                                  ] 23% 14 GiB/62.7 GiB
-        VMEM: [||||||||||||||                                                  ] 23% 14 GiB/62.7 GiB
-        GPU: GeForce RTX 2080, not used
-```
-
-### Running locally for manually managing process priority
-
-You can also run the **Lotus Worker** on the same machine as your **Lotus Miner**, so you can manually manage the process priority.
-To do so you have to first __disable all seal task types__ in the miner config. This is important to prevent conflicts between the two processes.
-
-You can then run the miner on your local-loopback interface; 
-
-```sh
-lotus-worker run
-```

documentation/en/mining-troubleshooting.md

@@ -1,66 +0,0 @@
-# Mining Troubleshooting
-
-## Config: Filecoin Proof Parameters directory
-
-If you want to put the **Filecoin Proof Parameters** in a different directory, use the following environment variable:
-
-```sh
-FIL_PROOFS_PARAMETER_CACHE
-```
-
-## Error: Can't acquire bellman.lock
-
-The **Bellman** lockfile is created to lock a GPU for a process. This bug can occur when this file isn't properly cleaned up:
-
-```sh
-mining block failed: computing election proof: github.com/filecoin-project/lotus/miner.(*Miner).mineOne
-```
-
-This bug occurs when the miner can't acquire the `bellman.lock`. To fix it you need to stop the `lotus-miner` and remove `/tmp/bellman.lock`.
-
-## Error: Failed to get api endpoint
-
-```sh
-lotus-miner info
-# WARN  main  lotus-miner/main.go:73  failed to get api endpoint: (/Users/myrmidon/.lotusminer) %!w(*errors.errorString=&{API not running (no endpoint)}):
-```
-
-If you see this, that means your **Lotus Miner** isn't ready yet. You need to finish [syncing the chain](https://lotu.sh/en+join-testnet).
-
-## Error: Your computer may not be fast enough
-
-```sh
-CAUTION: block production took longer than the block delay. Your computer may not be fast enough to keep up
-```
-
-If you see this, that means your computer is too slow and your blocks are not included in the chain, and you will not receive any rewards.
-
-## Error: No space left on device
-
-```sh
-lotus-miner sectors pledge
-# No space left on device (os error 28)
-```
-
-If you see this, that means `pledge-sector` wrote too much data to `$TMPDIR` which by default is the root partition (This is common for Linux setups). Usually your root partition does not get the largest partition of storage so you will need to change the environment variable to something else.
-
-## Error: GPU unused
-
-If you suspect that your GPU is not being used, first make sure it is properly configured as described in the [testing configuration page](hardware-mining.md). Once you've done that (and set the `BELLMAN_CUSTOM_GPU` as appropriate if necessary) you can verify your GPU is being used by running a quick lotus-bench benchmark.
-
-First, to watch GPU utilization run `nvtop` in one terminal, then in a separate terminal, run:
-
-```sh
-make bench
-./bench sealing --sector-size=2KiB
-```
-
-This process uses a fair amount of GPU, and generally takes ~4 minutes to complete. If you do not see any activity in nvtop from lotus during the entire process, it is likely something is misconfigured with your GPU.
-
-## Checking Sync Progress
-
-You can use this command to check how far behind you are on syncing:
-
-```sh
-date -d @$(./lotus chain getblock $(./lotus chain head) | jq .Timestamp)
-```

documentation/en/mining.md

@@ -1,149 +0,0 @@
-# Storage Mining
-
-Here are instructions to learn how to perform storage mining. For hardware specifications please read [this](https://lotu.sh/en+hardware-mining).
-
-It is useful to [join the Testnet](https://lotu.sh/en+join-testnet) prior to attempting storage mining for the first time.
-
-## Note: Using the Lotus Miner from China
-
-If you are trying to use `lotus-miner` from China. You should set this **environment variable** on your machine.
-
-```sh
-export IPFS_GATEWAY="https://proof-parameters.s3.cn-south-1.jdcloud-oss.com/ipfs/"
-```
-
-## Get started
-
-Please ensure that at least one **BLS address** (starts with `t3`) in your wallet exists with the following command:
-
-```sh
-lotus wallet list
-```
-
-If you do not have a bls address, create a new bls wallet:
-
-```sh
-lotus wallet new bls
-```
-
-With your wallet address:
-
-- Visit the [faucet](http://spacerace.faucet.glif.io/)
-- Paste the address you created under REQUEST.
-- Press the Request button.
-- Run `/lotus-miner init --owner=<blsAddress> --worker=<blsAddress>`
-
-You will have to wait some time for this operation to complete.
-
-## Mining
-
-To mine:
-
-```sh
-lotus-miner run
-```
-
-If you are downloading **Filecoin Proof Parameters**, the download can take some time.
-
-Get information about your miner:
-
-```sh
-lotus-miner info
-# example: miner id `t0111`
-```
-
-**Seal** random data to start producing **PoSts**:
-
-```sh
-lotus-miner sectors pledge
-```
-
-- Warning: On Linux configurations, this command will write data to `$TMPDIR` which is not usually the largest partition. You should point the value to a larger partition if possible.
-
-Get **miner power** and **sector usage**:
-
-```sh
-lotus state power
-# returns total power
-
-lotus state power <miner>
-
-lotus state sectors <miner>
-```
-
-## Performance tuning
-
-### `FIL_PROOFS_MAXIMIZE_CACHING=1` Environment variable
-
-This env var can be used with `lotus-miner`, `lotus-worker`, and `lotus-bench` to make the precommit1 step faster at the cost of some memory use (1x sector size)
-
-### `FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1` Environment variable
-
-This env var can be used with `lotus-miner`, `lotus-worker`, and `lotus-bench` to enable experimental precommit2 GPU acceleration
-
-### Setting multiaddresses
-
-Set multiaddresses for the miner to listen on in a miner's `config.toml` file
-(by default, it is located at `~/.lotusminer/config.toml`). The `ListenAddresses` in this file should be interface listen addresses (usually `/ip4/0.0.0.0/tcp/PORT`), and the `AnnounceAddresses` should match the addresses being passed to `set-addrs`.
-
-The addresses passed to `set-addrs` parameter in the commands below should be currently active and dialable; confirm they are before entering them.
-
-Once the config file has been updated, set the on-chain record of the miner's listen addresses:
-
-```
-lotus-miner actor set-addrs <multiaddr_1> <multiaddr_2> ... <multiaddr_n>
-```
-
-This updates the `MinerInfo` object in the miner's actor, which will be looked up
-when a client attempts to make a deal. Any number of addresses can be provided.
-
-Example:
-
-```
-lotus-miner actor set-addrs /ip4/123.123.73.123/tcp/12345 /ip4/223.223.83.223/tcp/23456
-```
-
-# Separate address for windowPoSt messages
-
-WindowPoSt is the mechanism through which storage is verified in Filecoin. It requires miners to submit proofs for all sectors every 24h, which require sending messages to the chain.
-
-Because many other mining related actions require sending messages to the chain, and not all of those are "high value", it may be desirable to use a separate account to send PoSt messages from. This allows for setting lower GasFeeCaps on the lower value messages without creating head-of-line blocking problems for the PoSt messages in congested chain conditions
-
-To set this up, first create a new account, and send it some funds for gas fees:
-```sh
-lotus wallet new bls
-t3defg...
-
-lotus send t3defg... 100
-```
-
-Next add the control address
-```sh
-lotus-miner actor control set t3defg...
-Add t3defg...
-Pass --really-do-it to actually execute this action
-```
-
-Now actually set the addresses
-```sh
-lotus-miner actor control set --really-do-it t3defg...
-Add t3defg...
-Message CID: bafy2..
-```
-
-Wait for the message to land on chain
-```sh
-lotus state wait-msg bafy2..
-...
-Exit Code: 0
-...
-```
-
-Check miner control address list to make sure the address was correctly setup
-```sh
-lotus-miner actor control list
-name       ID      key           use    balance
-owner      t01111  t3abcd...  other  300 FIL
-worker     t01111  t3abcd...  other  300 FIL
-control-0  t02222  t3defg...  post   100 FIL
-```

documentation/en/payment-channels.md

@@ -1,111 +0,0 @@
-# Payment Channels
-
-Payment channels are used to transfer funds between two actors.
-
-For example in lotus a payment channel is created when a client wants to fetch data from a provider.
-The client sends vouchers for the payment channel, and the provider sends data in response.
-
-The payment channel is created on-chain with an initial amount.
-Vouchers allow the client and the provider to exchange funds incrementally off-chain.
-The provider can submit vouchers to chain at any stage.
-Either party to the payment channel can settle the payment channel on chain.
-After a settlement period (currently 12 hours) either party to the payment channel can call collect on chain.
-Collect sends the value of submitted vouchers to the channel recipient (the provider), and refunds the remaining channel balance to the channel creator (the client).
-
-Vouchers have a lane, a nonce and a value, where vouchers with a higher nonce supersede vouchers with a lower nonce in the same lane.
-Each deal is created on a different lane. 
-
-Note that payment channels and vouchers can be used for any situation in which two parties need to incrementally transfer value between each other off-chain.
-
-## Using the CLI
-
-For example a client creates a payment channel to a provider with value 10 FIL.
-
-```sh
-$ lotus paych add-funds <client addr> <provider addr> 10
-<channel addr>
-```
-
-The client creates a voucher in lane 0 (implied) with nonce 1 (implied) and value 2.
-
-```sh
-$ lotus paych voucher create <channel addr> 2
-<voucher>
-```
-
-The client sends the voucher to the provider and the provider adds the voucher to their local store.
-
-```sh
-$ lotus paych voucher add <channel addr> <voucher>
-```
-
-The provider sends some data to the client.
-
-The client creates a voucher in lane 0 (implied) with nonce 2 (implied) and value 4.
-
-```sh
-$ lotus paych voucher create <channel addr> 4
-<voucher>
-```
-
-The client sends the voucher to the provider and the provider adds the voucher and sends back more data.
-etc.
-
-The client can add value to the channel after it has been created by calling `paych add-funds` with the same client and provider addresses.
-
-```sh
-$ lotus paych add-funds <client addr> <provider addr> 5
-<channel addr> # Same address as above. Channel now has 15
-```
-
-Once the client has received all their data, they may settle the channel.
-Note that settlement doesn't have to be done immediately.
-For example the client may keep the channel open as long as it wants to continue making deals with the provider.
-
-```sh
-$ lotus paych settle <channel addr>
-```
-
-The provider can submit vouchers to chain (note that lotus does this automatically when it sees a settle message appear on chain).
-The provider may have received many vouchers with incrementally higher values.
-The provider should submit the best vouchers. Note that there will be one best voucher for each lane.
-
-```sh
-$ lotus paych voucher best-spendable <channel addr>
-<voucher>
-<voucher>
-<voucher>
-
-$ lotus paych voucher submit <channel addr> <voucher>
-```
-
-Once the settlement period is over, either the client or provider can call collect to disburse funds.
-
-```sh
-$ lotus paych collect <channel addr>
-```
-
-Check the status of a channel that is still being created using `lotus paych status-by-from-to`.
-
-```sh
-$ lotus paych status-by-from-to <from addr> <to addr>
-Creating channel
-  From:          t3sb6xzvs6rhlziatagevxpp3dwapdolurtkpn4kyh3kgoo4tn5o7lutjqlsnvpceztlhxu3lzzfe34rvpsjgq
-  To:            t1zip4sblhyrn4oxygzsm6nafbsynp2avmk3xafea
-  Pending Amt:   10000
-  Wait Sentinel: bafy2bzacedk2jidsyxcynusted35t5ipkhu2kpiodtwyjr3pimrhke6f5pqbm
-```
-
-Check the status of a channel that has been created using `lotus paych status`.
-
-```sh
-$ lotus paych status <channel addr>
-Channel exists
-  Channel:              t2nydpzhmeqkmid5smtqnowlr2mr5az6rexpmyv6i
-  From:                 t3sb6xzvs6rhlziatagevxpp3dwapdolurtkpn4kyh3kgoo4tn5o7lutjqlsnvpceztlhxu3lzzfe34rvpsjgq
-  To:                   t1zip4sblhyrn4oxygzsm6nafbsynp2avmk3xafea
-  Confirmed Amt:        10000
-  Pending Amt:          6000
-  Queued Amt:           3000
-  Voucher Redeemed Amt: 2000
-```

documentation/en/retrieving-data.md

@@ -1,27 +0,0 @@
-# Retrieving Data
-
-> There are recent bug reports with these instructions. If you happen to encounter any problems, please create a [GitHub issue](https://github.com/filecoin-project/lotus/issues/new) and a maintainer will address the problem as soon as they can.
-
-Here are the operations you can perform after you have stored and sealed a **Data CID** with the **Lotus Miner** in the network.
-
-If you would like to learn how to store a **Data CID** on a miner, read the instructions [here](https://lotu.sh/en+storing-data).
-
-## Find by Data CID
-
-```sh
-lotus client find <Data CID>
-# LOCAL
-# RETRIEVAL <miner>@<miner peerId>-<deal funds>-<size>
-```
-
-## Retrieve by Data CID
-
-All fields are required.
-
-```sh
-lotus client retrieve <Data CID> <outfile>
-```
-
-If the outfile does not exist it will be created in the Lotus repository directory.
-
-This command will initiate a **retrieval deal** and write the data to your computer. This process may take 2 to 10 minutes.

documentation/en/setting-a-static-port.md

@@ -1,54 +0,0 @@
-# Static Ports
-
-Depending on how your network is set up, you may need to set a static port to successfully connect to peers to perform storage deals with your **Lotus Miner**.
-
-## Setup
-
-To change the random **swarm port**, you may edit the `config.toml` file located under `$LOTUS_MINER_PATH`. The default location of this file is `$HOME/.lotusminer`.
-
-To change the port to `1347`:
-
-```sh
-[Libp2p]
-  ListenAddresses = ["/ip4/0.0.0.0/tcp/1347", "/ip6/::/tcp/1347"]
-```
-
-After changing the port value, restart your **daemon**.
-
-## Announce Addresses
-
-If the **swarm port** is port-forwarded from another address, it is possible to control what addresses
-are announced to the network.
-
-```sh
-[Libp2p]
-  AnnounceAddresses = ["/ip4/<public-ip>/tcp/1347"]
-```
-
-If non-empty, this array specifies the swarm addresses to announce to the network. If empty, the daemon will announce inferred swarm addresses.
-
-Similarly, it is possible to set `NoAnnounceAddresses` with an array of addresses to not announce to the network.
-
-## Ubuntu's Uncomplicated Firewall
-
-Open firewall manually:
-
-```sh
-ufw allow 1347/tcp
-```
-
-Or open and modify the profile located at `/etc/ufw/applications.d/lotus-daemon`:
-
-```sh
-[Lotus Daemon]
-title=Lotus Daemon
-description=Lotus Daemon firewall rules
-ports=1347/tcp
-```
-
-Then run these commands:
-
-```sh
-ufw update lotus-daemon
-ufw allow lotus-daemon
-```

documentation/en/setup-troubleshooting.md

@@ -1,46 +0,0 @@
-# Setup Troubleshooting
-
-## Config: Clearing data
-
-Here is a command that will delete your chain data, stored wallets, stored data and any miners you have set up:
-
-```sh
-rm -rf ~/.lotus ~/.lotusminer
-```
-
-This command usually resolves any issues with running `lotus` but it is not always required for updates. We will share information about when resetting your chain data and miners is required for an update in the future.
-
-## Error: Failed to connect bootstrap peer
-
-```sh
-WARN  peermgr peermgr/peermgr.go:131  failed to connect to bootstrap peer: failed to dial : all dials failed
-  * [/ip4/147.75.80.17/tcp/1347] failed to negotiate security protocol: connected to wrong peer
-```
-
-- Try running the build steps again and make sure that you have the latest code from GitHub.
-
-```sh
-ERROR hello hello/hello.go:81 other peer has different genesis!
-```
-
-- Try deleting your file system's `~/.lotus` directory. Check that it exists with `ls ~/.lotus`.
-
-```sh
-- repo is already locked
-```
-
-- You already have another lotus daemon running.
-
-## Config: Open files limit
-
-On most systems you can check the open files limit with:
-
-```sh
-ulimit -n
-```
-
-You can also modify this number by using the `ulimit` command. It gives you the ability to control the resources available for the shell or process started by it. If the number is below 10000, you can change it with the following command prior to starting the Lotus daemon:
-
-```sh
-ulimit -n 10000
-```

documentation/en/storing-data-troubleshooting.md

@@ -1,27 +0,0 @@
-# Storage Troubleshooting
-
-## Error: Routing: not found
-
-```sh
-WARN  main  lotus/main.go:72  routing: not found
-```
-
-- This miner is offline.
-
-## Error: Failed to start deal
-
-```sh
-WARN  main  lotus/main.go:72  failed to start deal: computing commP failed: generating CommP: Piece must be at least 127 bytes
-```
-
-- There is a minimum file size of 127 bytes.
-
-## Error: 0kb file response during retrieval
-
-In order to retrieve a file, it must be sealed. Miners can check sealing progress with this command:
-
-```sh
-lotus-miner sectors list
-```
-
-When sealing is complete, `pSet: NO` will become `pSet: YES`. From now on the **Data CID** is [retrievable](https://lotu.sh/en+retrieving-data) from the **Lotus Miner**.

documentation/en/storing-data.md

@@ -1,62 +0,0 @@
-# Storing Data
-
-> There are recent bug reports with these instructions. If you happen to encounter any problems, please create a [GitHub issue](https://github.com/filecoin-project/lotus/issues/new) and a maintainer will address the problem as soon as they can.
-
-Here are instructions for how to store data on the **Lotus Testnet**.
-
-## Adding a file locally
-
-Adding a file locally allows you to make miner deals on the **Lotus Testnet**.
-
-```sh
-lotus client import ./your-example-file.txt
-```
-
-Upon success, this command will return a **Data CID**.
-
-## List your local files
-
-The command to see a list of files by `CID`, `name`, `size` in bytes, and `status`:
-
-```sh
-lotus client local
-```
-
-An example of the output:
-
-```sh
-bafkreierupr5ioxn4obwly4i2a5cd2rwxqi6kwmcyyylifxjsmos7hrgpe Development/sample-1.txt 2332 ok
-bafkreieuk7h4zs5alzpdyhlph4lxkefowvwdho3a3pml6j7dam5mipzaii Development/sample-2.txt 30618 ok
-```
-
-## Make a Miner Deal on Lotus Testnet
-
-Get a list of all miners that can store data:
-
-```sh
-lotus state list-miners
-```
-
-Get the requirements of a miner you wish to store data with:
-
-```sh
-lotus client query-ask <miner>
-```
-
-Store a **Data CID** with a miner:
-
-```sh
-lotus client deal <Data CID> <miner> <price> <duration>
-```
-
-Check the status of a deal:
-
-```sh
-lotus client list-deals
-```
-
-- The `duration`, which represents how long the miner will keep your file hosted, is represented in blocks. Each block represents 25 seconds.
-
-Upon success, this command will return a **Deal CID**.
-
-The miner will need to **seal** the file before it can be retrieved. If the **Lotus Miner** is not running on a machine designed for sealing, the process will take a very long time.

documentation/en/storing-ipfs-integration.md

@@ -1,20 +0,0 @@
-# IPFS Integration
-
-Lotus supports making deals with data stored in IPFS, without having to re-import it into lotus.
-
-To enable this integration, you need to have an IPFS daemon running in the background.
-Then, open up `~/.lotus/config.toml` (or if you manually set `LOTUS_PATH`, look under that directory) 
-and look for the Client field, and set `UseIpfs` to `true`.
-
-```toml
-[Client]
-UseIpfs = true
-```
-
-After restarting the lotus daemon, you should be able to make deals with data in your IPFS node:
-
-```sh
-$ ipfs add -r SomeData
-QmSomeData
-$ ./lotus client deal QmSomeData t01000 0.0000000001 80000
-```

documentation/en/updating-lotus.md

@@ -1,14 +0,0 @@
-# Updating Lotus
-
-If you installed Lotus on your machine, you can upgrade to the latest version by doing the following:
-
-```sh
-# get the latest
-git pull origin master
-
-# clean and remake the binaries
-make clean && make build
-
-# instal binaries in correct location
-make install # or sudo make install if necessary
-```

extern/sector-storage/ffiwrapper/sealer_cgo.go

@@ -545,10 +545,7 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector abi.SectorID, keepU
 		defer done()
 
 		pf, err := openPartialFile(maxPieceSize, paths.Unsealed)
-		if xerrors.Is(err, os.ErrNotExist) {
+		if err == nil {
-			return xerrors.Errorf("opening partial file: %w", err)
-		}
-
 			var at uint64
 			for sr.HasNext() {
 				r, err := sr.NextRun()
@@ -573,6 +570,12 @@ func (sb *Sealer) FinalizeSector(ctx context.Context, sector abi.SectorID, keepU
 			if err := pf.Close(); err != nil {
 				return err
 			}
+		} else {
+			if !xerrors.Is(err, os.ErrNotExist) {
+				return xerrors.Errorf("opening partial file: %w", err)
+			}
+		}
+
 	}
 
 	paths, done, err := sb.sectors.AcquireSector(ctx, sector, storiface.FTCache, 0, storiface.PathStorage)

extern/sector-storage/worker_local.go

@@ -32,6 +32,7 @@ var pathTypes = []storiface.SectorFileType{storiface.FTUnsealed, storiface.FTSea
 type WorkerConfig struct {
 	SealProof abi.RegisteredSealProof
 	TaskTypes []sealtasks.TaskType
+	NoSwap    bool
 }
 
 type LocalWorker struct {
@@ -41,6 +42,7 @@ type LocalWorker struct {
 	sindex     stores.SectorIndex
 	ret        storiface.WorkerReturn
 	executor   func() (ffiwrapper.Storage, error)
+	noSwap     bool
 
 	ct          *workerCallTracker
 	acceptTasks map[sealtasks.TaskType]struct{}
@@ -69,6 +71,7 @@ func newLocalWorker(executor func() (ffiwrapper.Storage, error), wcfg WorkerConf
 		},
 		acceptTasks: acceptTasks,
 		executor:    executor,
+		noSwap:      wcfg.NoSwap,
 
 		closing: make(chan struct{}),
 	}
@@ -445,11 +448,16 @@ func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error) {
 		return storiface.WorkerInfo{}, xerrors.Errorf("getting memory info: %w", err)
 	}
 
+	memSwap := mem.VirtualTotal
+	if l.noSwap {
+		memSwap = 0
+	}
+
 	return storiface.WorkerInfo{
 		Hostname: hostname,
 		Resources: storiface.WorkerResources{
 			MemPhysical: mem.Total,
-			MemSwap:     mem.VirtualTotal,
+			MemSwap:     memSwap,
 			MemReserved: mem.VirtualUsed + mem.Total - mem.Available, // TODO: sub this process
 			CPUs:        uint64(runtime.NumCPU()),
 			GPUs:        gpus,

extern/storage-sealing/states_failed.go

@@ -38,16 +38,16 @@ func (m *Sealing) checkPreCommitted(ctx statemachine.Context, sector SectorInfo)
 	tok, _, err := m.api.ChainHead(ctx.Context())
 	if err != nil {
 		log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err)
-		return nil, true
+		return nil, false
 	}
 
 	info, err := m.api.StateSectorPreCommitInfo(ctx.Context(), m.maddr, sector.SectorNumber, tok)
 	if err != nil {
 		log.Errorf("handleSealPrecommit1Failed(%d): temp error: %+v", sector.SectorNumber, err)
-		return nil, true
+		return nil, false
 	}
 
-	return info, false
+	return info, true
 }
 
 func (m *Sealing) handleSealPrecommit1Failed(ctx statemachine.Context, sector SectorInfo) error {
@@ -108,7 +108,7 @@ func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorI
 	}
 
 	if pci, is := m.checkPreCommitted(ctx, sector); is && pci != nil {
-		if sector.PreCommitMessage != nil {
+		if sector.PreCommitMessage == nil {
 			log.Warn("sector %d is precommitted on chain, but we don't have precommit message", sector.SectorNumber)
 			return ctx.Send(SectorPreCommitLanded{TipSet: tok})
 		}

extern/test-vectors

@@ -1 +1 @@
-Subproject commit 6bea015edddde116001a4251dce3c4a9966c25d9
+Subproject commit 3a6e0b5e069b1452ce1a032aa315354d645f3ec4

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/buger/goterm v0.0.0-20200322175922-2f3e71b85129
+	github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b
 	github.com/coreos/go-systemd/v22 v22.0.0
 	github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e
 	github.com/dgraph-io/badger/v2 v2.2007.2
@@ -22,7 +23,7 @@ require (
 	github.com/fatih/color v1.8.0
 	github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200716204036-cddc56607e1d
 	github.com/filecoin-project/go-address v0.0.4
-	github.com/filecoin-project/go-bitfield v0.2.0
+	github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
 	github.com/filecoin-project/go-data-transfer v0.6.6
@@ -38,7 +39,7 @@ require (
 	github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b
 	github.com/filecoin-project/specs-actors v0.9.11
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796
-	github.com/filecoin-project/test-vectors/schema v0.0.1
+	github.com/filecoin-project/test-vectors/schema v0.0.3
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
 	github.com/go-kit/kit v0.10.0
 	github.com/go-ole/go-ole v1.2.4 // indirect
@@ -54,10 +55,11 @@ require (
 	github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834
 	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-cidutil v0.0.2
-	github.com/ipfs/go-datastore v0.4.4
+	github.com/ipfs/go-datastore v0.4.5
 	github.com/ipfs/go-ds-badger2 v0.1.1-0.20200708190120-187fc06f714e
 	github.com/ipfs/go-ds-leveldb v0.4.2
 	github.com/ipfs/go-ds-measure v0.1.0
+	github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459
 	github.com/ipfs/go-filestore v1.0.0
 	github.com/ipfs/go-fs-lock v0.0.6
 	github.com/ipfs/go-graphsync v0.2.1
@@ -126,10 +128,13 @@ require (
 	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
+	gopkg.in/cheggaaa/pb.v1 v1.0.28
 	gotest.tools v2.2.0+incompatible
 	launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect
 )
 
+replace github.com/filecoin-project/lotus => ./
+
 replace github.com/golangci/golangci-lint => github.com/golangci/golangci-lint v1.18.0
 
 replace github.com/filecoin-project/filecoin-ffi => ./extern/filecoin-ffi

go.sum

@@ -112,6 +112,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
+github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
@@ -125,6 +127,14 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
+github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs=
+github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
+github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY=
+github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
+github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b h1:OKALTB609+19AM7wsO0k8yMwAqjEIppcnYvyIhA+ZlQ=
+github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ=
+github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3 h1:2+dpIJzYMSbLi0587YXpi8tOJT52qCOI/1I0UNThc/I=
+github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -220,6 +230,8 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 h1:t6qDiuGYYngDqaLc2ZUvdtAg4UN
 github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs=
 github.com/filecoin-project/go-bitfield v0.2.0 h1:gCtLcjskIPtdg4NfN7gQZSQF9yrBQ7mkT0qCJxzGI2Q=
 github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
+github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3 h1:HQa4+yCYsLq1TLM0kopeAhSCLbtZ541cWEi5N5rO+9g=
+github.com/filecoin-project/go-bitfield v0.2.1-0.20200920172649-837cbe6a1ed3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
@@ -260,8 +272,8 @@ github.com/filecoin-project/specs-actors v0.9.11 h1:TnpG7HAeiUrfj0mJM7UaPW0P2137
 github.com/filecoin-project/specs-actors v0.9.11/go.mod h1:czlvLQGEX0fjLLfdNHD7xLymy6L3n7aQzRWzsYGf+ys=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796 h1:dJsTPWpG2pcTeojO2pyn0c6l+x/3MZYCBgo/9d11JEk=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200907031224-ed2e5cd13796/go.mod h1:nJRRM7Aa9XVvygr3W9k6xGF46RWzr2zxF/iGoAIfA/g=
-github.com/filecoin-project/test-vectors/schema v0.0.1 h1:5fNF76nl4qolEvcIsjc0kUADlTMVHO73tW4kXXPnsus=
+github.com/filecoin-project/test-vectors/schema v0.0.3 h1:1zuBo25B3016inbygYLgYFdpJ2m1BDTbAOCgABRleiU=
-github.com/filecoin-project/test-vectors/schema v0.0.1/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
+github.com/filecoin-project/test-vectors/schema v0.0.3/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6 h1:u/UEqS66A5ckRmS4yNpjmVH56sVtS/RfclBAYocb4as=
 github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ=
@@ -274,12 +286,16 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0=
 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1/go.mod h1:0eHX/BVySxPc6SE2mZRoppGq7qcEagxdmQnA3dzork8=
+github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
+github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo=
@@ -341,6 +357,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf h1:gFVkHXmVAhEbxZVDln5V9GKrLaluNoFHDbrZwAWZgws=
+github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -481,6 +499,8 @@ github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13X
 github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8=
 github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-datastore v0.4.5 h1:cwOUcGMLdLPWgu3SlrCckCMznaGADbPqE0r8h768/Dg=
+github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs=
 github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
 github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
 github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
@@ -499,6 +519,8 @@ github.com/ipfs/go-ds-leveldb v0.4.2 h1:QmQoAJ9WkPMUfBLnu1sBVy0xWWlJPg0m4kRAiJL9
 github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s=
 github.com/ipfs/go-ds-measure v0.1.0 h1:vE4TyY4aeLeVgnnPBC5QzKIjKrqzha0NCujTfgvVbVQ=
 github.com/ipfs/go-ds-measure v0.1.0/go.mod h1:1nDiFrhLlwArTME1Ees2XaBOl49OoCgd2A3f8EchMSY=
+github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459 h1:W3YMLEvOXqdW+sYMiguhWP6txJwQvIQqhvpU8yAMGQs=
+github.com/ipfs/go-ds-pebble v0.0.2-0.20200921225637-ce220f8ac459/go.mod h1:oh4liWHulKcDKVhCska5NLelE3MatWl+1FwSz3tY91g=
 github.com/ipfs/go-filestore v1.0.0 h1:QR7ekKH+q2AGiWDc7W2Q0qHuYSRZGUJqUn0GsegEPb0=
 github.com/ipfs/go-filestore v1.0.0/go.mod h1:/XOCuNtIe2f1YPbiXdYvD0BKLA0JR1MgPiFOdcuu9SM=
 github.com/ipfs/go-fs-lock v0.0.6 h1:sn3TWwNVQqSeNjlWy6zQ1uUGAZrV3hPOyEA6y1/N2a0=
@@ -506,6 +528,7 @@ github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28
 github.com/ipfs/go-graphsync v0.1.0/go.mod h1:jMXfqIEDFukLPZHqDPp8tJMbHO9Rmeb9CEGevngQbmE=
 github.com/ipfs/go-graphsync v0.2.1 h1:MdehhqBSuTI2LARfKLkpYnt0mUrqHs/mtuDnESXHBfU=
 github.com/ipfs/go-graphsync v0.2.1/go.mod h1:gEBvJUNelzMkaRPJTpg/jaKN4AQW/7wDWu0K92D8o10=
+github.com/ipfs/go-hamt-ipld v0.1.1 h1:0IQdvwnAAUKmDE+PMJa5y1QiwOPHpI9+eAbQEEEYthk=
 github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk=
 github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
 github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw=
@@ -1495,6 +1518,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROEUQ5ODw47tM3YWjygbs=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc=
+golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -1513,6 +1538,7 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -1614,6 +1640,7 @@ golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

node/builder.go

@@ -35,6 +35,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/gen/slashfilter"
 	"github.com/filecoin-project/lotus/chain/market"
 	"github.com/filecoin-project/lotus/chain/messagepool"
+	"github.com/filecoin-project/lotus/chain/messagesigner"
 	"github.com/filecoin-project/lotus/chain/metrics"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -260,6 +261,7 @@ func Online() Option {
 			Override(new(*store.ChainStore), modules.ChainStore),
 			Override(new(*stmgr.StateManager), stmgr.NewStateManager),
 			Override(new(*wallet.Wallet), wallet.NewWallet),
+			Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner),
 
 			Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker),
 			Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore),

node/impl/full/mpool.go

@@ -10,8 +10,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/messagepool"
+	"github.com/filecoin-project/lotus/chain/messagesigner"
-	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 )
@@ -22,9 +21,7 @@ type MpoolAPI struct {
 	WalletAPI
 	GasAPI
 
-	Chain *store.ChainStore
+	MessageSigner *messagesigner.MessageSigner
-
-	Mpool *messagepool.MessagePool
 
 	PushLocks *dtypes.MpoolLocker
 }
@@ -114,12 +111,14 @@ func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (ci
 }
 
 func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) {
+	cp := *msg
+	msg = &cp
 	inMsg := *msg
-	{
 	fromA, err := a.Stmgr.ResolveToKeyAddress(ctx, msg.From, nil)
 	if err != nil {
 		return nil, xerrors.Errorf("getting key address: %w", err)
 	}
+	{
 		done, err := a.PushLocks.TakeLock(ctx, fromA)
 		if err != nil {
 			return nil, xerrors.Errorf("taking lock: %w", err)
@@ -131,7 +130,7 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
 		return nil, xerrors.Errorf("MpoolPushMessage expects message nonce to be 0, was %d", msg.Nonce)
 	}
 
-	msg, err := a.GasAPI.GasEstimateMessageGas(ctx, msg, spec, types.EmptyTSK)
+	msg, err = a.GasAPI.GasEstimateMessageGas(ctx, msg, spec, types.EmptyTSK)
 	if err != nil {
 		return nil, xerrors.Errorf("GasEstimateMessageGas error: %w", err)
 	}
@@ -143,11 +142,9 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
 			inJson, outJson)
 	}
 
-	sign := func(from address.Address, nonce uint64) (*types.SignedMessage, error) {
-		msg.Nonce = nonce
 	if msg.From.Protocol() == address.ID {
-			log.Warnf("Push from ID address (%s), adjusting to %s", msg.From, from)
+		log.Warnf("Push from ID address (%s), adjusting to %s", msg.From, fromA)
-			msg.From = from
+		msg.From = fromA
 	}
 
 	b, err := a.WalletBalance(ctx, msg.From)
@@ -159,17 +156,17 @@ func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spe
 		return nil, xerrors.Errorf("mpool push: not enough funds: %s < %s", b, msg.Value)
 	}
 
-		return a.WalletSignMessage(ctx, from, msg)
+	smsg, err := a.MessageSigner.SignMessage(ctx, msg)
+	if err != nil {
+		return nil, xerrors.Errorf("mpool push: failed to sign message: %w", err)
 	}
 
-	var m *types.SignedMessage
+	_, err = a.Mpool.Push(smsg)
-again:
+	if err != nil {
-	m, err = a.Mpool.PushWithNonce(ctx, msg.From, sign)
+		return nil, xerrors.Errorf("mpool push: failed to push message: %w", err)
-	if err == messagepool.ErrTryAgain {
-		log.Debugf("temporary failure while pushing message: %s; retrying", err)
-		goto again
 	}
-	return m, err
+
+	return smsg, err
 }
 
 func (a *MpoolAPI) MpoolGetNonce(ctx context.Context, addr address.Address) (uint64, error) {

node/impl/full/sync.go

@@ -126,3 +126,22 @@ func (a *SyncAPI) SyncCheckBad(ctx context.Context, bcid cid.Cid) (string, error
 
 	return reason, nil
 }
+
+func (a *SyncAPI) SyncValidateTipset(ctx context.Context, tsk types.TipSetKey) (bool, error) {
+	ts, err := a.Syncer.ChainStore().LoadTipSet(tsk)
+	if err != nil {
+		return false, err
+	}
+
+	fts, err := a.Syncer.ChainStore().TryFillTipSet(ts)
+	if err != nil {
+		return false, err
+	}
+
+	err = a.Syncer.ValidateTipSet(ctx, fts, false)
+	if err != nil {
+		return false, err
+	}
+
+	return true, nil
+}

node/impl/full/wallet.go

@@ -90,3 +90,7 @@ func (a *WalletAPI) WalletImport(ctx context.Context, ki *types.KeyInfo) (addres
 func (a *WalletAPI) WalletDelete(ctx context.Context, addr address.Address) error {
 	return a.Wallet.DeleteKey(addr)
 }
+
+func (a *WalletAPI) WalletValidateAddress(ctx context.Context, str string) (address.Address, error) {
+	return address.NewFromString(str)
+}

node/repo/fsrepo.go

@@ -66,6 +66,7 @@ var ErrRepoExists = xerrors.New("repo exists")
 // FsRepo is struct for repo, use NewFS to create
 type FsRepo struct {
 	path       string
+	configPath string
 }
 
 var _ Repo = &FsRepo{}
@@ -79,9 +80,14 @@ func NewFS(path string) (*FsRepo, error) {
 
 	return &FsRepo{
 		path:       path,
+		configPath: filepath.Join(path, fsConfig),
 	}, nil
 }
 
+func (fsr *FsRepo) SetConfigPath(cfgPath string) {
+	fsr.configPath = cfgPath
+}
+
 func (fsr *FsRepo) Exists() (bool, error) {
 	_, err := os.Stat(filepath.Join(fsr.path, fsDatastore))
 	notexist := os.IsNotExist(err)
@@ -115,9 +121,7 @@ func (fsr *FsRepo) Init(t RepoType) error {
 }
 
 func (fsr *FsRepo) initConfig(t RepoType) error {
-	cfgP := filepath.Join(fsr.path, fsConfig)
+	_, err := os.Stat(fsr.configPath)
-
-	_, err := os.Stat(cfgP)
 	if err == nil {
 		// exists
 		return nil
@@ -125,7 +129,7 @@ func (fsr *FsRepo) initConfig(t RepoType) error {
 		return err
 	}
 
-	c, err := os.Create(cfgP)
+	c, err := os.Create(fsr.configPath)
 	if err != nil {
 		return err
 	}
@@ -216,6 +220,7 @@ func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) {
 	}
 	return &fsLockedRepo{
 		path:       fsr.path,
+		configPath: fsr.configPath,
 		repoType:   repoType,
 		closer:     closer,
 	}, nil
@@ -223,6 +228,7 @@ func (fsr *FsRepo) Lock(repoType RepoType) (LockedRepo, error) {
 
 type fsLockedRepo struct {
 	path       string
+	configPath string
 	repoType   RepoType
 	closer     io.Closer
 
@@ -277,7 +283,7 @@ func (fsr *fsLockedRepo) Config() (interface{}, error) {
 }
 
 func (fsr *fsLockedRepo) loadConfigFromDisk() (interface{}, error) {
-	return config.FromFile(fsr.join(fsConfig), defConfForType(fsr.repoType))
+	return config.FromFile(fsr.configPath, defConfForType(fsr.repoType))
 }
 
 func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error {
@@ -306,7 +312,7 @@ func (fsr *fsLockedRepo) SetConfig(c func(interface{})) error {
 	}
 
 	// write buffer of TOML bytes to config file
-	err = ioutil.WriteFile(fsr.join(fsConfig), buf.Bytes(), 0644)
+	err = ioutil.WriteFile(fsr.configPath, buf.Bytes(), 0644)
 	if err != nil {
 		return err
 	}

storage/wdpost_changehandler.go

@@ -0,0 +1,537 @@
+package storage
+
+import (
+	"context"
+	"sync"
+
+	"github.com/filecoin-project/go-state-types/abi"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+
+	"github.com/filecoin-project/go-state-types/dline"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+const SubmitConfidence = 4
+
+type CompleteGeneratePoSTCb func(posts []miner.SubmitWindowedPoStParams, err error)
+type CompleteSubmitPoSTCb func(err error)
+
+type changeHandlerAPI interface {
+	StateMinerProvingDeadline(context.Context, address.Address, types.TipSetKey) (*dline.Info, error)
+	startGeneratePoST(ctx context.Context, ts *types.TipSet, deadline *dline.Info, onComplete CompleteGeneratePoSTCb) context.CancelFunc
+	startSubmitPoST(ctx context.Context, ts *types.TipSet, deadline *dline.Info, posts []miner.SubmitWindowedPoStParams, onComplete CompleteSubmitPoSTCb) context.CancelFunc
+	onAbort(ts *types.TipSet, deadline *dline.Info)
+	failPost(err error, ts *types.TipSet, deadline *dline.Info)
+}
+
+type changeHandler struct {
+	api        changeHandlerAPI
+	actor      address.Address
+	proveHdlr  *proveHandler
+	submitHdlr *submitHandler
+}
+
+func newChangeHandler(api changeHandlerAPI, actor address.Address) *changeHandler {
+	posts := newPostsCache()
+	p := newProver(api, posts)
+	s := newSubmitter(api, posts)
+	return &changeHandler{api: api, actor: actor, proveHdlr: p, submitHdlr: s}
+}
+
+func (ch *changeHandler) start() {
+	go ch.proveHdlr.run()
+	go ch.submitHdlr.run()
+}
+
+func (ch *changeHandler) update(ctx context.Context, revert *types.TipSet, advance *types.TipSet) error {
+	// Get the current deadline period
+	di, err := ch.api.StateMinerProvingDeadline(ctx, ch.actor, advance.Key())
+	if err != nil {
+		return err
+	}
+
+	if !di.PeriodStarted() {
+		return nil // not proving anything yet
+	}
+
+	hc := &headChange{
+		ctx:     ctx,
+		revert:  revert,
+		advance: advance,
+		di:      di,
+	}
+
+	select {
+	case ch.proveHdlr.hcs <- hc:
+	case <-ch.proveHdlr.shutdownCtx.Done():
+	case <-ctx.Done():
+	}
+
+	select {
+	case ch.submitHdlr.hcs <- hc:
+	case <-ch.submitHdlr.shutdownCtx.Done():
+	case <-ctx.Done():
+	}
+
+	return nil
+}
+
+func (ch *changeHandler) shutdown() {
+	ch.proveHdlr.shutdown()
+	ch.submitHdlr.shutdown()
+}
+
+func (ch *changeHandler) currentTSDI() (*types.TipSet, *dline.Info) {
+	return ch.submitHdlr.currentTSDI()
+}
+
+// postsCache keeps a cache of PoSTs for each proving window
+type postsCache struct {
+	added chan *postInfo
+	lk    sync.RWMutex
+	cache map[abi.ChainEpoch][]miner.SubmitWindowedPoStParams
+}
+
+func newPostsCache() *postsCache {
+	return &postsCache{
+		added: make(chan *postInfo, 16),
+		cache: make(map[abi.ChainEpoch][]miner.SubmitWindowedPoStParams),
+	}
+}
+
+func (c *postsCache) add(di *dline.Info, posts []miner.SubmitWindowedPoStParams) {
+	c.lk.Lock()
+	defer c.lk.Unlock()
+
+	// TODO: clear cache entries older than chain finality
+	c.cache[di.Open] = posts
+
+	c.added <- &postInfo{
+		di:    di,
+		posts: posts,
+	}
+}
+
+func (c *postsCache) get(di *dline.Info) ([]miner.SubmitWindowedPoStParams, bool) {
+	c.lk.RLock()
+	defer c.lk.RUnlock()
+
+	posts, ok := c.cache[di.Open]
+	return posts, ok
+}
+
+type headChange struct {
+	ctx     context.Context
+	revert  *types.TipSet
+	advance *types.TipSet
+	di      *dline.Info
+}
+
+type currentPost struct {
+	di    *dline.Info
+	abort context.CancelFunc
+}
+
+type postResult struct {
+	ts       *types.TipSet
+	currPost *currentPost
+	posts    []miner.SubmitWindowedPoStParams
+	err      error
+}
+
+// proveHandler generates proofs
+type proveHandler struct {
+	api   changeHandlerAPI
+	posts *postsCache
+
+	postResults chan *postResult
+	hcs         chan *headChange
+
+	current *currentPost
+
+	shutdownCtx context.Context
+	shutdown    context.CancelFunc
+
+	// Used for testing
+	processedHeadChanges chan *headChange
+	processedPostResults chan *postResult
+}
+
+func newProver(
+	api changeHandlerAPI,
+	posts *postsCache,
+) *proveHandler {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &proveHandler{
+		api:         api,
+		posts:       posts,
+		postResults: make(chan *postResult),
+		hcs:         make(chan *headChange),
+		shutdownCtx: ctx,
+		shutdown:    cancel,
+	}
+}
+
+func (p *proveHandler) run() {
+	// Abort proving on shutdown
+	defer func() {
+		if p.current != nil {
+			p.current.abort()
+		}
+	}()
+
+	for p.shutdownCtx.Err() == nil {
+		select {
+		case <-p.shutdownCtx.Done():
+			return
+
+		case hc := <-p.hcs:
+			// Head changed
+			p.processHeadChange(hc.ctx, hc.advance, hc.di)
+			if p.processedHeadChanges != nil {
+				p.processedHeadChanges <- hc
+			}
+
+		case res := <-p.postResults:
+			// Proof generation complete
+			p.processPostResult(res)
+			if p.processedPostResults != nil {
+				p.processedPostResults <- res
+			}
+		}
+	}
+}
+
+func (p *proveHandler) processHeadChange(ctx context.Context, newTS *types.TipSet, di *dline.Info) {
+	// If the post window has expired, abort the current proof
+	if p.current != nil && newTS.Height() >= p.current.di.Close {
+		// Cancel the context on the current proof
+		p.current.abort()
+
+		// Clear out the reference to the proof so that we can immediately
+		// start generating a new proof, without having to worry about state
+		// getting clobbered when the abort completes
+		p.current = nil
+	}
+
+	// Only generate one proof at a time
+	if p.current != nil {
+		return
+	}
+
+	// If the proof for the current post window has been generated, check the
+	// next post window
+	_, complete := p.posts.get(di)
+	for complete {
+		di = nextDeadline(di)
+		_, complete = p.posts.get(di)
+	}
+
+	// Check if the chain is above the Challenge height for the post window
+	if newTS.Height() < di.Challenge {
+		return
+	}
+
+	p.current = &currentPost{di: di}
+	curr := p.current
+	p.current.abort = p.api.startGeneratePoST(ctx, newTS, di, func(posts []miner.SubmitWindowedPoStParams, err error) {
+		p.postResults <- &postResult{ts: newTS, currPost: curr, posts: posts, err: err}
+	})
+}
+
+func (p *proveHandler) processPostResult(res *postResult) {
+	di := res.currPost.di
+	if res.err != nil {
+		// Proving failed so inform the API
+		p.api.failPost(res.err, res.ts, di)
+		log.Warnf("Aborted window post Proving (Deadline: %+v)", di)
+		p.api.onAbort(res.ts, di)
+
+		// Check if the current post has already been aborted
+		if p.current == res.currPost {
+			// If the current post was not already aborted, setting it to nil
+			// marks it as complete so that a new post can be started
+			p.current = nil
+		}
+		return
+	}
+
+	// Completed processing this proving window
+	p.current = nil
+
+	// Add the proofs to the cache
+	p.posts.add(di, res.posts)
+}
+
+type submitResult struct {
+	pw  *postWindow
+	err error
+}
+
+type SubmitState string
+
+const (
+	SubmitStateStart      SubmitState = "SubmitStateStart"
+	SubmitStateSubmitting SubmitState = "SubmitStateSubmitting"
+	SubmitStateComplete   SubmitState = "SubmitStateComplete"
+)
+
+type postWindow struct {
+	ts          *types.TipSet
+	di          *dline.Info
+	submitState SubmitState
+	abort       context.CancelFunc
+}
+
+type postInfo struct {
+	di    *dline.Info
+	posts []miner.SubmitWindowedPoStParams
+}
+
+// submitHandler submits proofs on-chain
+type submitHandler struct {
+	api   changeHandlerAPI
+	posts *postsCache
+
+	submitResults chan *submitResult
+	hcs           chan *headChange
+
+	postWindows       map[abi.ChainEpoch]*postWindow
+	getPostWindowReqs chan *getPWReq
+
+	shutdownCtx context.Context
+	shutdown    context.CancelFunc
+
+	currentCtx context.Context
+	currentTS  *types.TipSet
+	currentDI  *dline.Info
+	getTSDIReq chan chan *tsdi
+
+	// Used for testing
+	processedHeadChanges   chan *headChange
+	processedSubmitResults chan *submitResult
+	processedPostReady     chan *postInfo
+}
+
+func newSubmitter(
+	api changeHandlerAPI,
+	posts *postsCache,
+) *submitHandler {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &submitHandler{
+		api:               api,
+		posts:             posts,
+		submitResults:     make(chan *submitResult),
+		hcs:               make(chan *headChange),
+		postWindows:       make(map[abi.ChainEpoch]*postWindow),
+		getPostWindowReqs: make(chan *getPWReq),
+		getTSDIReq:        make(chan chan *tsdi),
+		shutdownCtx:       ctx,
+		shutdown:          cancel,
+	}
+}
+
+func (s *submitHandler) run() {
+	// On shutdown, abort in-progress submits
+	defer func() {
+		for _, pw := range s.postWindows {
+			if pw.abort != nil {
+				pw.abort()
+			}
+		}
+	}()
+
+	for s.shutdownCtx.Err() == nil {
+		select {
+		case <-s.shutdownCtx.Done():
+			return
+
+		case hc := <-s.hcs:
+			// Head change
+			s.processHeadChange(hc.ctx, hc.revert, hc.advance, hc.di)
+			if s.processedHeadChanges != nil {
+				s.processedHeadChanges <- hc
+			}
+
+		case pi := <-s.posts.added:
+			// Proof generated
+			s.processPostReady(pi)
+			if s.processedPostReady != nil {
+				s.processedPostReady <- pi
+			}
+
+		case res := <-s.submitResults:
+			// Submit complete
+			s.processSubmitResult(res)
+			if s.processedSubmitResults != nil {
+				s.processedSubmitResults <- res
+			}
+
+		case pwreq := <-s.getPostWindowReqs:
+			// used by getPostWindow() to sync with run loop
+			pwreq.out <- s.postWindows[pwreq.di.Open]
+
+		case out := <-s.getTSDIReq:
+			// used by currentTSDI() to sync with run loop
+			out <- &tsdi{ts: s.currentTS, di: s.currentDI}
+		}
+	}
+}
+
+// processHeadChange is called when the chain head changes
+func (s *submitHandler) processHeadChange(ctx context.Context, revert *types.TipSet, advance *types.TipSet, di *dline.Info) {
+	s.currentCtx = ctx
+	s.currentTS = advance
+	s.currentDI = di
+
+	// Start tracking the current post window if we're not already
+	// TODO: clear post windows older than chain finality
+	if _, ok := s.postWindows[di.Open]; !ok {
+		s.postWindows[di.Open] = &postWindow{
+			di:          di,
+			ts:          advance,
+			submitState: SubmitStateStart,
+		}
+	}
+
+	// Apply the change to all post windows
+	for _, pw := range s.postWindows {
+		s.processHeadChangeForPW(ctx, revert, advance, pw)
+	}
+}
+
+func (s *submitHandler) processHeadChangeForPW(ctx context.Context, revert *types.TipSet, advance *types.TipSet, pw *postWindow) {
+	revertedToPrevDL := revert != nil && revert.Height() < pw.di.Open
+	expired := advance.Height() >= pw.di.Close
+
+	// If the chain was reverted back to the previous deadline, or if the post
+	// window has expired, abort submit
+	if pw.submitState == SubmitStateSubmitting && (revertedToPrevDL || expired) {
+		// Replace the aborted postWindow with a new one so that we can
+		// submit again at any time without the state getting clobbered
+		// when the abort completes
+		abort := pw.abort
+		if abort != nil {
+			pw = &postWindow{
+				di:          pw.di,
+				ts:          advance,
+				submitState: SubmitStateStart,
+			}
+			s.postWindows[pw.di.Open] = pw
+
+			// Abort the current submit
+			abort()
+		}
+	} else if pw.submitState == SubmitStateComplete && revertedToPrevDL {
+		// If submit for this deadline has completed, but the chain was
+		// reverted back to the previous deadline, reset the submit state to the
+		// starting state, so that it can be resubmitted
+		pw.submitState = SubmitStateStart
+	}
+
+	// Submit the proof to chain if the proof has been generated and the chain
+	// height is above confidence
+	s.submitIfReady(ctx, advance, pw)
+}
+
+// processPostReady is called when a proof generation completes
+func (s *submitHandler) processPostReady(pi *postInfo) {
+	pw, ok := s.postWindows[pi.di.Open]
+	if ok {
+		s.submitIfReady(s.currentCtx, s.currentTS, pw)
+	}
+}
+
+// submitIfReady submits a proof if the chain is high enough and the proof
+// has been generated for this deadline
+func (s *submitHandler) submitIfReady(ctx context.Context, advance *types.TipSet, pw *postWindow) {
+	// If the window has expired, there's nothing more to do.
+	if advance.Height() >= pw.di.Close {
+		return
+	}
+
+	// Check if we're already submitting, or already completed submit
+	if pw.submitState != SubmitStateStart {
+		return
+	}
+
+	// Check if we've reached the confidence height to submit
+	if advance.Height() < pw.di.Open+SubmitConfidence {
+		return
+	}
+
+	// Check if the proofs have been generated for this deadline
+	posts, ok := s.posts.get(pw.di)
+	if !ok {
+		return
+	}
+
+	// If there was nothing to prove, move straight to the complete state
+	if len(posts) == 0 {
+		pw.submitState = SubmitStateComplete
+		return
+	}
+
+	// Start submitting post
+	pw.submitState = SubmitStateSubmitting
+	pw.abort = s.api.startSubmitPoST(ctx, advance, pw.di, posts, func(err error) {
+		s.submitResults <- &submitResult{pw: pw, err: err}
+	})
+}
+
+// processSubmitResult is called with the response to a submit
+func (s *submitHandler) processSubmitResult(res *submitResult) {
+	if res.err != nil {
+		// Submit failed so inform the API and go back to the start state
+		s.api.failPost(res.err, res.pw.ts, res.pw.di)
+		log.Warnf("Aborted window post Submitting (Deadline: %+v)", res.pw.di)
+		s.api.onAbort(res.pw.ts, res.pw.di)
+
+		res.pw.submitState = SubmitStateStart
+		return
+	}
+
+	// Submit succeeded so move to complete state
+	res.pw.submitState = SubmitStateComplete
+}
+
+type tsdi struct {
+	ts *types.TipSet
+	di *dline.Info
+}
+
+func (s *submitHandler) currentTSDI() (*types.TipSet, *dline.Info) {
+	out := make(chan *tsdi)
+	s.getTSDIReq <- out
+	res := <-out
+	return res.ts, res.di
+}
+
+type getPWReq struct {
+	di  *dline.Info
+	out chan *postWindow
+}
+
+func (s *submitHandler) getPostWindow(di *dline.Info) *postWindow {
+	out := make(chan *postWindow)
+	s.getPostWindowReqs <- &getPWReq{di: di, out: out}
+	return <-out
+}
+
+// nextDeadline gets deadline info for the subsequent deadline
+func nextDeadline(currentDeadline *dline.Info) *dline.Info {
+	periodStart := currentDeadline.PeriodStart
+	newDeadline := currentDeadline.Index + 1
+	if newDeadline == miner.WPoStPeriodDeadlines {
+		newDeadline = 0
+		periodStart = periodStart + miner.WPoStProvingPeriod
+	}
+
+	return NewDeadlineInfo(periodStart, newDeadline, currentDeadline.CurrentEpoch)
+}
+
+func NewDeadlineInfo(periodStart abi.ChainEpoch, deadlineIdx uint64, currEpoch abi.ChainEpoch) *dline.Info {
+	return dline.NewInfo(periodStart, deadlineIdx, currEpoch, miner.WPoStPeriodDeadlines, miner.WPoStProvingPeriod, miner.WPoStChallengeWindow, miner.WPoStChallengeLookback, miner.FaultDeclarationCutoff)
+}

storage/wdpost_nextdl_test.go

@@ -0,0 +1,38 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+)
+
+func TestNextDeadline(t *testing.T) {
+	periodStart := abi.ChainEpoch(0)
+	deadlineIdx := 0
+	currentEpoch := abi.ChainEpoch(10)
+
+	di := NewDeadlineInfo(periodStart, uint64(deadlineIdx), currentEpoch)
+	require.EqualValues(t, 0, di.Index)
+	require.EqualValues(t, 0, di.PeriodStart)
+	require.EqualValues(t, -20, di.Challenge)
+	require.EqualValues(t, 0, di.Open)
+	require.EqualValues(t, 60, di.Close)
+
+	for i := 1; i < 1+int(miner.WPoStPeriodDeadlines)*2; i++ {
+		di = nextDeadline(di)
+		deadlineIdx = i % int(miner.WPoStPeriodDeadlines)
+		expPeriodStart := int(miner.WPoStProvingPeriod) * (i / int(miner.WPoStPeriodDeadlines))
+		expOpen := expPeriodStart + deadlineIdx*int(miner.WPoStChallengeWindow)
+		expClose := expOpen + int(miner.WPoStChallengeWindow)
+		expChallenge := expOpen - int(miner.WPoStChallengeLookback)
+		//fmt.Printf("%d: %d@%d %d-%d (%d)\n", i, expPeriodStart, deadlineIdx, expOpen, expClose, expChallenge)
+		require.EqualValues(t, deadlineIdx, di.Index)
+		require.EqualValues(t, expPeriodStart, di.PeriodStart)
+		require.EqualValues(t, expOpen, di.Open)
+		require.EqualValues(t, expClose, di.Close)
+		require.EqualValues(t, expChallenge, di.Challenge)
+	}
+}

storage/wdpost_run.go

@@ -29,15 +29,21 @@ import (
 	"github.com/filecoin-project/lotus/journal"
 )
 
-func (s *WindowPoStScheduler) failPost(err error, deadline *dline.Info) {
+func (s *WindowPoStScheduler) failPost(err error, ts *types.TipSet, deadline *dline.Info) {
 	journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
+		c := evtCommon{Error: err}
+		if ts != nil {
+			c.Deadline = deadline
+			c.Height = ts.Height()
+			c.TipSet = ts.Cids()
+		}
 		return WdPoStSchedulerEvt{
-			evtCommon: s.getEvtCommon(err),
+			evtCommon: c,
 			State:     SchedulerStateFaulted,
 		}
 	})
 
-	log.Errorf("TODO")
+	log.Errorf("Got err %w - TODO handle errors", err)
 	/*s.failLk.Lock()
 	if eps > s.failed {
 		s.failed = eps
@@ -45,11 +51,28 @@ func (s *WindowPoStScheduler) failPost(err error, deadline *dline.Info) {
 	s.failLk.Unlock()*/
 }
 
-func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info, ts *types.TipSet) {
+// recordProofsEvent records a successful proofs_processed event in the
-	ctx, abort := context.WithCancel(ctx)
+// journal, even if it was a noop (no partitions).
+func (s *WindowPoStScheduler) recordProofsEvent(partitions []miner.PoStPartition, mcid cid.Cid) {
+	journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} {
+		return &WdPoStProofsProcessedEvt{
+			evtCommon:  s.getEvtCommon(nil),
+			Partitions: partitions,
+			MessageCID: mcid,
+		}
+	})
+}
 
-	s.abort = abort
+// startGeneratePoST kicks off the process of generating a PoST
-	s.activeDeadline = deadline
+func (s *WindowPoStScheduler) startGeneratePoST(
+	ctx context.Context,
+	ts *types.TipSet,
+	deadline *dline.Info,
+	completeGeneratePoST CompleteGeneratePoSTCb,
+) context.CancelFunc {
+	ctx, abort := context.WithCancel(ctx)
+	go func() {
+		defer abort()
 
 		journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
 			return WdPoStSchedulerEvt{
@@ -58,54 +81,104 @@ func (s *WindowPoStScheduler) doPost(ctx context.Context, deadline *dline.Info,
 			}
 		})
 
-	go func() {
+		posts, err := s.runGeneratePoST(ctx, ts, deadline)
-		defer abort()
+		completeGeneratePoST(posts, err)
+	}()
 
-		ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.doPost")
+	return abort
+}
+
+// runGeneratePoST generates the PoST
+func (s *WindowPoStScheduler) runGeneratePoST(
+	ctx context.Context,
+	ts *types.TipSet,
+	deadline *dline.Info,
+) ([]miner.SubmitWindowedPoStParams, error) {
+	ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.generatePoST")
 	defer span.End()
 
-		// recordProofsEvent records a successful proofs_processed event in the
-		// journal, even if it was a noop (no partitions).
-		recordProofsEvent := func(partitions []miner.PoStPartition, mcid cid.Cid) {
-			journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStProofs], func() interface{} {
-				return &WdPoStProofsProcessedEvt{
-					evtCommon:  s.getEvtCommon(nil),
-					Partitions: partitions,
-					MessageCID: mcid,
-				}
-			})
-		}
-
 	posts, err := s.runPost(ctx, *deadline, ts)
 	if err != nil {
-			log.Errorf("run window post failed: %+v", err)
+		log.Errorf("runPost failed: %+v", err)
-			s.failPost(err, deadline)
+		return nil, err
-			return
 	}
 
 	if len(posts) == 0 {
-			recordProofsEvent(nil, cid.Undef)
+		s.recordProofsEvent(nil, cid.Undef)
-			return
 	}
 
-		for i := range posts {
+	return posts, nil
-			post := &posts[i]
+}
-			sm, err := s.submitPost(ctx, post)
-			if err != nil {
-				log.Errorf("submit window post failed: %+v", err)
-				s.failPost(err, deadline)
-			} else {
-				recordProofsEvent(post.Partitions, sm.Cid())
-			}
-		}
 
+// startSubmitPoST kicks of the process of submitting PoST
+func (s *WindowPoStScheduler) startSubmitPoST(
+	ctx context.Context,
+	ts *types.TipSet,
+	deadline *dline.Info,
+	posts []miner.SubmitWindowedPoStParams,
+	completeSubmitPoST CompleteSubmitPoSTCb,
+) context.CancelFunc {
+
+	ctx, abort := context.WithCancel(ctx)
+	go func() {
+		defer abort()
+
+		err := s.runSubmitPoST(ctx, ts, deadline, posts)
+		if err == nil {
 			journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
 				return WdPoStSchedulerEvt{
 					evtCommon: s.getEvtCommon(nil),
 					State:     SchedulerStateSucceeded,
 				}
 			})
+		}
+		completeSubmitPoST(err)
 	}()
+
+	return abort
+}
+
+// runSubmitPoST submits PoST
+func (s *WindowPoStScheduler) runSubmitPoST(
+	ctx context.Context,
+	ts *types.TipSet,
+	deadline *dline.Info,
+	posts []miner.SubmitWindowedPoStParams,
+) error {
+	if len(posts) == 0 {
+		return nil
+	}
+
+	ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.submitPoST")
+	defer span.End()
+
+	// Get randomness from tickets
+	commEpoch := deadline.Open
+	commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil)
+	if err != nil {
+		err = xerrors.Errorf("failed to get chain randomness from tickets for windowPost (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err)
+		log.Errorf("submitPost failed: %+v", err)
+
+		return err
+	}
+
+	var submitErr error
+	for i := range posts {
+		// Add randomness to PoST
+		post := &posts[i]
+		post.ChainCommitEpoch = commEpoch
+		post.ChainCommitRand = commRand
+
+		// Submit PoST
+		sm, submitErr := s.submitPost(ctx, post)
+		if submitErr != nil {
+			log.Errorf("submit window post failed: %+v", submitErr)
+		} else {
+			s.recordProofsEvent(post.Partitions, sm.Cid())
+		}
+	}
+
+	return submitErr
 }
 
 func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.BitField) (bitfield.BitField, error) {
@@ -396,7 +469,7 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty
 
 	rand, err := s.api.ChainGetRandomnessFromBeacon(ctx, ts.Key(), crypto.DomainSeparationTag_WindowedPoStChallengeSeed, di.Challenge, buf.Bytes())
 	if err != nil {
-		return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err)
+		return nil, xerrors.Errorf("failed to get chain randomness from beacon for window post (ts=%d; deadline=%d): %w", ts.Height(), di, err)
 	}
 
 	// Get the partitions for the given deadline
@@ -540,19 +613,6 @@ func (s *WindowPoStScheduler) runPost(ctx context.Context, di dline.Info, ts *ty
 		posts = append(posts, params)
 	}
 
-	// Compute randomness after generating proofs so as to reduce the impact
-	// of chain reorgs (which change randomness)
-	commEpoch := di.Open
-	commRand, err := s.api.ChainGetRandomnessFromTickets(ctx, ts.Key(), crypto.DomainSeparationTag_PoStChainCommit, commEpoch, nil)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to get chain randomness for window post (ts=%d; deadline=%d): %w", ts.Height(), commEpoch, err)
-	}
-
-	for i := range posts {
-		posts[i].ChainCommitEpoch = commEpoch
-		posts[i].ChainCommitRand = commRand
-	}
-
 	return posts, nil
 }
 
@@ -593,6 +653,7 @@ func (s *WindowPoStScheduler) batchPartitions(partitions []api.Partition) ([][]a
 		}
 		batches = append(batches, partitions[i:end])
 	}
+
 	return batches, nil
 }
 

storage/wdpost_run_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-bitfield"
+
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
@@ -177,7 +178,10 @@ func TestWDPostDoPost(t *testing.T) {
 		FaultDeclarationCutoff: miner0.FaultDeclarationCutoff,
 	}
 	ts := mockTipSet(t)
-	scheduler.doPost(ctx, di, ts)
+
+	scheduler.startGeneratePoST(ctx, ts, di, func(posts []miner.SubmitWindowedPoStParams, err error) {
+		scheduler.startSubmitPoST(ctx, ts, di, posts, func(err error) {})
+	})
 
 	// Read the window PoST messages
 	for i := 0; i < expectedMsgCount; i++ {

storage/wdpost_sched.go

@@ -22,8 +22,6 @@ import (
 	"go.opencensus.io/trace"
 )
 
-const StartConfidence = 4 // TODO: config
-
 type WindowPoStScheduler struct {
 	api              storageMinerApi
 	feeCfg           config.MinerFeeConfig
@@ -31,16 +29,11 @@ type WindowPoStScheduler struct {
 	faultTracker     sectorstorage.FaultTracker
 	proofType        abi.RegisteredPoStProof
 	partitionSectors uint64
+	ch               *changeHandler
 
 	actor  address.Address
 	worker address.Address
 
-	cur *types.TipSet
-
-	// if a post is in progress, this indicates for which ElectionPeriodStart
-	activeDeadline *dline.Info
-	abort          context.CancelFunc
-
 	evtTypes [4]journal.EventType
 
 	// failed abi.ChainEpoch // eps
@@ -77,16 +70,17 @@ func NewWindowedPoStScheduler(api storageMinerApi, fc config.MinerFeeConfig, sb
 	}, nil
 }
 
-func deadlineEquals(a, b *dline.Info) bool {
+type changeHandlerAPIImpl struct {
-	if a == nil || b == nil {
+	storageMinerApi
-		return b == a
+	*WindowPoStScheduler
-	}
-
-	return a.PeriodStart == b.PeriodStart && a.Index == b.Index && a.Challenge == b.Challenge
 }
 
 func (s *WindowPoStScheduler) Run(ctx context.Context) {
-	defer s.abortActivePoSt()
+	// Initialize change handler
+	chImpl := &changeHandlerAPIImpl{storageMinerApi: s.api, WindowPoStScheduler: s}
+	s.ch = newChangeHandler(chImpl, s.actor)
+	defer s.ch.shutdown()
+	s.ch.start()
 
 	var notifs <-chan []*api.HeadChange
 	var err error
@@ -125,17 +119,18 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) {
 					continue
 				}
 
-				if err := s.update(ctx, chg.Val); err != nil {
+				ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.headChange")
-					log.Errorf("%+v", err)
-				}
 
+				s.update(ctx, nil, chg.Val)
+
+				span.End()
 				gotCur = true
 				continue
 			}
 
 			ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.headChange")
 
-			var lowest, highest *types.TipSet = s.cur, nil
+			var lowest, highest *types.TipSet = nil, nil
 
 			for _, change := range changes {
 				if change.Val == nil {
@@ -149,12 +144,7 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) {
 				}
 			}
 
-			if err := s.revert(ctx, lowest); err != nil {
+			s.update(ctx, lowest, highest)
-				log.Error("handling head reverts in window post sched: %+v", err)
-			}
-			if err := s.update(ctx, highest); err != nil {
-				log.Error("handling head updates in window post sched: %+v", err)
-			}
 
 			span.End()
 		case <-ctx.Done():
@@ -163,95 +153,40 @@ func (s *WindowPoStScheduler) Run(ctx context.Context) {
 	}
 }
 
-func (s *WindowPoStScheduler) revert(ctx context.Context, newLowest *types.TipSet) error {
+func (s *WindowPoStScheduler) update(ctx context.Context, revert, apply *types.TipSet) {
-	if s.cur == newLowest {
+	if apply == nil {
-		return nil
+		log.Error("no new tipset in window post WindowPoStScheduler.update")
+		return
 	}
-	s.cur = newLowest
+	err := s.ch.update(ctx, revert, apply)
-
-	newDeadline, err := s.api.StateMinerProvingDeadline(ctx, s.actor, newLowest.Key())
 	if err != nil {
-		return err
+		log.Errorf("handling head updates in window post sched: %+v", err)
 	}
-
-	if !deadlineEquals(s.activeDeadline, newDeadline) {
-		s.abortActivePoSt()
-	}
-
-	return nil
 }
 
-func (s *WindowPoStScheduler) update(ctx context.Context, new *types.TipSet) error {
+// onAbort is called when generating proofs or submitting proofs is aborted
-	if new == nil {
+func (s *WindowPoStScheduler) onAbort(ts *types.TipSet, deadline *dline.Info) {
-		return xerrors.Errorf("no new tipset in window post sched update")
-	}
-
-	di, err := s.api.StateMinerProvingDeadline(ctx, s.actor, new.Key())
-	if err != nil {
-		return err
-	}
-
-	if deadlineEquals(s.activeDeadline, di) {
-		return nil // already working on this deadline
-	}
-
-	if !di.PeriodStarted() {
-		return nil // not proving anything yet
-	}
-
-	s.abortActivePoSt()
-
-	// TODO: wait for di.Challenge here, will give us ~10min more to compute windowpost
-	//  (Need to get correct deadline above, which is tricky)
-
-	if di.Open+StartConfidence >= new.Height() {
-		log.Info("not starting window post yet, waiting for startconfidence", di.Open, di.Open+StartConfidence, new.Height())
-		return nil
-	}
-
-	/*s.failLk.Lock()
-	if s.failed > 0 {
-		s.failed = 0
-		s.activeEPS = 0
-	}
-	s.failLk.Unlock()*/
-	log.Infof("at %d, do window post for P %d, dd %d", new.Height(), di.PeriodStart, di.Index)
-
-	s.doPost(ctx, di, new)
-
-	return nil
-}
-
-func (s *WindowPoStScheduler) abortActivePoSt() {
-	if s.activeDeadline == nil {
-		return // noop
-	}
-
-	if s.abort != nil {
-		s.abort()
-
 	journal.J.RecordEvent(s.evtTypes[evtTypeWdPoStScheduler], func() interface{} {
+		c := evtCommon{}
+		if ts != nil {
+			c.Deadline = deadline
+			c.Height = ts.Height()
+			c.TipSet = ts.Cids()
+		}
 		return WdPoStSchedulerEvt{
-				evtCommon: s.getEvtCommon(nil),
+			evtCommon: c,
 			State:     SchedulerStateAborted,
 		}
 	})
-
-		log.Warnf("Aborting window post (Deadline: %+v)", s.activeDeadline)
-	}
-
-	s.activeDeadline = nil
-	s.abort = nil
 }
 
-// getEvtCommon populates and returns common attributes from state, for a
-// WdPoSt journal event.
 func (s *WindowPoStScheduler) getEvtCommon(err error) evtCommon {
 	c := evtCommon{Error: err}
-	if s.cur != nil {
+	currentTS, currentDeadline := s.ch.currentTSDI()
-		c.Deadline = s.activeDeadline
+	if currentTS != nil {
-		c.Height = s.cur.Height()
+		c.Deadline = currentDeadline
-		c.TipSet = s.cur.Cids()
+		c.Height = currentTS.Height()
+		c.TipSet = currentTS.Cids()
 	}
 	return c
 }

tools/dockers/docker-examples/README.md

@@ -11,7 +11,7 @@ In this `docker-examples/` directory are community-contributed Docker and Docker
    - local node for a developer (`api-local-`)
    - hosted endpoint for apps / multiple developers (`api-hosted-`)
 - **For a local devnet or shared devnet**
-   - basic local devnet (also see [lotus docs on setting up a local devnet](https://lotu.sh/en+setup-local-dev-net))
+   - basic local devnet (also see [lotus docs on setting up a local devnet](https://docs.filecoin.io/build/local-devnet/))
    - shared devnet