Merge branch 'master' into feat/chainwatch-pg

.circleci/config.yml

@@ -68,7 +68,7 @@ jobs:
       - run: sudo apt-get install npm
       - restore_cache:
           name: restore go mod cache
-          key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/lotus/go.mod" }}
+          key: v1-go-deps-{{ arch }}-{{ checksum "/home/circleci/project/go.mod" }}
       - run:
           command: make buildall
       - store_artifacts:

CHANGELOG.md

@@ -0,0 +1,7 @@
+# lotus changelog
+
+## 0.1.0 / 2019-12-11
+
+We are very excited to release **lotus** 0.1.0. This is our testnet release. To install lotus and join the testnet, please visit [docs.lotu.sh](docs.lotu.sh). Please file bug reports as [issues](https://github.com/filecoin-project/lotus/issues).
+
+A huge thank you to all contributors for this testnet release!

Makefile

@@ -1,3 +1,5 @@
+SHELL=/usr/bin/env bash
+
 all: build
 .PHONY: all
 
@@ -77,6 +79,9 @@ lotus-shed: $(BUILD_DEPS)
 BINS+=lotus-seal-worker
 
 build: lotus lotus-storage-miner lotus-seal-worker
+	@[[ $$(type -P "lotus") ]] && echo "Caution: you have \
+an existing lotus binary in your PATH. This may cause problems if you don't run 'sudo make install'" || true
+
 .PHONY: build
 
 install:

build/bootstrap/bootstrappers.pi

@@ -1,2 +1,6 @@
-/ip4/147.75.80.29/tcp/1347/p2p/12D3KooWDzb12XyoKT4uJAqmRVsSYqY9EZczXeWJ7WqehugBTVAT
-/ip4/147.75.80.17/tcp/1347/p2p/12D3KooWDsfpmaYPouFT2RxvSf8eCuUS63T4dAKvDPqzWKdv7Qc7
+/dns4/lotus-bootstrap-0.dfw.fil-test.net/tcp/1347/p2p/12D3KooWHwGBSiLR5ts7KW9MgH4BMzC2iXe18kwAQ8Ee3LUd1jeR
+/dns4/lotus-bootstrap-1.dfw.fil-test.net/tcp/1347/p2p/12D3KooWCLFaawdhLGcSpiqg43DtZ9QzPQ6HcB8Vvyu2Cnta8UWc
+/dns4/lotus-bootstrap-0.fra.fil-test.net/tcp/1347/p2p/12D3KooWMmaL7eaUCF6tVAghVmgozxz4uztbuFUQv6dyFpHRarHR
+/dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWLLpNYoKdf9NgcWudBhXLdTcXncqAsTzozw1scMMu6nS5
+/dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWCNL9vXaXwNs3Bu8uRAJK4pxpCyPeM7jZLSDpJma1wrV8
+/dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWNGGxFda1eC5U2YKAgs4ypoFHn3Z3xHCsjmFdrCcytoxm

build/params_shared.go

@@ -83,10 +83,6 @@ var InitialReward *big.Int
 
 const FilecoinPrecision = 1_000_000_000_000_000_000
 
-// six years
-// Epochs
-const HalvingPeriodEpochs = 6 * 365 * 24 * 60 * 2
-
 // TODO: Move other important consts here
 
 func init() {

build/params_testnet.go

@@ -31,4 +31,4 @@ const InteractivePoRepDelay = 8
 const InteractivePoRepConfidence = 6
 
 // Bytes
-var MinimumMinerPower uint64 = 20 << 30 // 20GB
+var MinimumMinerPower uint64 = 512 << 30 // 512GB

build/version.go

@@ -1,7 +1,7 @@
 package build
 
 // Version is the local build version, set by build system
-const Version = "0.11.0"
+const Version = "0.1.0"
 
 // APIVersion is a hex semver version of the rpc api exposed
 //
@@ -12,15 +12,19 @@ const Version = "0.11.0"
 //                   R R  H
 //                   |\vv/|
 //                   vv  vv
-const APIVersion = 0x000b01
+const APIVersion = 0x000100
 
 const (
 	MajorMask = 0xff0000
 	MinorMask = 0xffff00
 	PatchMask = 0xffffff
+
+	MajorOnlyMask = 0xff0000
+	MinorOnlyMask = 0x00ff00
+	PatchOnlyMask = 0x0000ff
 )
 
 // VersionInts returns (major, minor, patch) versions
 func VersionInts(version uint32) (uint32, uint32, uint32) {
-	return (version & MajorMask) >> 16, (version & MinorMask) >> 8, version & PatchMask
+	return (version & MajorOnlyMask) >> 16, (version & MinorOnlyMask) >> 8, version & PatchOnlyMask
 }

chain/store/store.go

@@ -155,6 +155,8 @@ func (cs *ChainStore) SubHeadChanges(ctx context.Context) chan []*HeadChange {
 
 	go func() {
 		defer close(out)
+		var unsubOnce sync.Once
+
 		for {
 			select {
 			case val, ok := <-subch:
@@ -170,7 +172,9 @@ func (cs *ChainStore) SubHeadChanges(ctx context.Context) chan []*HeadChange {
 				case <-ctx.Done():
 				}
 			case <-ctx.Done():
-				go cs.bestTips.Unsub(subch)
+				unsubOnce.Do(func() {
+					go cs.bestTips.Unsub(subch)
+				})
 			}
 		}
 	}()

chain/sub/incoming.go

@@ -33,6 +33,9 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
 		}
 
 		go func() {
+			log.Infof("New block over pubsub: %s", blk.Cid())
+
+			start := time.Now()
 			log.Debug("about to fetch messages for block from pubsub")
 			bmsgs, err := s.Bsync.FetchMessagesByCids(context.TODO(), blk.BlsMessages)
 			if err != nil {
@@ -46,7 +49,8 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
 				return
 			}
 
-			log.Debugw("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom())
+			took := time.Since(start)
+			log.Infow("new block over pubsub", "cid", blk.Header.Cid(), "source", msg.GetFrom(), "msgfetch", took)
 			if delay := time.Now().Unix() - int64(blk.Header.Timestamp); delay > 5 {
 				log.Warnf("Received block with large delay %d from miner %s", delay, blk.Header.Miner)
 			}

chain/sync_manager.go

@@ -203,9 +203,6 @@ func (stb *syncTargetBucket) sameChainAs(ts *types.TipSet) bool {
 		if types.CidArrsEqual(ts.Parents(), t.Cids()) {
 			return true
 		}
-		if types.CidArrsEqual(ts.Parents(), t.Parents()) {
-			return true
-		}
 	}
 	return false
 }
@@ -283,6 +280,7 @@ func (sm *SyncManager) syncScheduler() {
 }
 
 func (sm *SyncManager) scheduleIncoming(ts *types.TipSet) {
+	log.Info("scheduling incoming tipset sync: ", ts.Cids())
 	if sm.getBootstrapState() == BSStateSelected {
 		sm.setBootstrapState(BSStateScheduled)
 		sm.syncTargets <- ts
@@ -295,7 +293,7 @@ func (sm *SyncManager) scheduleIncoming(ts *types.TipSet) {
 			break
 		}
 
-		if types.CidArrsEqual(ts.Parents(), acts.Cids()) || types.CidArrsEqual(ts.Parents(), acts.Parents()) {
+		if types.CidArrsEqual(ts.Parents(), acts.Cids()) {
 			// sync this next, after that sync process finishes
 			relatedToActiveSync = true
 		}

chain/sync_manager_test.go

@@ -74,6 +74,7 @@ func TestSyncManager(t *testing.T) {
 	b := mock.TipSet(mock.MkBlock(a, 1, 2))
 	c1 := mock.TipSet(mock.MkBlock(b, 1, 3))
 	c2 := mock.TipSet(mock.MkBlock(b, 2, 4))
+	c3 := mock.TipSet(mock.MkBlock(b, 3, 5))
 	d := mock.TipSet(mock.MkBlock(c1, 4, 5))
 
 	runSyncMgrTest(t, "testBootstrap", 1, func(t *testing.T, sm *SyncManager, stc chan *syncOp) {
@@ -120,4 +121,30 @@ func TestSyncManager(t *testing.T) {
 
 		assertGetSyncOp(t, stc, d)
 	})
+
+	runSyncMgrTest(t, "testSyncIncomingTipset", 1, func(t *testing.T, sm *SyncManager, stc chan *syncOp) {
+		sm.SetPeerHead(ctx, "peer1", a)
+		assertGetSyncOp(t, stc, a)
+
+		sm.SetPeerHead(ctx, "peer2", b)
+		op := <-stc
+		op.done()
+
+		sm.SetPeerHead(ctx, "peer2", c1)
+		op1 := <-stc
+		fmt.Println("op1: ", op1.ts.Cids())
+
+		sm.SetPeerHead(ctx, "peer2", c2)
+		sm.SetPeerHead(ctx, "peer2", c3)
+
+		op1.done()
+
+		op2 := <-stc
+		fmt.Println("op2: ", op2.ts.Cids())
+		op2.done()
+
+		op3 := <-stc
+		fmt.Println("op3: ", op3.ts.Cids())
+		op3.done()
+	})
 }

chain/vm/vm_test.go

@@ -9,10 +9,12 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
+const HalvingPeriodEpochs = 6 * 365 * 24 * 60 * 2
+
 func TestBlockReward(t *testing.T) {
 	coffer := types.FromFil(build.MiningRewardTotal).Int
 	sum := new(big.Int)
-	N := build.HalvingPeriodEpochs
+	N := HalvingPeriodEpochs
 	for i := 0; i < N; i++ {
 		a := MiningReward(types.BigInt{coffer})
 		sum = sum.Add(sum, a.Int)

cli/client.go

@@ -236,7 +236,7 @@ var clientRetrieveCmd = &cli.Command{
 		}
 
 		if err := api.ClientRetrieve(ctx, offers[0].Order(payer), cctx.Args().Get(1)); err != nil {
-			return err
+			return xerrors.Errorf("Retrieval Failed: %w", err)
 		}
 
 		fmt.Println("Success")

cli/net.go

@@ -2,6 +2,8 @@ package cli
 
 import (
 	"fmt"
+	"sort"
+	"strings"
 
 	"gopkg.in/urfave/cli.v2"
 
@@ -34,6 +36,10 @@ var netPeers = &cli.Command{
 			return err
 		}
 
+		sort.Slice(peers, func(i, j int) bool {
+			return strings.Compare(string(peers[i].ID), string(peers[j].ID)) > 0
+		})
+
 		for _, peer := range peers {
 			fmt.Println(peer)
 		}

cli/params.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"github.com/docker/go-units"
 	"github.com/filecoin-project/lotus/build"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
@@ -10,13 +11,18 @@ var fetchParamCmd = &cli.Command{
 	Name:  "fetch-params",
 	Usage: "Fetch proving parameters",
 	Flags: []cli.Flag{
-		&cli.Uint64Flag{
+		&cli.StringFlag{
 			Name:  "proving-params",
-			Usage: "download params used creating proofs for given size",
+			Usage: "download params used creating proofs for given size, i.e. 32GiB",
 		},
 	},
 	Action: func(cctx *cli.Context) error {
-		err := build.GetParams(cctx.Uint64("proving-params"))
+		sectorSizeInt, err := units.FromHumanSize(cctx.String("proving-params"))
+		if err != nil {
+			return err
+		}
+		sectorSize := uint64(sectorSizeInt)
+		err = build.GetParams(sectorSize)
 		if err != nil {
 			return xerrors.Errorf("fetching proof parameters: %w", err)
 		}

cli/version.go

@@ -19,7 +19,11 @@ var versionCmd = &cli.Command{
 		ctx := ReqContext(cctx)
 		// TODO: print more useful things
 
-		fmt.Println(api.Version(ctx))
+		v, err := api.Version(ctx)
+		if err != nil {
+			return err
+		}
+		fmt.Println(v)
 		cli.VersionPrinter(cctx)
 		return nil
 	},

cmd/lotus-bench/main.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
+	"github.com/docker/go-units"
 	"io/ioutil"
 	"math/big"
 	"math/rand"
@@ -65,9 +66,10 @@ func main() {
 				Value: "~/.lotus-bench",
 				Usage: "Path to the storage directory that will store sectors long term",
 			},
-			&cli.Uint64Flag{
+			&cli.StringFlag{
 				Name:  "sector-size",
-				Value: 1024,
+				Value: "1GiB",
+				Usage: "size of the sectors in bytes, i.e. 32GiB",
 			},
 			&cli.BoolFlag{
 				Name:  "no-gpu",
@@ -127,7 +129,11 @@ func main() {
 				return err
 			}
 
-			sectorSize := c.Uint64("sector-size")
+			sectorSizeInt, err := units.FromHumanSize(c.String("sector-size"))
+			if err != nil {
+				return err
+			}
+			sectorSize := uint64(sectorSizeInt)
 
 			mds := datastore.NewMapDatastore()
 			cfg := &sectorbuilder.Config{

cmd/lotus-fountain/main.go

@@ -100,7 +100,7 @@ var runCmd = &cli.Command{
 				IPRate:      time.Minute,
 				IPBurst:     5,
 				WalletRate:  15 * time.Minute,
-				WalletBurst: 1,
+				WalletBurst: 2,
 			}),
 			minerLimiter: NewLimiter(LimiterConfig{
 				TotalRate:   time.Second,
@@ -108,7 +108,7 @@ var runCmd = &cli.Command{
 				IPRate:      10 * time.Minute,
 				IPBurst:     2,
 				WalletRate:  1 * time.Hour,
-				WalletBurst: 1,
+				WalletBurst: 2,
 			}),
 		}
 
@@ -150,7 +150,7 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) {
 	// Limit based on wallet address
 	limiter := h.limiter.GetWalletLimiter(to.String())
 	if !limiter.Allow() {
-		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		http.Error(w, http.StatusText(http.StatusTooManyRequests)+": wallet limit", http.StatusTooManyRequests)
 		return
 	}
 
@@ -170,13 +170,13 @@ func (h *handler) send(w http.ResponseWriter, r *http.Request) {
 
 	limiter = h.limiter.GetIPLimiter(reqIP)
 	if !limiter.Allow() {
-		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		http.Error(w, http.StatusText(http.StatusTooManyRequests)+": IP limit", http.StatusTooManyRequests)
 		return
 	}
 
 	// General limiter to allow throttling all messages that can make it into the mpool
 	if !h.limiter.Allow() {
-		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		http.Error(w, http.StatusText(http.StatusTooManyRequests)+": global limit", http.StatusTooManyRequests)
 		return
 	}
 
@@ -221,20 +221,20 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) {
 	// Limit based on wallet address
 	limiter := h.minerLimiter.GetWalletLimiter(owner.String())
 	if !limiter.Allow() {
-		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		http.Error(w, http.StatusText(http.StatusTooManyRequests)+": wallet limit", http.StatusTooManyRequests)
 		return
 	}
 
 	// Limit based on IP
 	limiter = h.minerLimiter.GetIPLimiter(r.RemoteAddr)
 	if !limiter.Allow() {
-		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		http.Error(w, http.StatusText(http.StatusTooManyRequests)+": IP limit", http.StatusTooManyRequests)
 		return
 	}
 
 	// General limiter owner allow throttling all messages that can make it into the mpool
 	if !h.minerLimiter.Allow() {
-		http.Error(w, http.StatusText(http.StatusTooManyRequests), http.StatusTooManyRequests)
+		http.Error(w, http.StatusText(http.StatusTooManyRequests)+": global limit", http.StatusTooManyRequests)
 		return
 	}
 

cmd/lotus/daemon.go

@@ -47,6 +47,11 @@ var DaemonCmd = &cli.Command{
 			Name:  "genesis",
 			Usage: "genesis file to use for first node run",
 		},
+		&cli.StringFlag{
+			Name:   "genesis-timestamp",
+			Hidden: true,
+			Usage:  "set the timestamp for the genesis block that will be created",
+		},
 		&cli.BoolFlag{
 			Name:  "bootstrap",
 			Value: true,
@@ -84,7 +89,7 @@ var DaemonCmd = &cli.Command{
 			if cctx.String(preSealedSectorsFlag) == "" {
 				return xerrors.Errorf("must also pass file with miner preseal info to `--%s`", preSealedSectorsFlag)
 			}
-			genesis = node.Override(new(modules.Genesis), testing.MakeGenesis(cctx.String(makeGenFlag), cctx.String(preSealedSectorsFlag)))
+			genesis = node.Override(new(modules.Genesis), testing.MakeGenesis(cctx.String(makeGenFlag), cctx.String(preSealedSectorsFlag), cctx.String("genesis-timestamp")))
 		}
 
 		var api api.FullNode

documentation/cn/api-scripting-support.md

@@ -0,0 +1,3 @@
+# API Scripting Support
+
+工作正在进行中

documentation/cn/join-testnet.md

@@ -1,3 +1,3 @@
-# Join DevNet
+# Join TestNet
 
 工作正在进行中

documentation/cn/updating-lotus.md

@@ -0,0 +1,3 @@
+# Updating Lotus
+
+工作正在进行中

documentation/en/api-scripting-support.md

@@ -0,0 +1,31 @@
+# API Scripting Support
+
+You may want to delegate the work **Lotus Storage Miner** or **Lotus Node** perform to other machines. Here is how to setup the necessary authorization and environment variables.
+
+## Generate a JWT
+
+To generate a JWT for your environment variables, use this command:
+
+```sh
+lotus auth create-token --perm admin
+lotus-storage-miner auth create-token --perm admin
+```
+
+## Environment variables
+
+Environmental variables are variables that are defined for the current shell and are inherited by any child shells or processes. Environmental variables are used to pass information into processes that are spawned from the shell.
+
+Using the JWT you generated, you can assign it and the **multiaddr** to the appropriate environment variable.
+
+```sh
+# Lotus Node
+FULLNODE_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/1234/http"
+
+# Lotus Storage Miner
+STORAGE_API_INFO="JWT_TOKEN:/ip4/127.0.0.1/tcp/2345/http"
+```
+
+* The **Lotus Node**'s `mutliaddr` is in `~/.lotus/api`.
+* The default token is in `~/.lotus/token`.
+* The **Lotus Storage Miner**'s `multiaddr` is in `~/.lotusstorage/config`.
+* The default token is in `~/.lotusstorage/token`.

documentation/en/api-troubleshooting.md

@@ -17,7 +17,7 @@ curl -X POST \
 
 ## Sending a CID
 
-If you do not serialize the CID as a JSON IPLD link, you will receive an error. Here is an example of a broken CURL request:
+If you do not serialize the CID as a [JSON IPLD link](https://did-ipid.github.io/ipid-did-method/#txref), you will receive an error. Here is an example of a broken CURL request:
 
 ```sh
 curl -X POST \

documentation/en/api.md

@@ -4,9 +4,9 @@ Here is an early overview of how to make API calls.
 
 Implementation details for the **JSON-RPC** package are [here](https://github.com/filecoin-project/lotus/tree/master/lib/jsonrpc).
 
-## Overview
+## Overview: How do you modify the config.toml to change the API endpoint?
 
-API requests are made against `127.0.0.1:1234` unless you modify `~/.lotus/api`. 
+API requests are made against `127.0.0.1:1234` unless you modify `.lotus/config.toml`. 
 
 Options:
 
@@ -16,7 +16,11 @@ Options:
 
 ## What methods can I use?
 
-Every `method` is available in [api/api.go](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go). 
+For now, you can look into different files to find methods available to you based on your needs:
+
+* [Both Lotus node + storage miner APIs](https://github.com/filecoin-project/lotus/blob/master/api/api_common.go)
+* [Lotus node API](https://github.com/filecoin-project/lotus/blob/master/api/api_full.go)
+* [Storage miner API](https://github.com/filecoin-project/lotus/blob/master/api/api_storage.go)
 
 The necessary permissions for each are in [api/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/struct.go).
 
@@ -49,7 +53,7 @@ curl -X POST \
 
 > In the future we will add a playground to make it easier to build and experiment with API requests.
 
-## Authorization
+## CURL authorization
 
 To authorize your request, you will need to include the **JWT** in a HTTP header, for example:
 
@@ -59,23 +63,23 @@ To authorize your request, you will need to include the **JWT** in a HTTP header
 
 Admin token is stored in `~/.lotus/token` for the **Lotus Node** or `~/.lotusstorage/token` for the **Lotus Storage Miner**.
 
-## Authorization types
+## How do I generate a token?
+
+To generate a JWT with custom permissions, use this command:
+
+```sh
+# Lotus Node
+lotus auth create-token --perm admin
+
+# Lotus Storage Miner
+lotus-storage-miner auth create-token --perm admin
+```
+
+## What authorization level should I use?
 
 When viewing [api/struct.go](https://github.com/filecoin-project/lotus/blob/master/api/struct.go), you will encounter these types:
 
 - `read` - Read node state, no private data.
-- `write` - Write to local store / chain, read private data.
-- `sign` - Use private keys stored in wallet for signing.
-- `admin` - Manage permissions.
-
-Payload
-
-```json
-{
-  "Allow": [
-    "read", 
-    "write",
-    /* other options */
-  ]
-}
-```
+- `write` - Write to local store / chain, and `read` permissions.
+- `sign` - Use private keys stored in wallet for signing, `read` and `write` permissions.
+- `admin` - Manage permissions, `read`, `write`, and `sign` permissions.

documentation/en/dev-tools-pond-ui.md

@@ -27,6 +27,6 @@ Don't leave Pond unattended for more than 10 hours, the web client will eventual
 ## Troubleshooting
 
 - Turn it off and on - Start at the top
-- `rm -rf ~/.lotus ~/.lotusstorage/`
+- `rm -rf ~/.lotus ~/.lotusstorage/`, this command will delete chain sync data, stored wallets, and other configurations so be careful.
 - Verify you have the correct versions of dependencies
 - If stuck on a bad fork, try `lotus chain sethead --genesis`

documentation/en/getting-started.md

@@ -7,7 +7,7 @@ For more details about Filecoin, check out the [Filecoin Spec](https://github.co
 ## What can I learn here?
 
 - How to install Lotus on [Arch Linux](https://docs.lotu.sh/en+install-lotus-arch), [Ubuntu](https://docs.lotu.sh/en+install-lotus-ubuntu), or [MacOS](https://docs.lotu.sh/en+install-lotus-macos).
-- Joining the [Lotus DevNet](https://docs.lotu.sh/en+join-devnet).
+- Joining the [Lotus TestNet](https://docs.lotu.sh/en+join-testnet).
 - [Storing](https://docs.lotu.sh/en+storing-data) or [retrieving](https://docs.lotu.sh/en+retrieving-data) data.
 - Mining Filecoin using the **Lotus Storage Miner** in your [CLI](https://docs.lotu.sh/en+mining).
 
@@ -18,6 +18,6 @@ Lotus is architected modularly to keep clean API boundaries while using the same
 - The **Lotus Node** 
 - The **Lotus Storage Miner**
 
-The **Lotus Storage Miner** is intended to be run on the machine that manages a single storage miner instance, and is meant to communicate with the **Lotus Node** via the websockets **JSON-RPC** API for all of the chain interaction needs.
+The **Lotus Storage Miner** is intended to be run on the machine that manages a single storage miner instance, and is meant to communicate with the **Lotus Node** via the websocket **JSON-RPC** API for all of the chain interaction needs.
 
 This way, a mining operation may easily run a **Lotus Storage Miner** or many of them, connected to one or many **Lotus Node** instances.

documentation/en/hardware-mining.md

@@ -1,5 +1,50 @@
-# Mining Hardware
+# Protocol Labs Standard Testing Configuration
 
-> This page is a work in progress. EVERYTHING HERE CAN CHANGE AND WILL CHANGE. PURCHASE HARDWARE AT YOUR OWN RISK.
+> This documentation page describes the standard testing configuration the Protocol Labs team has used to test **Lotus Storage Miner**s on Lotus. There is no guarantee this testing configuration will be suitable for Filecoin storage mining at MainNet launch. If you need to buy new hardware to join the Filecoin TestNet, we recommend to buy no more hardware than you require for testing. To learn more please read this [Protocol Labs Standard Testing Configuration post](https://filecoin.io/blog/filecoin-testnet-mining/).
 
-Please check out this [GitHub issue](https://github.com/filecoin-project/lotus/issues/694) to see benchmarks from existing hardware setups if you plan on participating in the **Filecoin TestNet**
+**Sector sizes** and **minimum pledged storage** required to mine blocks are two very important Filecoin TestNet parameters that impact hardware decisions. We will continue to refine all parameters during TestNet. 
+
+BECAUSE OF THIS, OUR STANDARD TESTING CONFIGURATION FOR FILECOIN MAINNET CAN AND WILL CHANGE. YOU HAVE BEEN WARNED.
+
+## Example configuration
+
+The setup below is a minimal example for sealing 32 GiB sectors on Lotus:
+
+* 2 TB of hard drive space.
+* 8 core CPU
+* 128 GiB of RAM
+
+## TestNet discoveries
+
+* If you only have 128GiB of ram, enabling 256GB of **NVMe** swap on an SSD will help you avoid out-of-memory issues while mining.
+
+## Benchmarked GPUs
+
+GPUs are a must for getting **block rewards**. Here are a few that have been confirmed to generate **SNARKs** quickly enough to successfully mine blocks on the Lotus TestNet.
+
+* GeForce RTX 2080 Ti
+* GeForce RTX 2080 SUPER
+* GeForce RTX 2080
+* GeForce GTX 1080 Ti
+* GeForce GTX 1080
+* GeForce GTX 1060
+
+## Testing other GPUs
+
+If you want to test a GPU that is not explicitly supported, you can use the following configuration flag:
+
+```sh
+BELLMAN_CUSTOM_GPU="<NAME>:<NUMBER_OF_CORES>"
+```
+
+Here is an example of trying a GeForce GTX 1660 ti with 1536 cores.
+
+```sh
+BELLMAN_CUSTOM_GPU="GeForce GTX 1660 Ti:1536"
+```
+
+To get the number of cores for your GPU, you will need to check your card’s specifications.
+
+## Benchmarking
+
+Here is a [benchmarking tool](https://github.com/filecoin-project/lotus/tree/testnet-staging/cmd/lotus-bench) and a [GitHub issue thread](https://github.com/filecoin-project/lotus/issues/694) for those who wish to experiment with and contribute hardware setups for the **Filecoin TestNet**.

documentation/en/hardware.md

@@ -2,6 +2,6 @@
 
 > This page is a work in progress. Exact mining requirements are still in the works.
 
-Lotus can build and run on most [Linux](https://ubuntu.com/) and [MacOS](https://www.apple.com/macos) systems with at least 8GB of RAM.
+Lotus can build and run on most [Linux](https://ubuntu.com/) and [MacOS](https://www.apple.com/macos) systems with at least 8GiB of RAM.
 
 Windows is not yet supported.

documentation/en/install-lotus-arch.md

@@ -37,8 +37,8 @@ cd lotus/
 Install
 
 ```sh
-make clean all
+make clean && make all
 sudo make install
 ```
 
-After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus DevNet](https://docs.lotu.sh/en+join-devnet).
+After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus TestNet](https://docs.lotu.sh/en+join-testnet).

documentation/en/install-lotus-macos.md

@@ -42,11 +42,7 @@ In your terminal, enter this command to install Homebrew:
 Use the command `brew install` to install the following packages:
 
 ```sh
-brew install go
-brew install bzr
-brew install jq
-brew install pkg-config
-brew install rustup
+brew install go bzr jq pkg-config rustup
 ```
 
 ## Clone
@@ -59,8 +55,8 @@ cd lotus/
 ## Build
 
 ```sh
-make clean all
+make clean && make all
 sudo make install
 ```
 
-After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus DevNet](https://docs.lotu.sh/en+join-devnet).
+After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus TestNet](https://docs.lotu.sh/en+join-testnet).

documentation/en/install-lotus-ubuntu.md

@@ -40,8 +40,8 @@ cd lotus/
 Install
 
 ```sh
-make clean all
+make clean && make all
 sudo make install
 ```
 
-After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus DevNet](https://docs.lotu.sh/en+join-devnet).
+After installing Lotus, you can run the `lotus` command directly from your CLI to see usage documentation. Next, you can join the [Lotus TestNet](https://docs.lotu.sh/en+join-testnet).

documentation/en/join-testnet.md

@@ -1,10 +1,10 @@
-# Join DevNet
+# Join TestNet
 
 ## Introduction
 
-Anyone can set up a **Lotus Node** and connect to the **Lotus DevNet**. This is the best way to explore the current CLI and the **Filecoin Decentralized Storage Market**.
+Anyone can set up a **Lotus Node** and connect to the **Lotus TestNet**. This is the best way to explore the current CLI and the **Filecoin Decentralized Storage Market**.
 
-If you have installed older versions, you may need to clear existing chain data and miners if you run into any errors. You can use this command:
+If you have installed older versions, you may need to clear existing chain data, stored wallets and miners if you run into any errors. You can use this command:
 
 ```sh
 rm -rf ~/.lotus ~/.lotusstorage
@@ -26,7 +26,7 @@ lotus net peers | wc -l
 
 In order to connect to the network, you need to be connected to at least 1 peer. If you’re seeing 0 peers, read our [troubleshooting notes](https://docs.lotu.sh/en+setup-troubleshooting).
 
-## Synchronize
+## Chain sync
 
 While the daemon is running, the next requirement is to sync the chain. Run the command below to start the chain sync progress. To see current chain height, visit the [network stats page](http://stats.testnet.filecoin.io/).
 
@@ -35,27 +35,29 @@ lotus sync wait
 ```
 
 * This step will take anywhere between 30 minutes to a few hours.
-* You will be able to perform **Lotus DevNet** operations after it is finished.
+* You will be able to perform **Lotus TestNet** operations after it is finished.
 
 ## Create your first address
 
+Initialize a wallet using BLS signature formats:
+
 ```sh
 lotus wallet new bls
 ```
 
-Here is an example of the response
+Here is an example of the response:
 
 ```sh
 t3vhfme4qfvegqaz7m7q6o6afjcs67n6kpzv7t2eozio4chwpafwa2y4l7zhwd5eom7jmihzdg4s52dpvnclza
 ```
 
-- Visit the [faucet](https://lotus-faucet.kittyhawk.wtf/funds.html)
+- Visit the [faucet](https://lotus-faucet.kittyhawk.wtf/funds.html) to add funds.
 - Paste the address you created.
 - Press the send button.
 
 ## Check wallet address balance
 
-Wallet balances in the devnet are in **FIL**, the smallest denomination of FIL is an **attoFil**, where 1 attoFil = 10^-18 FIL.
+Wallet balances in the Lotus TestNet are in **FIL**, the smallest denomination of FIL is an **attoFil**, where 1 attoFil = 10^-18 FIL.
 
 ```sh
 lotus wallet balance <YOUR_NEW_ADDRESS>
@@ -63,6 +65,14 @@ lotus wallet balance <YOUR_NEW_ADDRESS>
 
 You will not see any attoFIL in your wallet if your **chain** is not fully synced.
 
+## Send FIL to another wallet
+
+To send FIL to another wallet, use this command:
+
+```
+lotus send <target> <amount>
+```
+
 ## Monitor the dashboard
 
-To see the latest network activity, including **chain block height**, **block height**, **blocktime**, **total network power**, largest **block producer miner**, check out the [monitoring dashboard](https://lotus-metrics.kittyhawk.wtf).
+To see the latest network activity, including **chain block height**, **block height**, **blocktime**, **total network power**, largest **block producer miner**, check out the [monitoring dashboard](https://stats.testnet.filecoin.io).

documentation/en/mining-troubleshooting.md

@@ -1,14 +1,37 @@
 # Mining Troubleshooting
 
+## Bellman Lockfile
+
+The **Bellman** lockfile is created to lock a GPU for a process. This bug can occur when this file isn't properly cleaned up:
+
+```sh
+mining block failed: computing election proof: github.com/filecoin-project/lotus/miner.(*Miner).mineOne
+```
+
+This bug occurs when the storage miner can't acquire the `bellman.lock`. To fix it you need to stop the `lotus-storage-miner` and remove `/tmp/bellman.lock`.
+
+## Your miner is not ready
+
 ```sh
 lotus-storage-miner info
 # WARN  main  lotus-storage-miner/main.go:73  failed to get api endpoint: (/Users/myrmidon/.lotusstorage) %!w(*errors.errorString=&{API not running (no endpoint)}):
 ```
 
-If you see this, that means your **Lotus Storage Miner** isn't ready yet.
+If you see this, that means your **Lotus Storage Miner** isn't ready yet. You need to finish [syncing the chain](https://docs.lotu.sh/en+join-testnet).
+
+## Your computer is too slow
 
 ```sh
 CAUTION: block production took longer than the block delay. Your computer may not be fast enough to keep up
 ```
 
 If you see this, that means your computer is too slow and your blocks are not included in the chain, and you will not receive any rewards.
+
+## Running out of storage
+
+```
+lotus-storage-miner pledge-sector
+# No space left on device (os error 28)
+```
+
+If you see this, that means `pledge-sector` wrote too much data to `$TMPDIR` which by default is the root partition (This is common for Linux setups). Usually your root partition does not get the largest partition of storage so you will need to change the environment variable to something else.

documentation/en/mining.md

@@ -2,7 +2,7 @@
 
 Here are instructions to learn how to perform storage mining. For hardware specifications please read [this](https://docs.lotu.sh/en+hardware-mining).
 
-It is useful to [join the DevNet](https://docs.lotu.sh/en+join-devnet) prior to attempting storage mining for the first time.
+It is useful to [join the TestNet](https://docs.lotu.sh/en+join-testnet) prior to attempting storage mining for the first time.
 
 NOTE: While a miner is running, there will be many `WARN` and `ERROR` logs.
 
@@ -17,7 +17,7 @@ lotus wallet list
 With your wallet address:
 
 - Visit the [faucet](https://lotus-faucet.kittyhawk.wtf/miner.html)
-- Click "Create Miner
+- Click "Create Miner"
 - DO NOT REFRESH THE PAGE. THIS OPERATION CAN TAKE SOME TIME.
 
 The task will be complete when you see:
@@ -50,6 +50,8 @@ To mine:
 lotus-storage-miner run
 ```
 
+If you are downloading **Filecoin Proof Parameters**, the download can take some time.
+
 Get information about your miner:
 
 ```sh
@@ -63,6 +65,8 @@ lotus-storage-miner info
 lotus-storage-miner pledge-sector
 ```
 
+* Warning: On Linux configurations, this command will write data to `$TMPDIR` which is not usually the largest partition. You should point the value to a larger partition if possible.
+
 Get **miner power** and **sector usage**:
 
 ```sh

documentation/en/retrieving-data.md

@@ -2,7 +2,7 @@
 
 > There are recent bug reports with these instructions. If you happen to encounter any problems, please create a [GitHub issue](https://github.com/filecoin-project/lotus/issues/new) and a maintainer will address the problem as soon as they can.
 
-Here are the operations you can perform after you have stored a **Data CID** with the **Lotus Storage Miner** in the network.
+Here are the operations you can perform after you have stored and sealed a **Data CID** with the **Lotus Storage Miner** in the network.
 
 If you would like to learn how to store a **Data CID** on a miner, read the instructions [here](https://docs.lotu.sh/en+storing-data).
 

documentation/en/setting-a-static-port.md

@@ -1,6 +1,6 @@
 # Static Ports
 
-For a **storage deal**, you can set a static port for the **Lotus Storage Miner**.
+Depending on how your network is set up, you may need to set a static port to successfully connect to peers to perform storage deals with your **Lotus Storage Miner**. 
 
 ## Setup
 

documentation/en/setup-troubleshooting.md

@@ -1,6 +1,6 @@
 # Setup Troubleshooting
 
-Here is a command that will delete your chain data and any miners you have set up:
+Here is a command that will delete your chain data, stored wallets, stored data and any miners you have set up:
 
 ```sh
 rm -rf ~/.lotus ~/.lotusstorage
@@ -23,6 +23,12 @@ ERROR hello hello/hello.go:81 other peer has different genesis!
 
 * Try deleting your file system's `~/.lotus` directory. Check that it exists with `ls ~/.lotus`.
 
+```sh
+- repo is already locked
+```
+
+* You already have another lotus deamon running.
+
 ## Failed messages
 
 Some errors will occur that do not prevent Lotus from working:

documentation/en/storing-data-troubleshooting.md

@@ -11,3 +11,13 @@ WARN  main  lotus/main.go:72  failed to start deal: computing commP failed: gene
 ```
 
 * There is a minimum file size of 127 bytes.
+
+## Troubleshooting Sealing
+
+Miners can check sealing progress with this command:
+
+```sh
+lotus-storage-miner sectors list
+```
+
+When sealing is complete, `pSet: NO` will become `pSet: YES`. From now on the **Data CID** is [retrievable](https://docs.lotu.sh/en+retrieving-data) from the **Lotus Storage Miner**.

documentation/en/storing-data.md

@@ -2,9 +2,11 @@
 
 > There are recent bug reports with these instructions. If you happen to encounter any problems, please create a [GitHub issue](https://github.com/filecoin-project/lotus/issues/new) and a maintainer will address the problem as soon as they can.
 
-Here are instructions for how to store data on the **Lotus DevNet**.
+Here are instructions for how to store data on the **Lotus TestNet**.
 
-## Adding a file
+## Adding a file locally
+
+Adding a file locally allows you to make miner deals on the **Lotus TestNet**.
 
 ```sh
 lotus client import ./your-example-file.txt
@@ -12,7 +14,7 @@ lotus client import ./your-example-file.txt
 
 Upon success, this command will return a **Data CID**.
 
-## List local files
+## List your local files
 
 The command to see a list of files by `CID`, `name`, `size` in bytes, and `status`:
 
@@ -27,7 +29,7 @@ bafkreierupr5ioxn4obwly4i2a5cd2rwxqi6kwmcyyylifxjsmos7hrgpe Development/sample-1
 bafkreieuk7h4zs5alzpdyhlph4lxkefowvwdho3a3pml6j7dam5mipzaii Development/sample-2.txt 30618 ok
 ```
 
-## Make a Miner Deal on DevNet
+## Make a Miner Deal on Lotus TestNet
 
 Get a list of all miners that can store data:
 
@@ -48,8 +50,8 @@ lotus client deal <Data CID> <miner> <price> <duration>
 ```
 
 * Price is in attoFIL.
-* The `duration`, which represents how long the miner will keep your file hosted, is represented in blocks. Each block represents 30 seconds.
+* The `duration`, which represents how long the miner will keep your file hosted, is represented in blocks. Each block represents 45 seconds.
 
 Upon success, this command will return a **Deal CID**. 
 
-From now on the **Data CID** is [retrievable](https://docs.lotu.sh/en+retrieving-data) from the **Lotus Storage Miner**.
+The storage miner will need to **seal** the file before it can be retrieved. If the **Lotus Storage Miner** is not running on a machine designed for sealing, the process will take a very long time. 

documentation/en/updating-lotus.md

@@ -3,12 +3,16 @@
 If you installed Lotus on your machine, you can upgrade to the latest version by doing the following:
 
 ```sh
+# get the latest
 git pull origin master
+
+# clean and remake the binaries
+make clean && make build
 ```
 
 Sometimes when you run Lotus after a pull, certain commands such as `lotus daemon` may break. 
 
-Here is a command that will delete your chain data and any miners you have set up:
+Here is a command that will delete your chain data, stored wallets and any miners you have set up:
 
 ```sh
 rm -rf ~/.lotus ~/.lotusstorage

extern/filecoin-ffi

@@ -1 +1 @@
-Subproject commit 7ddde8e0f553289c1bee3d861e559297b4d16044
+Subproject commit e32f5efc808b92560f9d7f92a5d312b5ac403b7d

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/GeertJohan/go.rice v1.0.0
 	github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
+	github.com/docker/go-units v0.4.0
 	github.com/fatih/color v1.7.0 // indirect
 	github.com/filecoin-project/chain-validation v0.0.3
 	github.com/filecoin-project/filecoin-ffi v0.0.0-20191204125133-ebb3e13addf1

go.sum

@@ -68,6 +68,8 @@ github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhY
 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=

lib/sectorbuilder/sectorbuilder.go

@@ -692,15 +692,15 @@ func fallbackPostChallengeCount(sectors uint64) uint64 {
 }
 
 func (sb *SectorBuilder) ImportFrom(osb *SectorBuilder, symlink bool) error {
-	if err := migrate(osb.cacheDir, sb.cacheDir, true); err != nil {
+	if err := migrate(osb.cacheDir, sb.cacheDir, symlink); err != nil {
 		return err
 	}
 
-	if err := migrate(osb.sealedDir, sb.sealedDir, true); err != nil {
+	if err := migrate(osb.sealedDir, sb.sealedDir, symlink); err != nil {
 		return err
 	}
 
-	if err := migrate(osb.stagedDir, sb.stagedDir, true); err != nil {
+	if err := migrate(osb.stagedDir, sb.stagedDir, symlink); err != nil {
 		return err
 	}
 

lib/statestore/store.go

@@ -8,6 +8,7 @@ import (
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/query"
 	cbg "github.com/whyrusleeping/cbor-gen"
+	"go.uber.org/multierr"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/lib/cborutil"
@@ -139,6 +140,8 @@ func (st *StateStore) List(out interface{}) error {
 	outT := reflect.TypeOf(out).Elem().Elem()
 	rout := reflect.ValueOf(out)
 
+	var errs error
+
 	for {
 		res, ok := res.NextSync()
 		if !ok {
@@ -151,7 +154,8 @@ func (st *StateStore) List(out interface{}) error {
 		elem := reflect.New(outT)
 		err := cborutil.ReadCborRPC(bytes.NewReader(res.Value), elem.Interface())
 		if err != nil {
-			return err
+			errs = multierr.Append(errs, xerrors.Errorf("decoding state for key '%s': %w", res.Key, err))
+			continue
 		}
 
 		rout.Elem().Set(reflect.Append(rout.Elem(), elem.Elem()))

node/builder.go

@@ -217,7 +217,6 @@ func Online() Option {
 			Override(RunBlockSyncKey, modules.RunBlockSync),
 			Override(RunPeerMgrKey, modules.RunPeerMgr),
 			Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks),
-			Override(HeadMetricsKey, metrics.SendHeadNotifs("")),
 
 			Override(new(*discovery.Local), discovery.NewLocal),
 			Override(new(discovery.PeerResolver), modules.RetrievalResolver),
@@ -313,7 +312,9 @@ func ConfigFullNode(c interface{}) Option {
 
 	return Options(
 		ConfigCommon(&cfg.Common),
-		Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)),
+		If(cfg.Metrics.HeadNotifs,
+			Override(HeadMetricsKey, metrics.SendHeadNotifs(cfg.Metrics.Nickname)),
+		),
 		If(cfg.Metrics.PubsubTracing,
 			Override(new(*pubsub.PubSub), lp2p.GossipSub(lp2p.PubsubTracer())),
 		),

node/config/def.go

@@ -42,6 +42,7 @@ type Libp2p struct {
 
 type Metrics struct {
 	Nickname      string
+	HeadNotifs     bool
 	PubsubTracing bool
 }
 

node/hello/hello.go

@@ -139,7 +139,7 @@ func (hs *Service) SayHello(ctx context.Context, pid peer.ID) error {
 		hmsg = &Message{}
 		s.SetReadDeadline(time.Now().Add(10 * time.Second))
 		err := cborutil.ReadCborRPC(s, hmsg) // ignore error
-		ok := err != nil
+		ok := err == nil
 
 		t3 := time.Now()
 		lat := t3.Sub(t0)

node/modules/testing/genesis.go

@@ -68,7 +68,7 @@ func MakeGenesisMem(out io.Writer, gmc *gen.GenMinerCfg) func(bs dtypes.ChainBlo
 	}
 }
 
-func MakeGenesis(outFile, presealInfo string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
+func MakeGenesis(outFile, presealInfo, timestamp string) func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
 	return func(bs dtypes.ChainBlockstore, w *wallet.Wallet) modules.Genesis {
 		return func() (*types.BlockHeader, error) {
 			glog.Warn("Generating new random genesis block, note that this SHOULD NOT happen unless you are setting up new network")
@@ -119,7 +119,18 @@ func MakeGenesis(outFile, presealInfo string) func(bs dtypes.ChainBlockstore, w
 				addrs[miner.Worker] = types.FromFil(100000)
 			}
 
-			b, err := gen.MakeGenesisBlock(bs, addrs, gmc, uint64(time.Now().Unix()))
+			ts := uint64(time.Now().Unix())
+			if timestamp != "" {
+				t, err := time.Parse(time.RFC3339, timestamp)
+				if err != nil {
+					return nil, xerrors.Errorf("parsing input genesis timestamp: %w", err)
+				}
+
+				glog.Infof("will use %s as the genesis timestamp", t)
+				ts = uint64(t.Unix())
+			}
+
+			b, err := gen.MakeGenesisBlock(bs, addrs, gmc, ts)
 			if err != nil {
 				return nil, err
 			}

peermgr/peermgr.go

@@ -20,7 +20,7 @@ var log = logging.Logger("peermgr")
 
 const (
 	MaxFilPeers = 32
-	MinFilPeers = 8
+	MinFilPeers = 12
 )
 
 type MaybePeerMgr struct {

retrieval/client.go

@@ -100,7 +100,7 @@ type clientStream struct {
 func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64, total types.BigInt, miner peer.ID, client, minerAddr address.Address, out io.Writer) error {
 	s, err := c.h.NewStream(ctx, miner, ProtocolID)
 	if err != nil {
-		return err
+		return xerrors.Errorf("failed to open stream to miner for retrieval query: %w", err)
 	}
 	defer s.Close()
 
@@ -149,7 +149,6 @@ func (c *Client) RetrieveUnixfs(ctx context.Context, root cid.Cid, size uint64,
 
 		cst.offset += toFetch
 	}
-	log.Info("RETRIEVE SUCCESSFUL")
 	return nil
 }
 
@@ -173,13 +172,12 @@ func (cst *clientStream) doOneExchange(ctx context.Context, toFetch uint64, out
 	}
 
 	if err := cborutil.WriteCborRPC(cst.stream, deal); err != nil {
-		return err
+		return xerrors.Errorf("sending incremental retrieval request: %w", err)
 	}
 
 	var resp DealResponse
 	if err := cborutil.ReadCborRPC(cst.peeker, &resp); err != nil {
-		log.Error(err)
-		return err
+		return xerrors.Errorf("reading retrieval response: %w", err)
 	}
 
 	if resp.Status != Accepted {

storage/sectorblocks/blockstore.go

@@ -2,6 +2,8 @@ package sectorblocks
 
 import (
 	"context"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/storage"
 	"io/ioutil"
 
 	blocks "github.com/ipfs/go-block-format"
@@ -72,11 +74,22 @@ func (s *SectorBlockStore) Get(c cid.Cid) (blocks.Block, error) {
 		return nil, blockstore.ErrNotFound
 	}
 
-	best := refs[0] // TODO: better strategy (e.g. look for already unsealed)
-
-	si, err := s.sectorBlocks.Miner.GetSectorInfo(best.SectorID)
-	if err != nil {
-		return nil, xerrors.Errorf("getting sector info: %w", err)
+	// TODO: better strategy (e.g. look for already unsealed)
+	var best api.SealedRef
+	var bestSi storage.SectorInfo
+	for _, r := range refs {
+		si, err := s.sectorBlocks.Miner.GetSectorInfo(r.SectorID)
+		if err != nil {
+			return nil, xerrors.Errorf("getting sector info: %w", err)
+		}
+		if si.State == api.Proving {
+			best = r
+			bestSi = si
+			break
+		}
+	}
+	if bestSi.State == api.UndefinedSectorState {
+		return nil, xerrors.New("no sealed sector found")
 	}
 
 	log.Infof("reading block %s from sector %d(+%d;%d)", c, best.SectorID, best.Offset, best.Size)
@@ -85,8 +98,8 @@ func (s *SectorBlockStore) Get(c cid.Cid) (blocks.Block, error) {
 		best.SectorID,
 		best.Offset,
 		best.Size,
-		si.Ticket.TicketBytes,
-		si.CommD,
+		bestSi.Ticket.TicketBytes,
+		bestSi.CommD,
 	)
 	if err != nil {
 		return nil, xerrors.Errorf("unsealing block: %w", err)

storage/sectors.go

@@ -65,7 +65,7 @@ func (m *Miner) UpdateSectorState(ctx context.Context, sector uint64, snonce uin
 func (m *Miner) sectorStateLoop(ctx context.Context) error {
 	trackedSectors, err := m.ListSectors()
 	if err != nil {
-		return xerrors.Errorf("loading sector list: %w", err)
+		log.Errorf("loading sector list: %+v", err)
 	}
 
 	go func() {

storage/sectors_test.go

@@ -0,0 +1,56 @@
+package storage
+
+import (
+	"bytes"
+	"testing"
+
+	"gotest.tools/assert"
+
+	"github.com/filecoin-project/lotus/lib/cborutil"
+)
+
+func TestSectorInfoSelialization(t *testing.T) {
+	si := &SectorInfo{
+		State:    123,
+		SectorID: 234,
+		Nonce:    345,
+		Pieces: []Piece{{
+			DealID: 1234,
+			Size:   5,
+			CommP:  []byte{3},
+		}},
+		CommD: []byte{32, 4},
+		CommR: nil,
+		Proof: nil,
+		Ticket: SealTicket{
+			BlockHeight: 345,
+			TicketBytes: []byte{87, 78, 7, 87},
+		},
+		PreCommitMessage: nil,
+		Seed:             SealSeed{},
+		CommitMessage:    nil,
+		FaultReportMsg:   nil,
+		LastErr:          "hi",
+	}
+
+	b, err := cborutil.Dump(si)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var si2 SectorInfo
+	if err := cborutil.ReadCborRPC(bytes.NewReader(b), &si); err != nil {
+		return
+	}
+
+	assert.Equal(t, si.State, si2.State)
+	assert.Equal(t, si.Nonce, si2.Nonce)
+	assert.Equal(t, si.SectorID, si2.SectorID)
+
+	assert.Equal(t, si.Pieces, si2.Pieces)
+	assert.Equal(t, si.CommD, si2.CommD)
+	assert.Equal(t, si.Ticket, si2.Ticket)
+
+	assert.Equal(t, si, si2)
+
+}

tools/stats/metrics.go

@@ -14,6 +14,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/address"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/multiformats/go-multihash"
 
 	_ "github.com/influxdata/influxdb1-client"
 	models "github.com/influxdata/influxdb1-client/models"
@@ -124,8 +125,11 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti
 		if err != nil {
 			return err
 		}
+		p := NewPoint("chain.election", 1)
+		p.AddTag("miner", blockheader.Miner.String())
+		pl.AddPoint(p)
 
-		p := NewPoint("chain.blockheader_size", len(bs))
+		p = NewPoint("chain.blockheader_size", len(bs))
 		pl.AddPoint(p)
 	}
 
@@ -212,7 +216,13 @@ func RecordTipsetMessagesPoints(ctx context.Context, api api.FullNode, pl *Point
 		}
 
 		p = NewPoint("chain.message_count", 1)
-		p.AddTag("actor", actor.Code.String())
+
+		dm, err := multihash.Decode(actor.Code.Hash())
+		if err != nil {
+			continue
+		}
+
+		p.AddTag("actor", string(dm.Digest))
 		p.AddTag("method", fmt.Sprintf("%d", msg.Message.Method))
 		p.AddTag("exitcode", fmt.Sprintf("%d", recp[i].ExitCode))
 		pl.AddPoint(p)