Merge pull request #975 from filecoin-project/feat/cli-slashing

chain slash-consensus command

.dockerignore

@@ -0,0 +1 @@
+.gitignore

.gitignore

@@ -2,6 +2,7 @@
 /lotus-storage-miner
 /lotus-seal-worker
 /lotus-seed
+/lotus-health
 /pond
 /townhall
 /fountain

Dockerfile

@@ -0,0 +1,91 @@
+FROM golang:1.13.4-buster
+MAINTAINER ldoublewood <ldoublewood@gmail.com>
+
+ENV SRC_DIR /lotus
+
+RUN apt-get update && apt-get install -y && apt-get install -y ca-certificates llvm clang mesa-opencl-icd ocl-icd-opencl-dev
+
+RUN curl -sSf https://sh.rustup.rs | sh -s -- -y
+
+
+# Get su-exec, a very minimal tool for dropping privileges,
+# and tini, a very minimal init daemon for containers
+ENV SUEXEC_VERSION v0.2
+ENV TINI_VERSION v0.18.0
+RUN set -x \
+  && cd /tmp \
+  && git clone https://github.com/ncopa/su-exec.git \
+  && cd su-exec \
+  && git checkout -q $SUEXEC_VERSION \
+  && make \
+  && cd /tmp \
+  && wget -q -O tini https://github.com/krallin/tini/releases/download/$TINI_VERSION/tini \
+  && chmod +x tini
+
+# Download packages first so they can be cached.
+COPY go.mod go.sum $SRC_DIR/
+COPY extern/ $SRC_DIR/extern/
+RUN cd $SRC_DIR \
+  && go mod download
+
+COPY Makefile $SRC_DIR
+
+# Because extern/filecoin-ffi building script need to get version number from git
+COPY .git/ $SRC_DIR/.git/
+COPY .gitmodules $SRC_DIR/
+
+# Download dependence first
+RUN cd $SRC_DIR \
+  && mkdir $SRC_DIR/build \
+  && . $HOME/.cargo/env \
+  && make clean \
+  && make deps
+
+
+COPY . $SRC_DIR
+
+# Build the thing.
+RUN cd $SRC_DIR \
+  && . $HOME/.cargo/env \
+  && make
+
+# Now comes the actual target image, which aims to be as small as possible.
+FROM busybox:1-glibc
+MAINTAINER ldoublewood <ldoublewood@gmail.com>
+
+# Get the executable binary and TLS CAs from the build container.
+ENV SRC_DIR /lotus
+COPY --from=0 $SRC_DIR/lotus /usr/local/bin/lotus
+COPY --from=0 $SRC_DIR/lotus-storage-miner /usr/local/bin/lotus-storage-miner
+COPY --from=0 /tmp/su-exec/su-exec /sbin/su-exec
+COPY --from=0 /tmp/tini /sbin/tini
+COPY --from=0 /etc/ssl/certs /etc/ssl/certs
+
+
+# This shared lib (part of glibc) doesn't seem to be included with busybox.
+COPY --from=0 /lib/x86_64-linux-gnu/libdl-2.28.so /lib/libdl.so.2
+COPY --from=0 /lib/x86_64-linux-gnu/libutil-2.28.so /lib/libutil.so.1 
+COPY --from=0 /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /lib/libOpenCL.so.1
+COPY --from=0 /lib/x86_64-linux-gnu/librt-2.28.so /lib/librt.so.1
+COPY --from=0 /lib/x86_64-linux-gnu/libgcc_s.so.1 /lib/libgcc_s.so.1
+
+# WS port
+EXPOSE 1234
+# P2P port
+EXPOSE 5678
+
+
+# Create the home directory and switch to a non-privileged user.
+ENV HOME_PATH /data
+ENV PARAMCACHE_PATH /var/tmp/filecoin-proof-parameters
+
+RUN mkdir -p $HOME_PATH \
+  && adduser -D -h $HOME_PATH -u 1000 -G users lotus \
+  && chown lotus:users $HOME_PATH
+
+
+VOLUME $HOME_PATH
+VOLUME $PARAMCACHE_PATH
+
+# Execute the daemon subcommand by default
+CMD ["/sbin/tini", "--", "lotus", "daemon"]

Makefile

@@ -14,7 +14,7 @@ MODULES:=
 
 CLEAN:=
 BINS:=
-GOFLAGS+=-ldflags='-X="github.com/filecoin-project/lotus/build".CurrentCommit=+git$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null))'
+GOFLAGS+=-ldflags=-X="github.com/filecoin-project/lotus/build".CurrentCommit="+git$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null))"
 
 ## FFI
 

api/api_full.go

@@ -38,6 +38,7 @@ type FullNode interface {
 	ChainGetNode(ctx context.Context, p string) (interface{}, error)
 	ChainGetMessage(context.Context, cid.Cid) (*types.Message, error)
 	ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*store.HeadChange, error)
+	ChainExport(context.Context, *types.TipSet) (<-chan []byte, error)
 
 	// syncer
 	SyncState(context.Context) (*SyncState, error)
@@ -94,7 +95,7 @@ type FullNode interface {
 	//ClientListAsks() []Ask
 
 	// if tipset is nil, we'll use heaviest
-	StateCall(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error)
+	StateCall(context.Context, *types.Message, *types.TipSet) (*MethodCall, error)
 	StateReplay(context.Context, *types.TipSet, cid.Cid) (*ReplayResults, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error)
 	StateReadState(ctx context.Context, act *types.Actor, ts *types.TipSet) (*ActorState, error)
@@ -270,6 +271,11 @@ type ReplayResults struct {
 	Error   string
 }
 
+type MethodCall struct {
+	types.MessageReceipt
+	Error string
+}
+
 type ActiveSync struct {
 	Base   *types.TipSet
 	Target *types.TipSet

api/api_storage.go

@@ -19,7 +19,7 @@ const (
 
 	Unsealed      // sealing / queued
 	PreCommitting // on chain pre-commit
-	PreCommitted  // waiting for seed
+	WaitSeed      // waiting for seed
 	Committing
 	CommitWait // waiting for message to land on chain
 	Proving
@@ -45,7 +45,7 @@ const (
 	PreCommitFailed
 	SealCommitFailed
 	CommitFailed
-	_
+	PackingFailed
 	_
 	_
 	_
@@ -61,7 +61,7 @@ var SectorStates = []string{
 	Packing:              "Packing",
 	Unsealed:             "Unsealed",
 	PreCommitting:        "PreCommitting",
-	PreCommitted:         "PreCommitted",
+	WaitSeed:             "WaitSeed",
 	Committing:           "Committing",
 	CommitWait:           "CommitWait",
 	Proving:              "Proving",
@@ -70,6 +70,7 @@ var SectorStates = []string{
 	PreCommitFailed:  "PreCommitFailed",
 	SealCommitFailed: "SealCommitFailed",
 	CommitFailed:     "CommitFailed",
+	PackingFailed:    "PackingFailed",
 
 	FailedUnrecoverable: "FailedUnrecoverable",
 
@@ -107,6 +108,15 @@ type StorageMiner interface {
 	WorkerDone(ctx context.Context, task uint64, res sectorbuilder.SealRes) error
 }
 
+type SectorLog struct {
+	Kind      string
+	Timestamp uint64
+
+	Trace string
+
+	Message string
+}
+
 type SectorInfo struct {
 	SectorID uint64
 	State    SectorState
@@ -119,6 +129,8 @@ type SectorInfo struct {
 	Retries  uint64
 
 	LastErr string
+
+	Log []SectorLog
 }
 
 type SealedRef struct {

api/apistruct/struct.go

@@ -56,6 +56,7 @@ type FullNodeStruct struct {
 		ChainGetNode           func(ctx context.Context, p string) (interface{}, error)                             `perm:"read"`
 		ChainGetMessage        func(context.Context, cid.Cid) (*types.Message, error)                               `perm:"read"`
 		ChainGetPath           func(context.Context, types.TipSetKey, types.TipSetKey) ([]*store.HeadChange, error) `perm:"read"`
+		ChainExport            func(context.Context, *types.TipSet) (<-chan []byte, error)                          `perm:"read"`
 
 		SyncState          func(context.Context) (*api.SyncState, error)                `perm:"read"`
 		SyncSubmitBlock    func(ctx context.Context, blk *types.BlockMsg) error         `perm:"write"`
@@ -98,7 +99,7 @@ type FullNodeStruct struct {
 		StateMinerPeerID              func(ctx context.Context, m address.Address, ts *types.TipSet) (peer.ID, error)                   `perm:"read"`
 		StateMinerElectionPeriodStart func(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error)                `perm:"read"`
 		StateMinerSectorSize          func(context.Context, address.Address, *types.TipSet) (uint64, error)                             `perm:"read"`
-		StateCall                     func(context.Context, *types.Message, *types.TipSet) (*types.MessageReceipt, error)               `perm:"read"`
+		StateCall                     func(context.Context, *types.Message, *types.TipSet) (*api.MethodCall, error)                     `perm:"read"`
 		StateReplay                   func(context.Context, *types.TipSet, cid.Cid) (*api.ReplayResults, error)                         `perm:"read"`
 		StateGetActor                 func(context.Context, address.Address, *types.TipSet) (*types.Actor, error)                       `perm:"read"`
 		StateReadState                func(context.Context, *types.Actor, *types.TipSet) (*api.ActorState, error)                       `perm:"read"`
@@ -361,6 +362,10 @@ func (c *FullNodeStruct) ChainGetPath(ctx context.Context, from types.TipSetKey,
 	return c.Internal.ChainGetPath(ctx, from, to)
 }
 
+func (c *FullNodeStruct) ChainExport(ctx context.Context, ts *types.TipSet) (<-chan []byte, error) {
+	return c.Internal.ChainExport(ctx, ts)
+}
+
 func (c *FullNodeStruct) SyncState(ctx context.Context) (*api.SyncState, error) {
 	return c.Internal.SyncState(ctx)
 }
@@ -405,7 +410,7 @@ func (c *FullNodeStruct) StateMinerSectorSize(ctx context.Context, actor address
 	return c.Internal.StateMinerSectorSize(ctx, actor, ts)
 }
 
-func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error) {
+func (c *FullNodeStruct) StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.MethodCall, error) {
 	return c.Internal.StateCall(ctx, msg, ts)
 }
 

api/cbor_gen.go

@@ -98,114 +98,95 @@ func (t *PaymentInfo) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type map")
 	}
 
-	if extra != 3 {
-		return fmt.Errorf("cbor input had wrong number of fields")
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("PaymentInfo: map struct too large (%d)", extra)
 	}
 
 	var name string
+	n := extra
 
-	// t.Channel (address.Address) (struct)
+	for i := uint64(0); i < n; i++ {
 
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		name = string(sval)
-	}
-
-	if name != "Channel" {
-		return fmt.Errorf("expected struct map entry %s to be Channel", name)
-	}
-
-	{
-
-		if err := t.Channel.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.ChannelMessage (cid.Cid) (struct)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		name = string(sval)
-	}
-
-	if name != "ChannelMessage" {
-		return fmt.Errorf("expected struct map entry %s to be ChannelMessage", name)
-	}
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
+		{
+			sval, err := cbg.ReadString(br)
+			if err != nil {
 				return err
 			}
-		} else {
 
-			c, err := cbg.ReadCid(br)
+			name = string(sval)
+		}
+
+		switch name {
+		// t.Channel (address.Address) (struct)
+		case "Channel":
+
+			{
+
+				if err := t.Channel.UnmarshalCBOR(br); err != nil {
+					return err
+				}
+
+			}
+			// t.ChannelMessage (cid.Cid) (struct)
+		case "ChannelMessage":
+
+			{
+
+				pb, err := br.PeekByte()
+				if err != nil {
+					return err
+				}
+				if pb == cbg.CborNull[0] {
+					var nbuf [1]byte
+					if _, err := br.Read(nbuf[:]); err != nil {
+						return err
+					}
+				} else {
+
+					c, err := cbg.ReadCid(br)
+					if err != nil {
+						return xerrors.Errorf("failed to read cid field t.ChannelMessage: %w", err)
+					}
+
+					t.ChannelMessage = &c
+				}
+
+			}
+			// t.Vouchers ([]*types.SignedVoucher) (slice)
+		case "Vouchers":
+
+			maj, extra, err = cbg.CborReadHeader(br)
 			if err != nil {
-				return xerrors.Errorf("failed to read cid field t.ChannelMessage: %w", err)
+				return err
 			}
 
-			t.ChannelMessage = &c
+			if extra > cbg.MaxLength {
+				return fmt.Errorf("t.Vouchers: array too large (%d)", extra)
+			}
+
+			if maj != cbg.MajArray {
+				return fmt.Errorf("expected cbor array")
+			}
+			if extra > 0 {
+				t.Vouchers = make([]*types.SignedVoucher, extra)
+			}
+			for i := 0; i < int(extra); i++ {
+
+				var v types.SignedVoucher
+				if err := v.UnmarshalCBOR(br); err != nil {
+					return err
+				}
+
+				t.Vouchers[i] = &v
+			}
+
+		default:
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
-
-	}
-	// t.Vouchers ([]*types.SignedVoucher) (slice)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		name = string(sval)
-	}
-
-	if name != "Vouchers" {
-		return fmt.Errorf("expected struct map entry %s to be Vouchers", name)
-	}
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("t.Vouchers: array too large (%d)", extra)
-	}
-
-	if maj != cbg.MajArray {
-		return fmt.Errorf("expected cbor array")
-	}
-	if extra > 0 {
-		t.Vouchers = make([]*types.SignedVoucher, extra)
-	}
-	for i := 0; i < int(extra); i++ {
-
-		var v types.SignedVoucher
-		if err := v.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-		t.Vouchers[i] = &v
 	}
 
 	return nil
 }
-
 func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
@@ -276,84 +257,66 @@ func (t *SealedRef) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type map")
 	}
 
-	if extra != 3 {
-		return fmt.Errorf("cbor input had wrong number of fields")
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("SealedRef: map struct too large (%d)", extra)
 	}
 
 	var name string
+	n := extra
 
-	// t.SectorID (uint64) (uint64)
+	for i := uint64(0); i < n; i++ {
 
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
+		{
+			sval, err := cbg.ReadString(br)
+			if err != nil {
+				return err
+			}
+
+			name = string(sval)
 		}
 
-		name = string(sval)
-	}
+		switch name {
+		// t.SectorID (uint64) (uint64)
+		case "SectorID":
 
-	if name != "SectorID" {
-		return fmt.Errorf("expected struct map entry %s to be SectorID", name)
-	}
+			maj, extra, err = cbg.CborReadHeader(br)
+			if err != nil {
+				return err
+			}
+			if maj != cbg.MajUnsignedInt {
+				return fmt.Errorf("wrong type for uint64 field")
+			}
+			t.SectorID = uint64(extra)
+			// t.Offset (uint64) (uint64)
+		case "Offset":
 
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.SectorID = uint64(extra)
-	// t.Offset (uint64) (uint64)
+			maj, extra, err = cbg.CborReadHeader(br)
+			if err != nil {
+				return err
+			}
+			if maj != cbg.MajUnsignedInt {
+				return fmt.Errorf("wrong type for uint64 field")
+			}
+			t.Offset = uint64(extra)
+			// t.Size (uint64) (uint64)
+		case "Size":
 
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
+			maj, extra, err = cbg.CborReadHeader(br)
+			if err != nil {
+				return err
+			}
+			if maj != cbg.MajUnsignedInt {
+				return fmt.Errorf("wrong type for uint64 field")
+			}
+			t.Size = uint64(extra)
+
+		default:
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
-
-		name = string(sval)
 	}
 
-	if name != "Offset" {
-		return fmt.Errorf("expected struct map entry %s to be Offset", name)
-	}
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.Offset = uint64(extra)
-	// t.Size (uint64) (uint64)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		name = string(sval)
-	}
-
-	if name != "Size" {
-		return fmt.Errorf("expected struct map entry %s to be Size", name)
-	}
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.Size = uint64(extra)
 	return nil
 }
-
 func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
@@ -401,50 +364,56 @@ func (t *SealedRefs) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type map")
 	}
 
-	if extra != 1 {
-		return fmt.Errorf("cbor input had wrong number of fields")
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("SealedRefs: map struct too large (%d)", extra)
 	}
 
 	var name string
+	n := extra
 
-	// t.Refs ([]api.SealedRef) (slice)
+	for i := uint64(0); i < n; i++ {
 
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
+		{
+			sval, err := cbg.ReadString(br)
+			if err != nil {
+				return err
+			}
+
+			name = string(sval)
 		}
 
-		name = string(sval)
-	}
+		switch name {
+		// t.Refs ([]api.SealedRef) (slice)
+		case "Refs":
 
-	if name != "Refs" {
-		return fmt.Errorf("expected struct map entry %s to be Refs", name)
-	}
+			maj, extra, err = cbg.CborReadHeader(br)
+			if err != nil {
+				return err
+			}
 
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
+			if extra > cbg.MaxLength {
+				return fmt.Errorf("t.Refs: array too large (%d)", extra)
+			}
 
-	if extra > cbg.MaxLength {
-		return fmt.Errorf("t.Refs: array too large (%d)", extra)
-	}
+			if maj != cbg.MajArray {
+				return fmt.Errorf("expected cbor array")
+			}
+			if extra > 0 {
+				t.Refs = make([]SealedRef, extra)
+			}
+			for i := 0; i < int(extra); i++ {
 
-	if maj != cbg.MajArray {
-		return fmt.Errorf("expected cbor array")
-	}
-	if extra > 0 {
-		t.Refs = make([]SealedRef, extra)
-	}
-	for i := 0; i < int(extra); i++ {
+				var v SealedRef
+				if err := v.UnmarshalCBOR(br); err != nil {
+					return err
+				}
 
-		var v SealedRef
-		if err := v.UnmarshalCBOR(br); err != nil {
-			return err
+				t.Refs[i] = v
+			}
+
+		default:
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
-
-		t.Refs[i] = v
 	}
 
 	return nil

api/test/deals.go

@@ -65,7 +65,6 @@ func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
 		defer close(done)
 		for mine {
 			time.Sleep(blocktime)
-			fmt.Println("mining a block now")
 			if err := sn[0].MineOne(ctx); err != nil {
 				t.Error(err)
 			}

build/version.go

@@ -5,7 +5,7 @@ import "fmt"
 var CurrentCommit string
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "0.2.5"
+const BuildVersion = "0.2.6"
 
 var UserVersion = BuildVersion + CurrentCommit
 

chain/actors/actor_storagemarket.go

@@ -13,7 +13,7 @@ import (
 	"github.com/ipfs/go-hamt-ipld"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
+	cborutil "github.com/filecoin-project/go-cbor-util"
 	"github.com/filecoin-project/go-sectorbuilder"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors/aerrors"

chain/actors/actors_test.go

@@ -45,7 +45,7 @@ func setupVMTestEnv(t *testing.T) (*vm.VM, []address.Address, bstore.Blockstore)
 		t.Fatal(err)
 	}
 
-	stateroot, err := st.Flush()
+	stateroot, err := st.Flush(context.TODO())
 	if err != nil {
 		t.Fatal(err)
 	}

chain/actors/harness2_test.go

@@ -3,10 +3,11 @@ package actors_test
 import (
 	"bytes"
 	"context"
-	"github.com/filecoin-project/go-sectorbuilder"
 	"math/rand"
 	"testing"
 
+	"github.com/filecoin-project/go-sectorbuilder"
+
 	"github.com/ipfs/go-cid"
 	dstore "github.com/ipfs/go-datastore"
 	hamt "github.com/ipfs/go-hamt-ipld"
@@ -185,7 +186,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness {
 		t.Fatal(err)
 	}
 
-	stateroot, err := st.Flush()
+	stateroot, err := st.Flush(context.TODO())
 	if err != nil {
 		t.Fatal(err)
 	}

chain/blocksync/blocksync.go

@@ -3,17 +3,17 @@ package blocksync
 import (
 	"bufio"
 	"context"
+	"time"
 
 	"github.com/libp2p/go-libp2p-core/protocol"
 	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/go-cbor-util"
+	cborutil "github.com/filecoin-project/go-cbor-util"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 
 	"github.com/ipfs/go-cid"
-	cbor "github.com/ipfs/go-ipld-cbor"
 	logging "github.com/ipfs/go-log/v2"
 	inet "github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
@@ -25,11 +25,7 @@ type NewStreamFunc func(context.Context, peer.ID, ...protocol.ID) (inet.Stream,
 
 const BlockSyncProtocolID = "/fil/sync/blk/0.0.1"
 
-func init() {
-	cbor.RegisterCborType(BlockSyncRequest{})
-	cbor.RegisterCborType(BlockSyncResponse{})
-	cbor.RegisterCborType(BSTipSet{})
-}
+const BlockSyncMaxRequestLength = 800
 
 type BlockSyncService struct {
 	cs *store.ChainStore
@@ -55,8 +51,17 @@ func ParseBSOptions(optfield uint64) *BSOptions {
 }
 
 const (
-	BSOptBlocks   = 1 << 0
-	BSOptMessages = 1 << 1
+	BSOptBlocks = 1 << iota
+	BSOptMessages
+)
+
+const (
+	StatusOK            = uint64(0)
+	StatusPartial       = uint64(101)
+	StatusNotFound      = uint64(201)
+	StatusGoAway        = uint64(202)
+	StatusInternalError = uint64(203)
+	StatusBadRequest    = uint64(204)
 )
 
 type BlockSyncResponse struct {
@@ -93,28 +98,30 @@ func (bss *BlockSyncService) HandleStream(s inet.Stream) {
 		log.Warnf("failed to read block sync request: %s", err)
 		return
 	}
-	log.Infof("block sync request for: %s %d", req.Start, req.RequestLength)
+	log.Infow("block sync request", "start", req.Start, "len", req.RequestLength)
 
-	resp, err := bss.processRequest(ctx, &req)
+	resp, err := bss.processRequest(ctx, s.Conn().RemotePeer(), &req)
 	if err != nil {
 		log.Warn("failed to process block sync request: ", err)
 		return
 	}
 
+	writeDeadline := 60 * time.Second
+	s.SetDeadline(time.Now().Add(writeDeadline))
 	if err := cborutil.WriteCborRPC(s, resp); err != nil {
-		log.Warn("failed to write back response for handle stream: ", err)
+		log.Warnw("failed to write back response for handle stream", "err", err, "peer", s.Conn().RemotePeer())
 		return
 	}
 }
 
-func (bss *BlockSyncService) processRequest(ctx context.Context, req *BlockSyncRequest) (*BlockSyncResponse, error) {
+func (bss *BlockSyncService) processRequest(ctx context.Context, p peer.ID, req *BlockSyncRequest) (*BlockSyncResponse, error) {
 	_, span := trace.StartSpan(ctx, "blocksync.ProcessRequest")
 	defer span.End()
 
 	opts := ParseBSOptions(req.Options)
 	if len(req.Start) == 0 {
 		return &BlockSyncResponse{
-			Status:  204,
+			Status:  StatusBadRequest,
 			Message: "no cids given in blocksync request",
 		}, nil
 	}
@@ -122,20 +129,32 @@ func (bss *BlockSyncService) processRequest(ctx context.Context, req *BlockSyncR
 	span.AddAttributes(
 		trace.BoolAttribute("blocks", opts.IncludeBlocks),
 		trace.BoolAttribute("messages", opts.IncludeMessages),
+		trace.Int64Attribute("reqlen", int64(req.RequestLength)),
 	)
 
-	chain, err := bss.collectChainSegment(types.NewTipSetKey(req.Start...), req.RequestLength, opts)
+	reqlen := req.RequestLength
+	if reqlen > BlockSyncMaxRequestLength {
+		log.Warnw("limiting blocksync request length", "orig", req.RequestLength, "peer", p)
+		reqlen = BlockSyncMaxRequestLength
+	}
+
+	chain, err := bss.collectChainSegment(types.NewTipSetKey(req.Start...), reqlen, opts)
 	if err != nil {
 		log.Warn("encountered error while responding to block sync request: ", err)
 		return &BlockSyncResponse{
-			Status:  203,
+			Status:  StatusInternalError,
 			Message: err.Error(),
 		}, nil
 	}
 
+	status := StatusOK
+	if reqlen < req.RequestLength {
+		status = StatusPartial
+	}
+
 	return &BlockSyncResponse{
 		Chain:  chain,
-		Status: 0,
+		Status: status,
 	}, nil
 }
 

chain/blocksync/blocksync_client.go

@@ -18,9 +18,10 @@ import (
 	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/go-cbor-util"
+	cborutil "github.com/filecoin-project/go-cbor-util"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
+	incrt "github.com/filecoin-project/lotus/lib/increadtimeout"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/peermgr"
 )
@@ -44,15 +45,15 @@ func NewBlockSyncClient(bserv dtypes.ChainBlockService, h host.Host, pmgr peermg
 
 func (bs *BlockSync) processStatus(req *BlockSyncRequest, res *BlockSyncResponse) error {
 	switch res.Status {
-	case 101: // Partial Response
+	case StatusPartial: // Partial Response
 		return xerrors.Errorf("not handling partial blocksync responses yet")
-	case 201: // req.Start not found
+	case StatusNotFound: // req.Start not found
 		return xerrors.Errorf("not found")
-	case 202: // Go Away
+	case StatusGoAway: // Go Away
 		return xerrors.Errorf("not handling 'go away' blocksync responses yet")
-	case 203: // Internal Error
+	case StatusInternalError: // Internal Error
 		return xerrors.Errorf("block sync peer errored: %s", res.Message)
-	case 204:
+	case StatusBadRequest:
 		return xerrors.Errorf("block sync request invalid: %s", res.Message)
 	default:
 		return xerrors.Errorf("unrecognized response code: %d", res.Status)
@@ -194,11 +195,16 @@ func (bs *BlockSync) GetChainMessages(ctx context.Context, h *types.TipSet, coun
 			continue
 		}
 
-		if res.Status == 0 {
+		if res.Status == StatusOK {
 			bs.syncPeers.logGlobalSuccess(time.Since(start))
 			return res.Chain, nil
 		}
 
+		if res.Status == StatusPartial {
+			log.Warn("dont yet handle partial responses")
+			continue
+		}
+
 		err = bs.processStatus(req, res)
 		if err != nil {
 			log.Warnf("BlockSync peer %s response was an error: %s", p.String(), err)
@@ -241,16 +247,18 @@ func (bs *BlockSync) sendRequestToPeer(ctx context.Context, p peer.ID, req *Bloc
 		bs.RemovePeer(p)
 		return nil, xerrors.Errorf("failed to open stream to peer: %w", err)
 	}
-	s.SetDeadline(time.Now().Add(10 * time.Second))
-	defer s.SetDeadline(time.Time{})
+	s.SetWriteDeadline(time.Now().Add(5 * time.Second))
 
 	if err := cborutil.WriteCborRPC(s, req); err != nil {
+		s.SetWriteDeadline(time.Time{})
 		bs.syncPeers.logFailure(p, time.Since(start))
 		return nil, err
 	}
+	s.SetWriteDeadline(time.Time{})
 
 	var res BlockSyncResponse
-	if err := cborutil.ReadCborRPC(bufio.NewReader(s), &res); err != nil {
+	r := incrt.New(s, 50<<10, 5*time.Second)
+	if err := cborutil.ReadCborRPC(bufio.NewReader(r), &res); err != nil {
 		bs.syncPeers.logFailure(p, time.Since(start))
 		return nil, err
 	}

chain/gen/utils.go

@@ -254,7 +254,7 @@ func SetupStorageMarketActor(bs bstore.Blockstore, sroot cid.Cid, deals []actors
 		return cid.Undef, xerrors.Errorf("set storage market actor: %w", err)
 	}
 
-	return state.Flush()
+	return state.Flush(context.TODO())
 }
 
 type GenMinerCfg struct {
@@ -563,7 +563,7 @@ func MakeGenesisBlock(bs bstore.Blockstore, sys *types.VMSyscalls, balances map[
 		return nil, xerrors.Errorf("make initial state tree failed: %w", err)
 	}
 
-	stateroot, err := state.Flush()
+	stateroot, err := state.Flush(ctx)
 	if err != nil {
 		return nil, xerrors.Errorf("flush state tree failed: %w", err)
 	}
@@ -673,5 +673,5 @@ func AdjustInitActorStartID(ctx context.Context, bs blockstore.Blockstore, state
 		return cid.Undef, err
 	}
 
-	return tree.Flush()
+	return tree.Flush(ctx)
 }

chain/state/statetree.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ipfs/go-cid"
 	hamt "github.com/ipfs/go-hamt-ipld"
 	logging "github.com/ipfs/go-log/v2"
+	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
@@ -116,23 +117,29 @@ func (st *StateTree) GetActor(addr address.Address) (*types.Actor, error) {
 	return &act, nil
 }
 
-func (st *StateTree) Flush() (cid.Cid, error) {
+func (st *StateTree) Flush(ctx context.Context) (cid.Cid, error) {
+	ctx, span := trace.StartSpan(ctx, "stateTree.Flush")
+	defer span.End()
+
 	for addr, act := range st.actorcache {
-		if err := st.root.Set(context.TODO(), string(addr.Bytes()), act); err != nil {
+		if err := st.root.Set(ctx, string(addr.Bytes()), act); err != nil {
 			return cid.Undef, err
 		}
 	}
 	st.actorcache = make(map[address.Address]*types.Actor)
 
-	if err := st.root.Flush(context.TODO()); err != nil {
+	if err := st.root.Flush(ctx); err != nil {
 		return cid.Undef, err
 	}
 
-	return st.Store.Put(context.TODO(), st.root)
+	return st.Store.Put(ctx, st.root)
 }
 
-func (st *StateTree) Snapshot() error {
-	ss, err := st.Flush()
+func (st *StateTree) Snapshot(ctx context.Context) error {
+	ctx, span := trace.StartSpan(ctx, "stateTree.SnapShot")
+	defer span.End()
+
+	ss, err := st.Flush(ctx)
 	if err != nil {
 		return err
 	}

chain/state/statetree_test.go

@@ -1,6 +1,7 @@
 package state
 
 import (
+	"context"
 	"testing"
 
 	address "github.com/filecoin-project/go-address"
@@ -60,7 +61,7 @@ func BenchmarkStateTreeSetFlush(b *testing.B) {
 		if err != nil {
 			b.Fatal(err)
 		}
-		if _, err := st.Flush(); err != nil {
+		if _, err := st.Flush(context.TODO()); err != nil {
 			b.Fatal(err)
 		}
 	}
@@ -88,7 +89,7 @@ func BenchmarkStateTree10kGetActor(b *testing.B) {
 		}
 	}
 
-	if _, err := st.Flush(); err != nil {
+	if _, err := st.Flush(context.TODO()); err != nil {
 		b.Fatal(err)
 	}
 

chain/stmgr/call.go

@@ -8,13 +8,14 @@ import (
 	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/chain/vm"
 )
 
-func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate cid.Cid, r vm.Rand, bheight uint64) (*types.MessageReceipt, error) {
+func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate cid.Cid, r vm.Rand, bheight uint64) (*api.MethodCall, error) {
 	ctx, span := trace.StartSpan(ctx, "statemanager.CallRaw")
 	defer span.End()
 
@@ -54,14 +55,19 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate
 		return nil, xerrors.Errorf("apply message failed: %w", err)
 	}
 
+	var errs string
 	if ret.ActorErr != nil {
+		errs = ret.ActorErr.Error()
 		log.Warnf("chain call failed: %s", ret.ActorErr)
 	}
-	return &ret.MessageReceipt, nil
+	return &api.MethodCall{
+		MessageReceipt: ret.MessageReceipt,
+		Error:          errs,
+	}, nil
 
 }
 
-func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error) {
+func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.MethodCall, error) {
 	if ts == nil {
 		ts = sm.cs.GetHeaviestTipSet()
 	}

chain/stmgr/forks.go

@@ -91,7 +91,7 @@ func fixBlizzardAMTBug(ctx context.Context, sm *StateManager, pstate cid.Cid) (c
 		}
 	}
 
-	return st.Flush()
+	return st.Flush(ctx)
 }
 
 func fixMiner(ctx context.Context, cst *hamt.CborIpldStore, bs blockstore.Blockstore, mscid cid.Cid) (cid.Cid, error) {

chain/stmgr/stmgr.go

@@ -607,3 +607,32 @@ func (sm *StateManager) MarketBalance(ctx context.Context, addr address.Address,
 
 	return b[0], nil
 }
+
+func (sm *StateManager) ValidateChain(ctx context.Context, ts *types.TipSet) error {
+	tschain := []*types.TipSet{ts}
+	for ts.Height() != 0 {
+		next, err := sm.cs.LoadTipSet(ts.Parents())
+		if err != nil {
+			return err
+		}
+
+		tschain = append(tschain, next)
+		ts = next
+	}
+
+	lastState := tschain[len(tschain)-1].ParentState()
+	for i := len(tschain) - 1; i >= 0; i-- {
+		cur := tschain[i]
+		log.Infof("computing state (height: %d, ts=%s)", cur.Height(), cur.Cids())
+		if cur.ParentState() != lastState {
+			return xerrors.Errorf("tipset chain had state mismatch at height %d", cur.Height())
+		}
+		st, _, err := sm.TipSetState(ctx, cur)
+		if err != nil {
+			return err
+		}
+		lastState = st
+	}
+
+	return nil
+}

chain/store/store.go

@@ -1,10 +1,12 @@
 package store
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/binary"
 	"encoding/json"
+	"io"
 	"sync"
 
 	"github.com/filecoin-project/go-address"
@@ -19,11 +21,16 @@ import (
 
 	lru "github.com/hashicorp/golang-lru"
 	block "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-blockservice"
+	car "github.com/ipfs/go-car"
 	"github.com/ipfs/go-cid"
 	dstore "github.com/ipfs/go-datastore"
 	hamt "github.com/ipfs/go-hamt-ipld"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	bstore "github.com/ipfs/go-ipfs-blockstore"
+	format "github.com/ipfs/go-ipld-format"
 	logging "github.com/ipfs/go-log/v2"
+	dag "github.com/ipfs/go-merkledag"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	pubsub "github.com/whyrusleeping/pubsub"
 	"golang.org/x/xerrors"
@@ -917,6 +924,84 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h uint64, ts *types
 	}
 }
 
+func recurseLinks(bs blockstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) {
+	data, err := bs.Get(root)
+	if err != nil {
+		return nil, err
+	}
+
+	top, err := cbg.ScanForLinks(bytes.NewReader(data.RawData()))
+	if err != nil {
+		return nil, err
+	}
+
+	in = append(in, top...)
+	for _, c := range top {
+		var err error
+		in, err = recurseLinks(bs, c, in)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return in, nil
+}
+
+func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer) error {
+	if ts == nil {
+		ts = cs.GetHeaviestTipSet()
+	}
+	bsrv := blockservice.New(cs.bs, nil)
+	dserv := dag.NewDAGService(bsrv)
+	return car.WriteCarWithWalker(ctx, dserv, ts.Cids(), w, func(nd format.Node) ([]*format.Link, error) {
+		var b types.BlockHeader
+		if err := b.UnmarshalCBOR(bytes.NewBuffer(nd.RawData())); err != nil {
+			return nil, err
+		}
+
+		var out []*format.Link
+		for _, p := range b.Parents {
+			out = append(out, &format.Link{Cid: p})
+		}
+
+		cids, err := recurseLinks(cs.bs, b.Messages, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, c := range cids {
+			out = append(out, &format.Link{Cid: c})
+		}
+
+		if b.Height == 0 {
+			cids, err := recurseLinks(cs.bs, b.ParentStateRoot, nil)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, c := range cids {
+				out = append(out, &format.Link{Cid: c})
+			}
+		}
+
+		return out, nil
+	})
+}
+
+func (cs *ChainStore) Import(r io.Reader) (*types.TipSet, error) {
+	header, err := car.LoadCar(cs.Blockstore(), r)
+	if err != nil {
+		return nil, xerrors.Errorf("loadcar failed: %w", err)
+	}
+
+	root, err := cs.LoadTipSet(types.NewTipSetKey(header.Roots...))
+	if err != nil {
+		return nil, xerrors.Errorf("failed to load root tipset from chainfile: %w", err)
+	}
+
+	return root, nil
+}
+
 type chainRand struct {
 	cs   *ChainStore
 	blks []cid.Cid

chain/sync_test.go

@@ -73,7 +73,7 @@ func prepSyncTest(t testing.TB, h int) *syncTestUtil {
 
 	g, err := gen.NewGenerator()
 	if err != nil {
-		t.Fatal(err)
+		t.Fatalf("%+v", err)
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())

chain/types/blockheader.go

@@ -2,7 +2,6 @@ package types
 
 import (
 	"bytes"
-	"context"
 	"math/big"
 
 	"github.com/filecoin-project/go-sectorbuilder"
@@ -11,7 +10,6 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/minio/sha256-simd"
 	"github.com/multiformats/go-multihash"
-	"go.opencensus.io/trace"
 	xerrors "golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
@@ -116,22 +114,6 @@ func (blk *BlockHeader) SigningBytes() ([]byte, error) {
 	return blkcopy.Serialize()
 }
 
-func (blk *BlockHeader) CheckBlockSignature(ctx context.Context, worker address.Address) error {
-	_, span := trace.StartSpan(ctx, "checkBlockSignature")
-	defer span.End()
-
-	if blk.BlockSig == nil {
-		return xerrors.New("block signature not present")
-	}
-
-	sigb, err := blk.SigningBytes()
-	if err != nil {
-		return xerrors.Errorf("failed to get block signing bytes: %w", err)
-	}
-
-	return blk.BlockSig.Verify(worker, sigb)
-}
-
 type MsgMeta struct {
 	BlsMessages   cid.Cid
 	SecpkMessages cid.Cid

chain/types/blockheader_cgo.go

@@ -0,0 +1,27 @@
+//+build cgo
+
+package types
+
+import (
+	"context"
+
+	"github.com/filecoin-project/go-address"
+	"go.opencensus.io/trace"
+	"golang.org/x/xerrors"
+)
+
+func (blk *BlockHeader) CheckBlockSignature(ctx context.Context, worker address.Address) error {
+	_, span := trace.StartSpan(ctx, "checkBlockSignature")
+	defer span.End()
+
+	if blk.BlockSig == nil {
+		return xerrors.New("block signature not present")
+	}
+
+	sigb, err := blk.SigningBytes()
+	if err != nil {
+		return xerrors.Errorf("failed to get block signing bytes: %w", err)
+	}
+
+	return blk.BlockSig.Verify(worker, sigb)
+}

chain/validation/state.go

@@ -59,7 +59,7 @@ func NewState() *StateWrapper {
 	if err != nil {
 		panic(err) // Never returns error, the error return should be removed.
 	}
-	root, err := treeImpl.Flush()
+	root, err := treeImpl.Flush(context.TODO())
 	if err != nil {
 		panic(err)
 	}
@@ -208,7 +208,7 @@ func (s *StateWrapper) Signer() *keyStore {
 
 // Flushes a state tree to storage and sets this state's root to that tree's root CID.
 func (s *StateWrapper) flush(tree *state.StateTree) (err error) {
-	s.stateRoot, err = tree.Flush()
+	s.stateRoot, err = tree.Flush(context.TODO())
 	return
 }
 

chain/vm/invoker.go

@@ -161,6 +161,10 @@ func DecodeParams(b []byte, out interface{}) error {
 }
 
 func DumpActorState(code cid.Cid, b []byte) (interface{}, error) {
+	if code == actors.AccountCodeCid { // Account code special case
+		return nil, nil
+	}
+
 	i := newInvoker() // TODO: register builtins in init block
 
 	typ, ok := i.builtInState[code]

chain/vm/vm.go

@@ -430,7 +430,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, msg *types.Message) (*ApplyRet,
 	}
 
 	st := vm.cstate
-	if err := st.Snapshot(); err != nil {
+	if err := st.Snapshot(ctx); err != nil {
 		return nil, xerrors.Errorf("snapshot failed: %w", err)
 	}
 
@@ -534,7 +534,7 @@ func (vm *VM) Flush(ctx context.Context) (cid.Cid, error) {
 	from := vm.buf
 	to := vm.buf.Read()
 
-	root, err := vm.cstate.Flush()
+	root, err := vm.cstate.Flush(ctx)
 	if err != nil {
 		return cid.Undef, xerrors.Errorf("flushing vm: %w", err)
 	}

cli/chain.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
@@ -12,6 +13,7 @@ import (
 	"gopkg.in/urfave/cli.v2"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors"
 	types "github.com/filecoin-project/lotus/chain/types"
 )
 
@@ -26,6 +28,8 @@ var chainCmd = &cli.Command{
 		chainSetHeadCmd,
 		chainListCmd,
 		chainGetCmd,
+		chainExportCmd,
+		slashConsensusFault,
 	},
 }
 
@@ -172,6 +176,10 @@ var chainGetMsgCmd = &cli.Command{
 	Name:  "getmessage",
 	Usage: "Get and print a message by its cid",
 	Action: func(cctx *cli.Context) error {
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must pass a cid of a message to get")
+		}
+
 		api, closer, err := GetFullNodeAPI(cctx)
 		if err != nil {
 			return err
@@ -385,3 +393,112 @@ func printTipSet(format string, ts *types.TipSet) {
 
 	fmt.Println(format)
 }
+
+var chainExportCmd = &cli.Command{
+	Name:  "export",
+	Usage: "export chain to a car file",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name: "tipset",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must specify filename to export chain to")
+		}
+
+		fi, err := os.Create(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+		defer fi.Close()
+
+		ts, err := loadTipSet(ctx, cctx, api)
+		if err != nil {
+			return err
+		}
+
+		stream, err := api.ChainExport(ctx, ts)
+		if err != nil {
+			return err
+		}
+
+		for b := range stream {
+			_, err := fi.Write(b)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	},
+}
+
+var slashConsensusFault = &cli.Command{
+	Name:  "slash-consensus",
+	Usage: "Report consensus fault",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		c1, err := cid.Parse(cctx.Args().Get(0))
+		if err != nil {
+			return xerrors.Errorf("parsing cid 1: %w", err)
+		}
+
+		b1, err := api.ChainGetBlock(ctx, c1)
+		if err != nil {
+			return xerrors.Errorf("getting block 1: %w", err)
+		}
+
+		c2, err := cid.Parse(cctx.Args().Get(0))
+		if err != nil {
+			return xerrors.Errorf("parsing cid 2: %w", err)
+		}
+
+		b2, err := api.ChainGetBlock(ctx, c2)
+		if err != nil {
+			return xerrors.Errorf("getting block 2: %w", err)
+		}
+
+		def, err := api.WalletDefaultAddress(ctx)
+		if err != nil {
+			return err
+		}
+
+		params, err := actors.SerializeParams(&actors.ArbitrateConsensusFaultParams{
+			Block1: b1,
+			Block2: b2,
+		})
+
+		msg := &types.Message{
+			To:       actors.StoragePowerAddress,
+			From:     def,
+			Value:    types.NewInt(0),
+			GasPrice: types.NewInt(1),
+			GasLimit: types.NewInt(10000000),
+			Method:   actors.SPAMethods.ArbitrateConsensusFault,
+			Params:   params,
+		}
+
+		smsg, err := api.MpoolPushMessage(ctx, msg)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(smsg.Cid())
+
+		return nil
+	},
+}

cli/cmd.go

@@ -202,6 +202,7 @@ var Commands = []*cli.Command{
 	clientCmd,
 	fetchParamCmd,
 	mpoolCmd,
+	multisigCmd,
 	netCmd,
 	paychCmd,
 	sendCmd,

cli/mpool.go

@@ -124,7 +124,8 @@ var mpoolStat = &cli.Command{
 		for a, bkt := range buckets {
 			act, err := api.StateGetActor(ctx, a, ts)
 			if err != nil {
-				return err
+				fmt.Printf("%s, err: %s\n", a, err)
+				continue
 			}
 
 			cur := act.Nonce

cli/multisig.go

@@ -0,0 +1,400 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"strconv"
+	"text/tabwriter"
+
+	"github.com/filecoin-project/go-address"
+	actors "github.com/filecoin-project/lotus/chain/actors"
+	types "github.com/filecoin-project/lotus/chain/types"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"gopkg.in/urfave/cli.v2"
+)
+
+var multisigCmd = &cli.Command{
+	Name:  "msig",
+	Usage: "Interact with a multisig wallet",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "source",
+			Usage: "specify the account to send propose from",
+		},
+	},
+	Subcommands: []*cli.Command{
+		msigCreateCmd,
+		msigInspectCmd,
+		msigProposeCmd,
+		msigApproveCmd,
+	},
+}
+
+var msigCreateCmd = &cli.Command{
+	Name:  "create",
+	Usage: "Create a new multisig wallet",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name: "required",
+		},
+		&cli.StringFlag{
+			Name:  "value",
+			Usage: "initial funds to give to multisig",
+			Value: "0",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		var addrs []address.Address
+		for _, a := range cctx.Args().Slice() {
+			addr, err := address.NewFromString(a)
+			if err != nil {
+				return err
+			}
+			addrs = append(addrs, addr)
+		}
+
+		// get the address we're going to use to create the multisig (can be one of the above, as long as they have funds)
+		sendAddr, err := api.WalletDefaultAddress(ctx)
+		if err != nil {
+			return err
+		}
+
+		val := cctx.String("value")
+		filval, err := types.ParseFIL(val)
+		if err != nil {
+			return err
+		}
+
+		required := cctx.Uint64("required")
+		if required == 0 {
+			required = uint64(len(addrs))
+		}
+
+		// Set up constructor parameters for multisig
+		msigParams := &actors.MultiSigConstructorParams{
+			Signers:  addrs,
+			Required: required,
+		}
+
+		enc, err := actors.SerializeParams(msigParams)
+		if err != nil {
+			return err
+		}
+
+		// new actors are created by invoking 'exec' on the init actor with the constructor params
+		execParams := &actors.ExecParams{
+			Code:   actors.MultisigCodeCid,
+			Params: enc,
+		}
+
+		enc, err = actors.SerializeParams(execParams)
+		if err != nil {
+			return err
+		}
+
+		// now we create the message to send this with
+		msg := types.Message{
+			To:       actors.InitAddress,
+			From:     sendAddr,
+			Method:   actors.IAMethods.Exec,
+			Params:   enc,
+			GasPrice: types.NewInt(1),
+			GasLimit: types.NewInt(1000000),
+			Value:    types.BigInt(filval),
+		}
+
+		// send the message out to the network
+		smsg, err := api.MpoolPushMessage(ctx, &msg)
+		if err != nil {
+			return err
+		}
+
+		// wait for it to get mined into a block
+		wait, err := api.StateWaitMsg(ctx, smsg.Cid())
+		if err != nil {
+			return err
+		}
+
+		// check it executed successfully
+		if wait.Receipt.ExitCode != 0 {
+			fmt.Println("actor creation failed!")
+			return err
+		}
+
+		// get address of newly created miner
+		msigaddr, err := address.NewFromBytes(wait.Receipt.Return)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("Created new multisig: ", msigaddr.String())
+		// TODO: maybe register this somewhere
+		return nil
+	},
+}
+
+var msigInspectCmd = &cli.Command{
+	Name:  "inspect",
+	Usage: "Inspect a multisig wallet",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		if !cctx.Args().Present() {
+			return fmt.Errorf("must specify address of multisig to inspect")
+		}
+
+		maddr, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return err
+		}
+
+		act, err := api.StateGetActor(ctx, maddr, nil)
+		if err != nil {
+			return err
+		}
+
+		obj, err := api.ChainReadObj(ctx, act.Head)
+		if err != nil {
+			return err
+		}
+
+		var mstate actors.MultiSigActorState
+		if err := mstate.UnmarshalCBOR(bytes.NewReader(obj)); err != nil {
+			return err
+		}
+
+		fmt.Printf("Balance: %sfil\n", types.FIL(act.Balance))
+		fmt.Printf("Threshold: %d / %d\n", mstate.Required, len(mstate.Signers))
+		fmt.Println("Signers:")
+		for _, s := range mstate.Signers {
+			fmt.Printf("\t%s\n", s)
+		}
+		fmt.Println("Transactions: ", len(mstate.Transactions))
+		if len(mstate.Transactions) > 0 {
+			w := tabwriter.NewWriter(os.Stdout, 8, 4, 0, ' ', 0)
+			fmt.Fprintf(w, "ID\tState\tTo\tValue\tMethod\tParams\n")
+			for _, tx := range mstate.Transactions {
+				fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%d\t%x\n", tx.TxID, state(tx), tx.To, types.FIL(tx.Value), tx.Method, tx.Params)
+			}
+			w.Flush()
+		}
+
+		return nil
+	},
+}
+
+func state(tx actors.MTransaction) string {
+	if tx.Complete {
+		return "done"
+	}
+	if tx.Canceled {
+		return "canceled"
+	}
+	return "pending"
+}
+
+var msigProposeCmd = &cli.Command{
+	Name:  "propose",
+	Usage: "Propose a multisig transaction",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		if cctx.Args().Len() < 3 {
+			return fmt.Errorf("must pass multisig address, destination, and value")
+		}
+
+		if cctx.Args().Len() > 3 && cctx.Args().Len() != 5 {
+			return fmt.Errorf("usage: msig propose <msig addr> <desination> <value> [ <method> <params> ]")
+		}
+
+		msig, err := address.NewFromString(cctx.Args().Get(0))
+		if err != nil {
+			return err
+		}
+
+		dest, err := address.NewFromString(cctx.Args().Get(1))
+		if err != nil {
+			return err
+		}
+
+		value, err := types.ParseFIL(cctx.Args().Get(2))
+		if err != nil {
+			return err
+		}
+
+		var method uint64
+		var params []byte
+		if cctx.Args().Len() == 5 {
+			m, err := strconv.ParseUint(cctx.Args().Get(3), 10, 64)
+			if err != nil {
+				return err
+			}
+			method = m
+
+			p, err := hex.DecodeString(cctx.Args().Get(4))
+			if err != nil {
+				return err
+			}
+			params = p
+		}
+
+		enc, err := actors.SerializeParams(&actors.MultiSigProposeParams{
+			To:     dest,
+			Value:  types.BigInt(value),
+			Method: method,
+			Params: params,
+		})
+		if err != nil {
+			return err
+		}
+
+		var from address.Address
+		if cctx.IsSet("source") {
+			f, err := address.NewFromString(cctx.String("source"))
+			if err != nil {
+				return err
+			}
+			from = f
+		} else {
+			defaddr, err := api.WalletDefaultAddress(ctx)
+			if err != nil {
+				return err
+			}
+			from = defaddr
+		}
+
+		msg := &types.Message{
+			To:       msig,
+			From:     from,
+			Value:    types.NewInt(0),
+			Method:   actors.MultiSigMethods.Propose,
+			Params:   enc,
+			GasLimit: types.NewInt(100000),
+			GasPrice: types.NewInt(1),
+		}
+
+		smsg, err := api.MpoolPushMessage(ctx, msg)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("send proposal in message: ", smsg.Cid())
+
+		wait, err := api.StateWaitMsg(ctx, smsg.Cid())
+		if err != nil {
+			return err
+		}
+
+		if wait.Receipt.ExitCode != 0 {
+			return fmt.Errorf("proposal returned exit %d", wait.Receipt.ExitCode)
+		}
+
+		_, v, err := cbg.CborReadHeader(bytes.NewReader(wait.Receipt.Return))
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Transaction ID: %d\n", v)
+
+		return nil
+	},
+}
+
+var msigApproveCmd = &cli.Command{
+	Name:  "approve",
+	Usage: "Approve a multisig transaction",
+	Flags: []cli.Flag{},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := ReqContext(cctx)
+
+		if cctx.Args().Len() < 2 {
+			return fmt.Errorf("must pass multisig address and transaction ID")
+		}
+
+		msig, err := address.NewFromString(cctx.Args().Get(0))
+		if err != nil {
+			return err
+		}
+
+		txid, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64)
+		if err != nil {
+			return err
+		}
+
+		enc, err := actors.SerializeParams(&actors.MultiSigTxID{
+			TxID: txid,
+		})
+		if err != nil {
+			return err
+		}
+
+		var from address.Address
+		if cctx.IsSet("source") {
+			f, err := address.NewFromString(cctx.String("source"))
+			if err != nil {
+				return err
+			}
+			from = f
+		} else {
+			defaddr, err := api.WalletDefaultAddress(ctx)
+			if err != nil {
+				return err
+			}
+			from = defaddr
+		}
+
+		msg := &types.Message{
+			To:       msig,
+			From:     from,
+			Value:    types.NewInt(0),
+			Method:   actors.MultiSigMethods.Approve,
+			Params:   enc,
+			GasLimit: types.NewInt(100000),
+			GasPrice: types.NewInt(1),
+		}
+
+		smsg, err := api.MpoolPushMessage(ctx, msg)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("sent approval in message: ", smsg.Cid())
+
+		wait, err := api.StateWaitMsg(ctx, smsg.Cid())
+		if err != nil {
+			return err
+		}
+
+		if wait.Receipt.ExitCode != 0 {
+			return fmt.Errorf("approve returned exit %d", wait.Receipt.ExitCode)
+		}
+
+		return nil
+	},
+}

cli/send.go

@@ -15,7 +15,7 @@ var sendCmd = &cli.Command{
 	Flags: []cli.Flag{
 		&cli.StringFlag{
 			Name:  "source",
-			Usage: "optinally specifiy the account to send funds from",
+			Usage: "optionally specify the account to send funds from",
 		},
 	},
 	Action: func(cctx *cli.Context) error {

cli/state.go

@@ -1,18 +1,24 @@
 package cli
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"reflect"
+	"strconv"
 	"strings"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/api"
+	actors "github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/miner"
+	"github.com/libp2p/go-libp2p-core/peer"
 	"golang.org/x/xerrors"
 
 	"github.com/ipfs/go-cid"
+	cbg "github.com/whyrusleeping/cbor-gen"
 	"gopkg.in/urfave/cli.v2"
 )
 
@@ -39,6 +45,7 @@ var stateCmd = &cli.Command{
 		stateReadStateCmd,
 		stateListMessagesCmd,
 		stateComputeStateCmd,
+		stateCallCmd,
 	},
 }
 
@@ -528,6 +535,10 @@ var stateListMessagesCmd = &cli.Command{
 			Name:  "toheight",
 			Usage: "don't look before given block height",
 		},
+		&cli.BoolFlag{
+			Name:  "cids",
+			Usage: "print message CIDs instead of messages",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := GetFullNodeAPI(cctx)
@@ -568,6 +579,11 @@ var stateListMessagesCmd = &cli.Command{
 		}
 
 		for _, c := range msgs {
+			if cctx.Bool("cids") {
+				fmt.Println(c.String())
+				continue
+			}
+
 			m, err := api.ChainGetMessage(ctx, c)
 			if err != nil {
 				return err
@@ -648,3 +664,209 @@ var stateComputeStateCmd = &cli.Command{
 		return nil
 	},
 }
+
+var stateCallCmd = &cli.Command{
+	Name:  "call",
+	Usage: "Invoke a method on an actor locally",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "from",
+			Usage: "",
+			Value: actors.NetworkAddress.String(),
+		},
+		&cli.StringFlag{
+			Name:  "value",
+			Usage: "specify value field for invocation",
+			Value: "0",
+		},
+		&cli.StringFlag{
+			Name:  "ret",
+			Usage: "specify how to parse output (auto, raw, addr, big)",
+			Value: "auto",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() < 2 {
+			return fmt.Errorf("must specify at least actor and method to invoke")
+		}
+
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		toa, err := address.NewFromString(cctx.Args().First())
+		if err != nil {
+			return fmt.Errorf("given 'to' address %q was invalid: %w", cctx.Args().First(), err)
+		}
+
+		froma, err := address.NewFromString(cctx.String("from"))
+		if err != nil {
+			return fmt.Errorf("given 'from' address %q was invalid: %w", cctx.String("from"), err)
+		}
+
+		ts, err := loadTipSet(ctx, cctx, api)
+		if err != nil {
+			return err
+		}
+
+		method, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64)
+		if err != nil {
+			return fmt.Errorf("must pass method as a number")
+		}
+
+		value, err := types.ParseFIL(cctx.String("value"))
+		if err != nil {
+			return fmt.Errorf("failed to parse 'value': %s", err)
+		}
+
+		act, err := api.StateGetActor(ctx, toa, ts)
+		if err != nil {
+			return fmt.Errorf("failed to lookup target actor: %s", err)
+		}
+
+		params, err := parseParamsForMethod(act.Code, method, cctx.Args().Slice()[2:])
+		if err != nil {
+			return fmt.Errorf("failed to parse params: %s", err)
+		}
+
+		ret, err := api.StateCall(ctx, &types.Message{
+			From:     froma,
+			To:       toa,
+			Value:    types.BigInt(value),
+			GasLimit: types.NewInt(10000000000),
+			GasPrice: types.NewInt(0),
+			Method:   method,
+			Params:   params,
+		}, ts)
+		if err != nil {
+			return fmt.Errorf("state call failed: %s", err)
+		}
+
+		if ret.ExitCode != 0 {
+			return fmt.Errorf("invocation failed (exit: %d): %s", ret.ExitCode, ret.Error)
+		}
+
+		s, err := formatOutput(cctx.String("ret"), ret.Return)
+		if err != nil {
+			return fmt.Errorf("failed to format output: %s", err)
+		}
+
+		fmt.Printf("return: %s\n", s)
+
+		return nil
+	},
+}
+
+func formatOutput(t string, val []byte) (string, error) {
+	switch t {
+	case "raw", "hex":
+		return fmt.Sprintf("%x", val), nil
+	case "address", "addr", "a":
+		a, err := address.NewFromBytes(val)
+		if err != nil {
+			return "", err
+		}
+		return a.String(), nil
+	case "big", "int", "bigint":
+		bi := types.BigFromBytes(val)
+		return bi.String(), nil
+	case "fil":
+		bi := types.FIL(types.BigFromBytes(val))
+		return bi.String(), nil
+	case "pid", "peerid", "peer":
+		pid, err := peer.IDFromBytes(val)
+		if err != nil {
+			return "", err
+		}
+
+		return pid.Pretty(), nil
+	case "auto":
+		if len(val) == 0 {
+			return "", nil
+		}
+
+		a, err := address.NewFromBytes(val)
+		if err == nil {
+			return "address: " + a.String(), nil
+		}
+
+		pid, err := peer.IDFromBytes(val)
+		if err == nil {
+			return "peerID: " + pid.Pretty(), nil
+		}
+
+		bi := types.BigFromBytes(val)
+		return "bigint: " + bi.String(), nil
+	default:
+		return "", fmt.Errorf("unrecognized output type: %q", t)
+	}
+}
+
+func parseParamsForMethod(act cid.Cid, method uint64, args []string) ([]byte, error) {
+	if len(args) == 0 {
+		return nil, nil
+	}
+
+	var f interface{}
+	switch act {
+	case actors.StorageMarketCodeCid:
+		f = actors.StorageMarketActor{}.Exports()[method]
+	case actors.StorageMinerCodeCid:
+		f = actors.StorageMinerActor{}.Exports()[method]
+	case actors.StoragePowerCodeCid:
+		f = actors.StoragePowerActor{}.Exports()[method]
+	case actors.MultisigCodeCid:
+		f = actors.MultiSigActor{}.Exports()[method]
+	case actors.PaymentChannelCodeCid:
+		f = actors.PaymentChannelActor{}.Exports()[method]
+	default:
+		return nil, fmt.Errorf("the lazy devs didnt add support for that actor to this call yet")
+	}
+
+	rf := reflect.TypeOf(f)
+	if rf.NumIn() != 3 {
+		return nil, fmt.Errorf("expected referenced method to have three arguments")
+	}
+
+	paramObj := rf.In(2).Elem()
+	if paramObj.NumField() != len(args) {
+		return nil, fmt.Errorf("not enough arguments given to call that method (expecting %d)", paramObj.NumField())
+	}
+
+	p := reflect.New(paramObj)
+	for i := 0; i < len(args); i++ {
+		switch paramObj.Field(i).Type {
+		case reflect.TypeOf(address.Address{}):
+			a, err := address.NewFromString(args[i])
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse address: %s", err)
+			}
+			p.Elem().Field(i).Set(reflect.ValueOf(a))
+		case reflect.TypeOf(uint64(0)):
+			val, err := strconv.ParseUint(args[i], 10, 64)
+			if err != nil {
+				return nil, err
+			}
+			p.Elem().Field(i).Set(reflect.ValueOf(val))
+		case reflect.TypeOf(peer.ID("")):
+			pid, err := peer.IDB58Decode(args[i])
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse peer ID: %s", err)
+			}
+			p.Elem().Field(i).Set(reflect.ValueOf(pid))
+		default:
+			return nil, fmt.Errorf("unsupported type for call (TODO): %s", paramObj.Field(i).Type)
+		}
+	}
+
+	m := p.Interface().(cbg.CBORMarshaler)
+	buf := new(bytes.Buffer)
+	if err := m.MarshalCBOR(buf); err != nil {
+		return nil, fmt.Errorf("failed to marshal param object: %s", err)
+	}
+	return buf.Bytes(), nil
+}

cmd/lotus-chainwatch/storage.go

@@ -2,16 +2,18 @@ package main
 
 import (
 	"database/sql"
+	"encoding/hex"
 	"fmt"
-	"github.com/filecoin-project/lotus/api"
-	"golang.org/x/xerrors"
 	"sync"
 	"time"
 
+	"github.com/filecoin-project/go-address"
 	"github.com/ipfs/go-cid"
 	_ "github.com/lib/pq"
+	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
@@ -75,13 +77,22 @@ create table if not exists blocks
 	eprof bytea,
 	prand bytea,
 	ep0partial bytea,
-	ep0sector bigint not null,
-	ep0challangei bigint not null
+	ep0sector numeric not null,
+	ep0challangei numeric not null
 );
 
 create unique index if not exists block_cid_uindex
 	on blocks (cid);
 
+create materialized view if not exists state_heights
+    as select distinct height, parentstateroot from blocks;
+
+create unique index if not exists state_heights_uindex
+	on state_heights (height);
+
+create index if not exists state_heights_height_index
+	on state_heights (parentstateroot);
+
 create table if not exists id_address_map
 (
 	id text not null,
@@ -117,6 +128,38 @@ create index if not exists id_address_map_address_index
 create index if not exists id_address_map_id_index
 	on id_address_map (id);
 
+create or replace function actor_tips(epoch bigint)
+    returns table (id text,
+                    code text,
+                    head text,
+                    nonce int,
+                    balance text,
+                    stateroot text,
+                    height bigint,
+                    parentstateroot text) as
+$body$
+    select distinct on (id) * from actors
+        inner join state_heights sh on sh.parentstateroot = stateroot
+        where height < $1
+		order by id, height desc;
+$body$ language sql;
+
+create table if not exists actor_states
+(
+	head text not null,
+	code text not null,
+	state json not null
+);
+
+create unique index if not exists actor_states_head_code_uindex
+	on actor_states (head, code);
+
+create index if not exists actor_states_head_index
+	on actor_states (head);
+
+create index if not exists actor_states_code_head_index
+	on actor_states (head, code);
+
 create table if not exists messages
 (
 	cid text not null
@@ -124,11 +167,11 @@ create table if not exists messages
 			primary key,
 	"from" text not null,
 	"to" text not null,
-	nonce int not null,
+	nonce bigint not null,
 	value text not null,
-	gasprice int not null,
-	gaslimit int not null,
-	method int,
+	gasprice bigint not null,
+	gaslimit bigint not null,
+	method bigint,
 	params bytea
 );
 
@@ -184,7 +227,7 @@ create table if not exists miner_heads
 	worker text not null,
 	peerid text not null,
 	sectorsize bigint not null,
-	power bigint not null,
+	power decimal not null,
 	active bool,
 	ppe bigint not null,
 	slashed_at bigint not null,
@@ -192,6 +235,73 @@ create table if not exists miner_heads
 		primary key (head, addr)
 );
 
+create or replace function miner_tips(epoch bigint)
+    returns table (head text,
+                   addr text,
+                   stateroot text,
+                   sectorset text,
+                   setsize decimal,
+                   provingset text,
+                   provingsize decimal,
+                   owner text,
+                   worker text,
+                   peerid text,
+                   sectorsize bigint,
+                   power decimal,
+                   active bool,
+                   ppe bigint,
+                   slashed_at bigint,
+                   height bigint,
+                   parentstateroot text) as
+    $body$
+        select distinct on (addr) * from miner_heads
+            inner join state_heights sh on sh.parentstateroot = stateroot
+            where height < $1
+            order by addr, height desc;
+    $body$ language sql;
+
+create table if not exists deals
+(
+	id int not null,
+	pieceRef text not null,
+	pieceSize bigint not null,
+	client text not null,
+	provider text not null,
+	expiration decimal not null,
+	duration decimal not null,
+	epochPrice decimal not null,
+	collateral decimal not null,
+	constraint deals_pk
+		primary key (id)
+);
+
+create index if not exists deals_client_index
+	on deals (client);
+
+create unique index if not exists deals_id_uindex
+	on deals (id);
+
+create index if not exists deals_pieceRef_index
+	on deals (pieceRef);
+
+create index if not exists deals_provider_index
+	on deals (provider);
+
+create table if not exists deal_activations
+(
+	deal bigint not null
+		constraint deal_activations_deals_id_fk
+			references deals,
+	activation_epoch bigint not null,
+	constraint deal_activations_pk
+		primary key (deal)
+);
+
+create index if not exists deal_activations_activation_epoch_index
+	on deal_activations (activation_epoch);
+
+create unique index if not exists deal_activations_deal_uindex
+	on deal_activations (deal);
 `)
 	if err != nil {
 		return err
@@ -226,17 +336,15 @@ func (st *storage) hasList() map[cid.Cid]struct{} {
 	return out
 }
 
-func (st *storage) storeActors(actors map[address.Address]map[types.Actor]cid.Cid) error {
+func (st *storage) storeActors(actors map[address.Address]map[types.Actor]actorInfo) error {
+	// Basic
 	tx, err := st.db.Begin()
 	if err != nil {
 		return err
 	}
 	if _, err := tx.Exec(`
-
-create temp table a (like actors excluding constraints) on commit drop;
-
-
-`); err != nil {
+		create temp table a (like actors excluding constraints) on commit drop;
+	`); err != nil {
 		return xerrors.Errorf("prep temp: %w", err)
 	}
 
@@ -247,7 +355,7 @@ create temp table a (like actors excluding constraints) on commit drop;
 
 	for addr, acts := range actors {
 		for act, st := range acts {
-			if _, err := stmt.Exec(addr.String(), act.Code.String(), act.Head.String(), act.Nonce, act.Balance.String(), st.String()); err != nil {
+			if _, err := stmt.Exec(addr.String(), act.Code.String(), act.Head.String(), act.Nonce, act.Balance.String(), st.stateroot.String()); err != nil {
 				return err
 			}
 		}
@@ -261,7 +369,47 @@ create temp table a (like actors excluding constraints) on commit drop;
 		return xerrors.Errorf("actor put: %w", err)
 	}
 
-	return tx.Commit()
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	// States
+	tx, err = st.db.Begin()
+	if err != nil {
+		return err
+	}
+	if _, err := tx.Exec(`
+		create temp table a (like actor_states excluding constraints) on commit drop;
+	`); err != nil {
+		return xerrors.Errorf("prep temp: %w", err)
+	}
+
+	stmt, err = tx.Prepare(`copy a (head, code, state) from stdin `)
+	if err != nil {
+		return err
+	}
+
+	for _, acts := range actors {
+		for act, st := range acts {
+			if _, err := stmt.Exec(act.Head.String(), act.Code.String(), st.state); err != nil {
+				return err
+			}
+		}
+	}
+
+	if err := stmt.Close(); err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`insert into actor_states select * from a on conflict do nothing `); err != nil {
+		return xerrors.Errorf("actor put: %w", err)
+	}
+
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
@@ -602,11 +750,8 @@ func (st *storage) storeMpoolInclusions(msgs []api.MpoolUpdate) error {
 	}
 
 	if _, err := tx.Exec(`
-
-create temp table mi (like mpool_messages excluding constraints) on commit drop;
-
-
-`); err != nil {
+		create temp table mi (like mpool_messages excluding constraints) on commit drop;
+	`); err != nil {
 		return xerrors.Errorf("prep temp: %w", err)
 	}
 
@@ -639,6 +784,112 @@ create temp table mi (like mpool_messages excluding constraints) on commit drop;
 	return tx.Commit()
 }
 
+func (st *storage) storeDeals(deals map[string]actors.OnChainDeal) error {
+	tx, err := st.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`
+		create temp table d (like deals excluding constraints) on commit drop;
+	`); err != nil {
+		return xerrors.Errorf("prep temp: %w", err)
+	}
+
+	stmt, err := tx.Prepare(`copy d (id, pieceref, piecesize, client, "provider", expiration, duration, epochprice, collateral) from stdin `)
+	if err != nil {
+		return err
+	}
+
+	var bloat uint64
+
+	for id, deal := range deals {
+		if len(deal.PieceRef) > 40 {
+			bloat += uint64(len(deal.PieceRef))
+			continue
+		}
+		if _, err := stmt.Exec(
+			id,
+			hex.EncodeToString(deal.PieceRef),
+			deal.PieceSize,
+			deal.Client.String(),
+			deal.Provider.String(),
+			fmt.Sprint(deal.ProposalExpiration),
+			fmt.Sprint(deal.Duration),
+			deal.StoragePricePerEpoch.String(),
+			deal.StorageCollateral.String(),
+		); err != nil {
+			return err
+		}
+	}
+	if bloat > 0 {
+		log.Warnf("deal PieceRefs had %d bytes of garbage", bloat)
+	}
+
+	if err := stmt.Close(); err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`insert into deals select * from d on conflict do nothing `); err != nil {
+		return xerrors.Errorf("actor put: %w", err)
+	}
+
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	// Activations
+
+	tx, err = st.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`
+		create temp table d (like deal_activations excluding constraints) on commit drop;
+	`); err != nil {
+		return xerrors.Errorf("prep temp: %w", err)
+	}
+
+	stmt, err = tx.Prepare(`copy d (deal, activation_epoch) from stdin `)
+	if err != nil {
+		return err
+	}
+
+	for id, deal := range deals {
+		if deal.ActivationEpoch == 0 {
+			continue
+		}
+		if _, err := stmt.Exec(
+			id,
+			deal.ActivationEpoch,
+		); err != nil {
+			return err
+		}
+	}
+	if err := stmt.Close(); err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`insert into deal_activations select * from d on conflict do nothing `); err != nil {
+		return xerrors.Errorf("actor put: %w", err)
+	}
+
+	if err := tx.Commit(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (st *storage) refreshViews() error {
+	if _, err := st.db.Exec(`refresh materialized view state_heights`); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (st *storage) close() error {
 	return st.db.Close()
 }

cmd/lotus-chainwatch/sync.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"container/list"
 	"context"
+	"encoding/json"
 	"math"
 	"sync"
 
@@ -60,9 +61,14 @@ type minerInfo struct {
 	psize uint64
 }
 
+type actorInfo struct {
+	stateroot cid.Cid
+	state     string
+}
+
 func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipSet) {
 	addresses := map[address.Address]address.Address{}
-	actors := map[address.Address]map[types.Actor]cid.Cid{}
+	actors := map[address.Address]map[types.Actor]actorInfo{}
 	var alk sync.Mutex
 
 	log.Infof("Getting synced block list")
@@ -150,12 +156,26 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 						log.Error(err)
 						return
 					}
+					ast, err := api.StateReadState(ctx, act, ts)
+					if err != nil {
+						log.Error(err)
+						return
+					}
+					state, err := json.Marshal(ast.State)
+					if err != nil {
+						log.Error(err)
+						return
+					}
+
 					alk.Lock()
 					_, ok := actors[addr]
 					if !ok {
-						actors[addr] = map[types.Actor]cid.Cid{}
+						actors[addr] = map[types.Actor]actorInfo{}
+					}
+					actors[addr][*act] = actorInfo{
+						stateroot: bh.ParentStateRoot,
+						state:     string(state),
 					}
-					actors[addr][*act] = bh.ParentStateRoot
 					addresses[addr] = address.Undef
 					alk.Unlock()
 				})
@@ -181,13 +201,26 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 					log.Error(err)
 					return
 				}
+				ast, err := api.StateReadState(ctx, &act, ts)
+				if err != nil {
+					log.Error(err)
+					return
+				}
+				state, err := json.Marshal(ast.State)
+				if err != nil {
+					log.Error(err)
+					return
+				}
 
 				alk.Lock()
 				_, ok := actors[addr]
 				if !ok {
-					actors[addr] = map[types.Actor]cid.Cid{}
+					actors[addr] = map[types.Actor]actorInfo{}
+				}
+				actors[addr][act] = actorInfo{
+					stateroot: bh.ParentStateRoot,
+					state:     string(state),
 				}
-				actors[addr][act] = bh.ParentStateRoot
 				addresses[addr] = address.Undef
 				alk.Unlock()
 			}
@@ -228,7 +261,7 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 				miners[minerKey{
 					addr:      addr,
 					act:       actor,
-					stateroot: c,
+					stateroot: c.stateroot,
 				}] = &minerInfo{}
 			}
 		}
@@ -322,6 +355,29 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 		log.Infof("Sync stage done")
 	}
 
+	log.Infof("Get deals")
+
+	// TODO: incremental, gather expired
+	deals, err := api.StateMarketDeals(ctx, ts)
+	if err != nil {
+		log.Error(err)
+		return
+	}
+
+	log.Infof("Store deals")
+
+	if err := st.storeDeals(deals); err != nil {
+		log.Error(err)
+		return
+	}
+
+	log.Infof("Refresh views")
+
+	if err := st.refreshViews(); err != nil {
+		log.Error(err)
+		return
+	}
+
 	log.Infof("Sync done")
 }
 

cmd/lotus-health/main.go

@@ -2,12 +2,17 @@ package main
 
 import (
 	"context"
+	"errors"
 	"os"
+	"os/signal"
+	"syscall"
 	"time"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/filecoin-project/lotus/lib/jsonrpc"
 	cid "github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log"
 	"gopkg.in/urfave/cli.v2"
@@ -41,7 +46,7 @@ func main() {
 	}
 
 	if err := app.Run(os.Args); err != nil {
-		log.Warn(err)
+		log.Fatal(err)
 		return
 	}
 }
@@ -64,61 +69,72 @@ var watchHeadCmd = &cli.Command{
 			Value: "lotus-daemon.service",
 			Usage: "systemd unit name to restart on health check failure",
 		},
+		&cli.IntFlag{
+			Name:  "api-timeout",
+			Value: build.BlockDelay,
+			Usage: "timeout between API retries",
+		},
+		&cli.IntFlag{
+			Name:  "api-retries",
+			Value: 8,
+			Usage: "number of API retry attemps",
+		},
 	},
 	Action: func(c *cli.Context) error {
+		var headCheckWindow CidWindow
 		threshold := c.Int("threshold")
 		interval := time.Duration(c.Int("interval")) * time.Second
 		name := c.String("systemd-unit")
+		apiRetries := c.Int("api-retries")
+		apiTimeout := time.Duration(c.Int("api-timeout")) * time.Second
 
-		var headCheckWindow CidWindow
+		nCh := make(chan interface{}, 1)
+		sCh := make(chan os.Signal, 1)
+		signal.Notify(sCh, os.Interrupt, syscall.SIGTERM)
 
-		api, closer, err := lcli.GetFullNodeAPI(c)
+		api, closer, err := getFullNodeAPI(c, apiRetries, apiTimeout)
 		if err != nil {
 			return err
 		}
 		defer closer()
 		ctx := lcli.ReqContext(c)
 
-		if err := WaitForSyncComplete(ctx, api); err != nil {
-			log.Fatal(err)
-		}
-
-		ch := make(chan CidWindow, 1)
-		aCh := make(chan interface{}, 1)
-
 		go func() {
 			for {
-				headCheckWindow, err = updateWindow(ctx, api, headCheckWindow, threshold, ch)
-				if err != nil {
-					log.Fatal(err)
+				log.Info("Waiting for sync to complete")
+				if err := waitForSyncComplete(ctx, api, apiRetries, apiTimeout); err != nil {
+					nCh <- err
+					return
 				}
+				headCheckWindow, err = updateWindow(ctx, api, headCheckWindow, threshold, apiRetries, apiTimeout)
+				if err != nil {
+					log.Warn("Failed to connect to API. Restarting systemd service")
+					nCh <- nil
+					return
+				}
+				ok := checkWindow(headCheckWindow, threshold)
+				if !ok {
+					log.Warn("Chain head has not updated. Restarting systemd service")
+					nCh <- nil
+					break
+				}
+				log.Info("Chain head is healthy")
 				time.Sleep(interval)
 			}
+			return
 		}()
 
-		go func() {
-			result, err := alertHandler(name, aCh)
-			if err != nil {
-				log.Fatal(err)
-			}
-			if result != "done" {
-				log.Fatal("systemd unit failed to restart:", result)
-			}
-			log.Info("restarting health agent")
-			// Exit health agent and let supervisor restart health agent
-			// Restarting lotus systemd unit kills api connection
-			os.Exit(130)
-		}()
-
-		for {
-			ok := checkWindow(ch, int(interval))
-			if !ok {
-				log.Warn("chain head has not updated. Restarting systemd service")
-				aCh <- nil
-				break
-			}
-			log.Info("chain head is healthy")
+		restart, err := notifyHandler(name, nCh, sCh)
+		if err != nil {
+			return err
 		}
+		if restart != "done" {
+			return errors.New("Systemd unit failed to restart:" + restart)
+		}
+		log.Info("Restarting health agent")
+		// Exit health agent and let supervisor restart health agent
+		// Restarting lotus systemd unit kills api connection
+		os.Exit(130)
 		return nil
 	},
 }
@@ -128,52 +144,67 @@ var watchHeadCmd = &cli.Command{
  * compares slices of Cids when len is greater or equal to `t` - threshold
  * if all slices are equal, head has not updated and returns false
  */
-func checkWindow(ch chan CidWindow, t int) bool {
-	select {
-	case window := <-ch:
-		var dup int
-		windowLen := len(window)
-		if windowLen >= t {
-		cidWindow:
-			for i, cids := range window {
-				next := windowLen - 1 - i
-				// if array length is different, head is changing
-				if next >= 1 && len(window[next]) != len(window[next-1]) {
+func checkWindow(window CidWindow, t int) bool {
+	var dup int
+	windowLen := len(window)
+	if windowLen >= t {
+	cidWindow:
+		for i := range window {
+			next := windowLen - 1 - i
+			// if array length is different, head is changing
+			if next >= 1 && len(window[next]) != len(window[next-1]) {
+				break cidWindow
+			}
+			// if cids are different, head is changing
+			for j := range window[next] {
+				if next >= 1 && window[next][j] != window[next-1][j] {
 					break cidWindow
 				}
-				// if cids are different, head is changing
-				for j := range cids {
-					if next >= 1 && window[next][j] != window[next-1][j] {
-						break cidWindow
-					}
-				}
-				if i < (t - 1) {
-					dup++
-				}
 			}
-
-			if dup == (t - 1) {
-				return false
+			if i < (t - 1) {
+				dup++
 			}
 		}
-		return true
+
+		if dup == (t - 1) {
+			return false
+		}
 	}
+	return true
 }
 
 /*
- * reads channel of slices of slices of Cids
- * compares Cids when len of window is greater or equal to `t` - threshold
- * if all slices are the equal, head has not updated and returns false
+ * returns a slice of slices of Cids
+ * len of slice <= `t` - threshold
  */
-func updateWindow(ctx context.Context, a api.FullNode, w CidWindow, t int, ch chan CidWindow) (CidWindow, error) {
-	head, err := a.ChainHead(ctx)
+func updateWindow(ctx context.Context, a api.FullNode, w CidWindow, t int, r int, to time.Duration) (CidWindow, error) {
+	head, err := getHead(ctx, a, r, to)
 	if err != nil {
 		return nil, err
 	}
-
 	window := appendCIDsToWindow(w, head.Cids(), t)
-	ch <- window
-	return window, nil
+	return window, err
+}
+
+/*
+ * get chain head from API
+ * retries if API no available
+ * returns tipset
+ */
+func getHead(ctx context.Context, a api.FullNode, r int, t time.Duration) (*types.TipSet, error) {
+	for i := 0; i < r; i++ {
+		head, err := a.ChainHead(ctx)
+		if err != nil && i == (r-1) {
+			return nil, err
+		}
+		if err != nil {
+			log.Warnf("Call to API failed. Retrying in %.0fs", t.Seconds())
+			time.Sleep(t)
+			continue
+		}
+		return head, err
+	}
+	return nil, nil
 }
 
 /*
@@ -192,13 +223,13 @@ func appendCIDsToWindow(w CidWindow, c []cid.Cid, t int) CidWindow {
 /*
  * wait for node to sync
  */
-func WaitForSyncComplete(ctx context.Context, napi api.FullNode) error {
+func waitForSyncComplete(ctx context.Context, a api.FullNode, r int, t time.Duration) error {
 	for {
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
 		case <-time.After(3 * time.Second):
-			head, err := napi.ChainHead(ctx)
+			head, err := getHead(ctx, a, r, t)
 			if err != nil {
 				return err
 			}
@@ -209,3 +240,23 @@ func WaitForSyncComplete(ctx context.Context, napi api.FullNode) error {
 		}
 	}
 }
+
+/*
+ * A thin wrapper around lotus cli GetFullNodeAPI
+ * Adds retry logic
+ */
+func getFullNodeAPI(ctx *cli.Context, r int, t time.Duration) (api.FullNode, jsonrpc.ClientCloser, error) {
+	for i := 0; i < r; i++ {
+		api, closer, err := lcli.GetFullNodeAPI(ctx)
+		if err != nil && i == (r-1) {
+			return nil, nil, err
+		}
+		if err != nil {
+			log.Warnf("API connection failed. Retrying in %.0fs", t.Seconds())
+			time.Sleep(t)
+			continue
+		}
+		return api, closer, err
+	}
+	return nil, nil, nil
+}

cmd/lotus-health/main_test.go

@@ -28,12 +28,8 @@ func TestAppendCIDsToWindow(t *testing.T) {
 
 func TestCheckWindow(t *testing.T) {
 	assert := assert.New(t)
-	ch := make(chan CidWindow, 1)
-	och := make(chan bool, 1)
 	threshold := 3
-	go func() {
-		och <- checkWindow(ch, threshold)
-	}()
+
 	var healthyHeadCheckWindow CidWindow
 	healthyHeadCheckWindow = appendCIDsToWindow(healthyHeadCheckWindow, []cid.Cid{
 		makeCID("abcd"),
@@ -46,14 +42,79 @@ func TestCheckWindow(t *testing.T) {
 		makeCID("bbcd"),
 		makeCID("bbfe"),
 	}, threshold)
-	ch <- healthyHeadCheckWindow
-	select {
-	case ok := <-och:
-		assert.True(ok)
-	}
-	go func() {
-		och <- checkWindow(ch, threshold)
-	}()
+	ok := checkWindow(healthyHeadCheckWindow, threshold)
+	assert.True(ok)
+
+	var healthyHeadCheckWindow1 CidWindow
+	healthyHeadCheckWindow1 = appendCIDsToWindow(healthyHeadCheckWindow1, []cid.Cid{
+		makeCID("bbcd"),
+		makeCID("bbfe"),
+	}, threshold)
+	healthyHeadCheckWindow1 = appendCIDsToWindow(healthyHeadCheckWindow1, []cid.Cid{
+		makeCID("bbcd"),
+		makeCID("bbfe"),
+		makeCID("abcd"),
+	}, threshold)
+	healthyHeadCheckWindow1 = appendCIDsToWindow(healthyHeadCheckWindow1, []cid.Cid{
+		makeCID("abcd"),
+	}, threshold)
+	ok = checkWindow(healthyHeadCheckWindow1, threshold)
+	assert.True(ok)
+
+	var healthyHeadCheckWindow2 CidWindow
+	healthyHeadCheckWindow2 = appendCIDsToWindow(healthyHeadCheckWindow2, []cid.Cid{
+		makeCID("bbcd"),
+		makeCID("bbfe"),
+	}, threshold)
+	healthyHeadCheckWindow2 = appendCIDsToWindow(healthyHeadCheckWindow2, []cid.Cid{
+		makeCID("abcd"),
+	}, threshold)
+	ok = checkWindow(healthyHeadCheckWindow2, threshold)
+	assert.True(ok)
+
+	var healthyHeadCheckWindow3 CidWindow
+	healthyHeadCheckWindow3 = appendCIDsToWindow(healthyHeadCheckWindow3, []cid.Cid{
+		makeCID("abcd"),
+	}, threshold)
+	healthyHeadCheckWindow3 = appendCIDsToWindow(healthyHeadCheckWindow3, []cid.Cid{
+		makeCID("bbcd"),
+		makeCID("bbfe"),
+	}, threshold)
+	ok = checkWindow(healthyHeadCheckWindow3, threshold)
+	assert.True(ok)
+
+	var healthyHeadCheckWindow4 CidWindow
+	healthyHeadCheckWindow4 = appendCIDsToWindow(healthyHeadCheckWindow4, []cid.Cid{
+		makeCID("bbcd"),
+		makeCID("bbfe"),
+	}, threshold)
+	ok = checkWindow(healthyHeadCheckWindow4, threshold)
+	assert.True(ok)
+
+	var healthyHeadCheckWindow5 CidWindow
+	healthyHeadCheckWindow5 = appendCIDsToWindow(healthyHeadCheckWindow5, []cid.Cid{
+		makeCID("bbcd"),
+		makeCID("bbfe"),
+		makeCID("bbff"),
+	}, 5)
+	healthyHeadCheckWindow5 = appendCIDsToWindow(healthyHeadCheckWindow5, []cid.Cid{
+		makeCID("bbcd"),
+		makeCID("bbfe"),
+	}, 5)
+	healthyHeadCheckWindow5 = appendCIDsToWindow(healthyHeadCheckWindow5, []cid.Cid{
+		makeCID("abcd"),
+	}, 5)
+	healthyHeadCheckWindow5 = appendCIDsToWindow(healthyHeadCheckWindow5, []cid.Cid{
+		makeCID("cbcd"),
+		makeCID("cbfe"),
+	}, 5)
+	healthyHeadCheckWindow5 = appendCIDsToWindow(healthyHeadCheckWindow5, []cid.Cid{
+		makeCID("cbcd"),
+		makeCID("cbfe"),
+	}, 5)
+	ok = checkWindow(healthyHeadCheckWindow5, threshold)
+	assert.True(ok)
+
 	var unhealthyHeadCheckWindow CidWindow
 	unhealthyHeadCheckWindow = appendCIDsToWindow(unhealthyHeadCheckWindow, []cid.Cid{
 		makeCID("abcd"),
@@ -67,12 +128,33 @@ func TestCheckWindow(t *testing.T) {
 		makeCID("abcd"),
 		makeCID("fbcd"),
 	}, threshold)
-	ch <- unhealthyHeadCheckWindow
-	select {
-	case ok := <-och:
-		assert.False(ok)
-	}
+	ok = checkWindow(unhealthyHeadCheckWindow, threshold)
+	assert.False(ok)
 
+	var unhealthyHeadCheckWindow1 CidWindow
+	unhealthyHeadCheckWindow1 = appendCIDsToWindow(unhealthyHeadCheckWindow1, []cid.Cid{
+		makeCID("abcd"),
+		makeCID("fbcd"),
+	}, threshold)
+	unhealthyHeadCheckWindow1 = appendCIDsToWindow(unhealthyHeadCheckWindow1, []cid.Cid{
+		makeCID("abcd"),
+		makeCID("fbcd"),
+	}, threshold)
+	ok = checkWindow(unhealthyHeadCheckWindow1, threshold)
+	assert.True(ok)
+
+	var unhealthyHeadCheckWindow2 CidWindow
+	unhealthyHeadCheckWindow2 = appendCIDsToWindow(unhealthyHeadCheckWindow2, []cid.Cid{
+		makeCID("abcd"),
+	}, threshold)
+	unhealthyHeadCheckWindow2 = appendCIDsToWindow(unhealthyHeadCheckWindow2, []cid.Cid{
+		makeCID("abcd"),
+	}, threshold)
+	unhealthyHeadCheckWindow2 = appendCIDsToWindow(unhealthyHeadCheckWindow2, []cid.Cid{
+		makeCID("abcd"),
+	}, threshold)
+	ok = checkWindow(unhealthyHeadCheckWindow2, threshold)
+	assert.False(ok)
 }
 
 func makeCID(s string) cid.Cid {

cmd/lotus-health/notify.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"os"
+
 	"github.com/coreos/go-systemd/dbus"
 )
 
-func alertHandler(n string, ch chan interface{}) (string, error) {
+func notifyHandler(n string, ch chan interface{}, sCh chan os.Signal) (string, error) {
 	select {
+	// alerts to restart systemd unit
 	case <-ch:
 		statusCh := make(chan string, 1)
 		c, err := dbus.New()
@@ -20,5 +23,9 @@ func alertHandler(n string, ch chan interface{}) (string, error) {
 		case result := <-statusCh:
 			return result, nil
 		}
+	// SIGTERM
+	case <-sCh:
+		os.Exit(1)
+		return "", nil
 	}
 }

cmd/lotus-seal-worker/transfer.go

@@ -15,6 +15,14 @@ import (
 	"github.com/filecoin-project/lotus/lib/tarutil"
 )
 
+func (w *worker) sizeForType(typ string) int64 {
+	size := int64(w.sb.SectorSize())
+	if typ == "cache" {
+		size *= 10
+	}
+	return size
+}
+
 func (w *worker) fetch(typ string, sectorID uint64) error {
 	outname := filepath.Join(w.repo, typ, w.sb.SectorName(sectorID))
 
@@ -37,7 +45,7 @@ func (w *worker) fetch(typ string, sectorID uint64) error {
 		return xerrors.Errorf("non-200 code: %d", resp.StatusCode)
 	}
 
-	bar := pb.New64(resp.ContentLength)
+	bar := pb.New64(w.sizeForType(typ))
 	bar.ShowPercent = true
 	bar.ShowSpeed = true
 	bar.Units = pb.U_BYTES
@@ -88,7 +96,7 @@ func (w *worker) push(typ string, sectorID uint64) error {
 		return xerrors.Errorf("opening push reader: %w", err)
 	}
 
-	bar := pb.New64(0)
+	bar := pb.New64(w.sizeForType(typ))
 	bar.ShowPercent = true
 	bar.ShowSpeed = true
 	bar.ShowCounters = true

cmd/lotus-storage-miner/sectors.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"sort"
 	"strconv"
+	"text/tabwriter"
+	"time"
 
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
@@ -41,6 +44,12 @@ var sectorsCmd = &cli.Command{
 var sectorsStatusCmd = &cli.Command{
 	Name:  "status",
 	Usage: "Get the seal status of a sector by its ID",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "log",
+			Usage: "display event log",
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
 		if err != nil {
@@ -77,6 +86,17 @@ var sectorsStatusCmd = &cli.Command{
 		if status.LastErr != "" {
 			fmt.Printf("Last Error:\t\t%s\n", status.LastErr)
 		}
+
+		if cctx.Bool("log") {
+			fmt.Printf("--------\nEvent Log:\n")
+
+			for i, l := range status.Log {
+				fmt.Printf("%d.\t%s:\t[%s]\t%s\n", i, time.Unix(int64(l.Timestamp), 0), l.Kind, l.Message)
+				if l.Trace != "" {
+					fmt.Printf("\t%s\n", l.Trace)
+				}
+			}
+		}
 		return nil
 	},
 }
@@ -131,17 +151,19 @@ var sectorsListCmd = &cli.Command{
 			return list[i] < list[j]
 		})
 
+		w := tabwriter.NewWriter(os.Stdout, 8, 4, 1, ' ', 0)
+
 		for _, s := range list {
 			st, err := nodeApi.SectorsStatus(ctx, s)
 			if err != nil {
-				fmt.Printf("%d:\tError: %s\n", s, err)
+				fmt.Fprintf(w, "%d:\tError: %s\n", s, err)
 				continue
 			}
 
 			_, inSSet := commitedIDs[s]
 			_, inPSet := provingIDs[s]
 
-			fmt.Printf("%d: %s\tsSet: %s\tpSet: %s\ttktH: %d\tseedH: %d\tdeals: %v\n",
+			fmt.Fprintf(w, "%d: %s\tsSet: %s\tpSet: %s\ttktH: %d\tseedH: %d\tdeals: %v\n",
 				s,
 				api.SectorStates[st.State],
 				yesno(inSSet),
@@ -151,7 +173,8 @@ var sectorsListCmd = &cli.Command{
 				st.Deals,
 			)
 		}
-		return nil
+
+		return w.Flush()
 	},
 }
 

cmd/lotus/daemon.go

@@ -5,14 +5,21 @@ package main
 import (
 	"context"
 	"io/ioutil"
+	"os"
+	"runtime/pprof"
 
 	paramfetch "github.com/filecoin-project/go-paramfetch"
+	"github.com/filecoin-project/go-sectorbuilder"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	"github.com/multiformats/go-multiaddr"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/node"
 	"github.com/filecoin-project/lotus/node/modules"
 	"github.com/filecoin-project/lotus/node/modules/testing"
@@ -56,8 +63,32 @@ var DaemonCmd = &cli.Command{
 			Name:  "bootstrap",
 			Value: true,
 		},
+		&cli.StringFlag{
+			Name:  "import-chain",
+			Usage: "on first run, load chain from given file",
+		},
+		&cli.BoolFlag{
+			Name:  "halt-after-import",
+			Usage: "halt the process after importing chain from file",
+		},
+		&cli.StringFlag{
+			Name:  "pprof",
+			Usage: "specify name of file for writing cpu profile to",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
+		if prof := cctx.String("pprof"); prof != "" {
+			profile, err := os.Create(prof)
+			if err != nil {
+				return err
+			}
+
+			if err := pprof.StartCPUProfile(profile); err != nil {
+				return err
+			}
+			defer pprof.StopCPUProfile()
+		}
+
 		ctx := context.Background()
 		r, err := repo.NewFS(cctx.String("repo"))
 		if err != nil {
@@ -81,6 +112,16 @@ var DaemonCmd = &cli.Command{
 			}
 		}
 
+		chainfile := cctx.String("import-chain")
+		if chainfile != "" {
+			if err := ImportChain(r, chainfile); err != nil {
+				return err
+			}
+			if cctx.Bool("halt-after-import") {
+				return nil
+			}
+		}
+
 		genesis := node.Options()
 		if len(genBytes) > 0 {
 			genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genBytes))
@@ -129,3 +170,50 @@ var DaemonCmd = &cli.Command{
 		return serveRPC(api, stop, endpoint)
 	},
 }
+
+func ImportChain(r repo.Repo, fname string) error {
+	fi, err := os.Open(fname)
+	if err != nil {
+		return err
+	}
+
+	lr, err := r.Lock(repo.FullNode)
+	if err != nil {
+		return err
+	}
+	defer lr.Close()
+
+	ds, err := lr.Datastore("/blocks")
+	if err != nil {
+		return err
+	}
+
+	mds, err := lr.Datastore("/metadata")
+	if err != nil {
+		return err
+	}
+
+	bs := blockstore.NewBlockstore(ds)
+
+	cst := store.NewChainStore(bs, mds, vm.Syscalls(sectorbuilder.ProofVerifier))
+
+	log.Info("importing chain from file...")
+	ts, err := cst.Import(fi)
+	if err != nil {
+		return xerrors.Errorf("importing chain failed: %w", err)
+	}
+
+	stm := stmgr.NewStateManager(cst)
+
+	log.Infof("validating imported chain...")
+	if err := stm.ValidateChain(context.TODO(), ts); err != nil {
+		return xerrors.Errorf("chain validation failed: %w", err)
+	}
+
+	log.Info("accepting %s as new head", ts.Cids())
+	if err := cst.SetHead(ts); err != nil {
+		return err
+	}
+
+	return nil
+}

gen/main.go

@@ -122,11 +122,12 @@ func main() {
 		os.Exit(1)
 	}
 
-	err = gen.WriteMapEncodersToFile("./storage/sectors/cbor_gen.go", "sectors",
+	err = gen.WriteMapEncodersToFile("./storage/sealing/cbor_gen.go", "sealing",
 		sealing.SealTicket{},
 		sealing.SealSeed{},
 		sealing.Piece{},
 		sealing.SectorInfo{},
+		sealing.Log{},
 	)
 	if err != nil {
 		fmt.Println(err)

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/docker/go-units v0.4.0
 	github.com/filecoin-project/chain-validation v0.0.3
 	github.com/filecoin-project/filecoin-ffi v0.0.0-20191219131535-bb699517a590
-	github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f
+	github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5
 	github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc
 	github.com/filecoin-project/go-amt-ipld/v2 v2.0.0
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
@@ -20,7 +20,7 @@ require (
 	github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce
 	github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8
 	github.com/filecoin-project/go-paramfetch v0.0.1
-	github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689
+	github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200123143044-d9cc96c53c55
 	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
 	github.com/go-ole/go-ole v1.2.4 // indirect
@@ -32,10 +32,10 @@ require (
 	github.com/ipfs/go-bitswap v0.1.8
 	github.com/ipfs/go-block-format v0.0.2
 	github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c
-	github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1
+	github.com/ipfs/go-car v0.0.3-0.20200121013634-f188c0e24291
 	github.com/ipfs/go-cid v0.0.4
 	github.com/ipfs/go-datastore v0.3.1
-	github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521
+	github.com/ipfs/go-ds-badger2 v0.0.0-20200123200730-d75eb2678a5d
 	github.com/ipfs/go-filestore v0.0.2
 	github.com/ipfs/go-fs-lock v0.0.1
 	github.com/ipfs/go-graphsync v0.0.4
@@ -86,7 +86,7 @@ require (
 	github.com/prometheus/common v0.2.0
 	github.com/stretchr/testify v1.4.0
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
-	github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe
+	github.com/whyrusleeping/cbor-gen v0.0.0-20200121162646-b63bacf5eaf8
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
 	github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d
 	go.opencensus.io v0.22.2

go.sum

@@ -77,10 +77,10 @@ github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzA
 github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
 github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
-github.com/dgraph-io/badger/v2 v2.0.1 h1:+D6dhIqC6jIeCclnxMHqk4HPuXgrRN5UfBsLR4dNQ3A=
-github.com/dgraph-io/badger/v2 v2.0.1/go.mod h1:YoRSIp1LmAJ7zH7tZwRvjNMUYLxB4wl3ebYkaIruZ04=
-github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e h1:aeUNgwup7PnDOBAD1BOKAqzb/W/NksOj6r3dwKKuqfg=
-github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs=
+github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200120142413-c3333a5a830e h1:Jz7uYxTCDVrtL5tzPxPu6o7Ybhom8Az7sWmjUO1OkQc=
+github.com/dgraph-io/badger/v2 v2.0.1-rc1.0.20200120142413-c3333a5a830e/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM=
+github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU=
+github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
@@ -99,6 +99,8 @@ github.com/filecoin-project/chain-validation v0.0.3 h1:luT/8kJ0WdMIqQ9Bm31W4JkuY
 github.com/filecoin-project/chain-validation v0.0.3/go.mod h1:NCEGFjcWRjb8akWFSOXvU6n2efkWIqAeOKU6o5WBGQw=
 github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f h1:L2jaVU8TvWTx7iZPhlYvUE8vkoOnj778XuKavz8W36g=
 github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f/go.mod h1:rCbpXPva2NKF9/J4X6sr7hbKBgQCxyFtRj7KOZqoIms=
+github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5 h1:/MmWluswvDIbuPvBct4q6HeQgVm62O2DzWYTB38kt4A=
+github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0=
 github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc h1:cODZD2YzpTUtrOSxbEnWFcQHidNRZiRdvLxySjGvG/M=
 github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc/go.mod h1:KsFPWjF+UUYl6n9A+qbg4bjFgAOneicFZtDH/LQEX2U=
 github.com/filecoin-project/go-amt-ipld/v2 v2.0.0 h1:rQ7GTJbWE5XqUCPiBgs4gbTKD2wC7MynZhSaf7ZzlUM=
@@ -106,36 +108,18 @@ github.com/filecoin-project/go-amt-ipld/v2 v2.0.0/go.mod h1:PAZ5tvSfMfWE327osqFX
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:av5fw6wmm58FYMgJeoB/lK9XXrgdugYiTqkdxjTy9k8=
 github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
-github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
-github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
-github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
-github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
-github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
-github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
-github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY=
 github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww=
 github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8 h1:g3oodvSz+Ou+ObwcVBB2wyt8SHdWpwzMiNJ19U1zZNA=
 github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8/go.mod h1:c8NTjvFVy1Ud02mmGDjOiMeawY2t6ALfrrdvAB01FQc=
-github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
-github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
-github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
-github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
-github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
-github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
-github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
-github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
-github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
 github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyCXUE0rimz4L7ghoE=
 github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
 github.com/filecoin-project/go-sectorbuilder v0.0.1/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
-github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689 h1:2cT5bhm/5I0RY+HBIPdRRrtjCwLj33Qx6DHRs9TCslY=
-github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
+github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200123143044-d9cc96c53c55 h1:XChPRKPZL+/N6a3ccLmjCJ7JrR+SFLFJDllv0BkxW4I=
+github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200123143044-d9cc96c53c55/go.mod h1:ahsryULdwYoZ94K09HcfqX3QBwevWVldENSV/EdCbNg=
 github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ=
-github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ=
-github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
 github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -227,8 +211,9 @@ github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbR
 github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0=
 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
-github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 h1:Nq8xEW+2KZq7IkRlkOh0rTEUI8FgunhMoLj5EMkJzbQ=
 github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1/go.mod h1:rmd887mJxQRDfndfDEY3Liyx8gQVyfFFRSHdsnDSAlk=
+github.com/ipfs/go-car v0.0.3-0.20200121013634-f188c0e24291 h1:Yy0dcFWw8oDV/WJ4S/rkMQRWnJ3tGr9EbgDDv2JhVQw=
+github.com/ipfs/go-car v0.0.3-0.20200121013634-f188c0e24291/go.mod h1:AG6sBpd2PWMccpAG7XLFBBQ/4rfBEtzUNeO2GSMesYk=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
@@ -246,8 +231,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U
 github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
 github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s=
 github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk=
-github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521 h1:GSygNXXIRXFVntMR3Yr30imf40H9cYDAw72Qb0za2nE=
-github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521/go.mod h1:oDKWqmQbLyyml3mgMtHgB7qAf6cQOEsJPRLFzfwKQ5Q=
+github.com/ipfs/go-ds-badger2 v0.0.0-20200123200730-d75eb2678a5d h1:/Pn9fMp6ih2M/3vDeA2H+8mkTFQSpLxFuIM1HJD3Zlw=
+github.com/ipfs/go-ds-badger2 v0.0.0-20200123200730-d75eb2678a5d/go.mod h1:sTQFaWUoW0OvhXzfHnQ9j39L6fdlqDkptDYcpC1XrYE=
 github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
 github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8=
 github.com/ipfs/go-filestore v0.0.2 h1:pcYwpjtXXwirtbjBXKVJM9CTa9F7/8v1EkfnDaHTO3s=
@@ -320,6 +305,7 @@ github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb h1:tmWYgjltxwM7PD
 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
+github.com/ipld/go-ipld-prime v0.0.1/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
 github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 h1:fASnkvtR+SmB2y453RxmDD3Uvd4LonVUgFGk9JoDaZs=
 github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
 github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 h1:lSip43rAdyGA+yRQuy6ju0ucZkWpYc1F2CTQtZTVW/4=
@@ -727,12 +713,13 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:x
 github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
 github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
 github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
-github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe h1:n7En1uyDtknjLRDXebWlPGJoHvwL8AkNcSQzuOoYYYQ=
-github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
+github.com/whyrusleeping/cbor-gen v0.0.0-20200121162646-b63bacf5eaf8 h1:PHZv8Nu+95MBVNu3qSgg3ncxIv8hy4DzGAOBR9xYQRc=
+github.com/whyrusleeping/cbor-gen v0.0.0-20200121162646-b63bacf5eaf8/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
+github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8=
 github.com/whyrusleeping/go-smux-multiplex v3.0.16+incompatible/go.mod h1:34LEDbeKFZInPUrAG+bjuJmUXONGdEFW7XL0SpTY1y4=

lib/increadtimeout/incrt.go

@@ -0,0 +1,73 @@
+package incrt
+
+import (
+	"io"
+	"time"
+
+	logging "github.com/ipfs/go-log/v2"
+)
+
+var log = logging.Logger("incrt")
+
+var now = time.Now
+
+type ReaderDeadline interface {
+	Read([]byte) (int, error)
+	SetReadDeadline(time.Time) error
+}
+
+type incrt struct {
+	rd ReaderDeadline
+
+	waitPerByte time.Duration
+	wait        time.Duration
+	maxWait     time.Duration
+}
+
+// New creates an Incremental Reader Timeout, with minimum sustained speed of
+// minSpeed bytes per second and with maximum wait of maxWait
+func New(rd ReaderDeadline, minSpeed int64, maxWait time.Duration) io.Reader {
+	return &incrt{
+		rd:          rd,
+		waitPerByte: time.Second / time.Duration(minSpeed),
+		wait:        maxWait,
+		maxWait:     maxWait,
+	}
+}
+
+type errNoWait struct{}
+
+func (err errNoWait) Error() string {
+	return "wait time exceeded"
+}
+func (err errNoWait) Timeout() bool {
+	return true
+}
+
+func (crt *incrt) Read(buf []byte) (int, error) {
+	start := now()
+	if crt.wait == 0 {
+		return 0, errNoWait{}
+	}
+
+	err := crt.rd.SetReadDeadline(start.Add(crt.wait))
+	if err != nil {
+		log.Warnf("unable to set daedline: %+v", err)
+	}
+
+	n, err := crt.rd.Read(buf)
+
+	crt.rd.SetReadDeadline(time.Time{})
+	if err == nil {
+		dur := now().Sub(start)
+		crt.wait -= dur
+		crt.wait += time.Duration(n) * crt.waitPerByte
+		if crt.wait < 0 {
+			crt.wait = 0
+		}
+		if crt.wait > crt.maxWait {
+			crt.wait = crt.maxWait
+		}
+	}
+	return n, err
+}

lib/jsonrpc/handler.go

@@ -99,7 +99,7 @@ func (h handlers) register(namespace string, r interface{}) {
 // Handle
 
 type rpcErrFunc func(w func(func(io.Writer)), req *request, code int, err error)
-type chanOut func(reflect.Value) interface{}
+type chanOut func(reflect.Value, int64) error
 
 func (h handlers) handleReader(ctx context.Context, r io.Reader, w io.Writer, rpcError rpcErrFunc) {
 	wf := func(cb func(io.Writer)) {
@@ -222,16 +222,25 @@ func (h handlers) handle(ctx context.Context, req request, w func(func(io.Writer
 	if handler.valOut != -1 {
 		resp.Result = callResult[handler.valOut].Interface()
 	}
+	if resp.Result != nil && reflect.TypeOf(resp.Result).Kind() == reflect.Chan {
+		// Channel responses are sent from channel control goroutine.
+		// Sending responses here could cause deadlocks on writeLk, or allow
+		// sending channel messages before this rpc call returns
 
-	w(func(w io.Writer) {
-		if resp.Result != nil && reflect.TypeOf(resp.Result).Kind() == reflect.Chan {
-			// this must happen in the writer callback, otherwise we may start sending
-			// channel messages before we send this response
-
-			//noinspection GoNilness // already checked above
-			resp.Result = chOut(callResult[handler.valOut])
+		//noinspection GoNilness // already checked above
+		err = chOut(callResult[handler.valOut], *req.ID)
+		if err == nil {
+			return // channel goroutine handles responding
 		}
 
+		log.Warnf("failed to setup channel in RPC call to '%s': %+v", req.Method, err)
+		resp.Error = &respError{
+			Code:    1,
+			Message: err.(error).Error(),
+		}
+	}
+
+	w(func(w io.Writer) {
 		if err := json.NewEncoder(w).Encode(resp); err != nil {
 			log.Error(err)
 			return

lib/jsonrpc/rpc_test.go

@@ -375,5 +375,51 @@ func TestChan(t *testing.T) {
 	serverHandler.wait <- struct{}{}
 	_, ok = <-sub
 	require.Equal(t, false, ok)
+}
 
+func TestControlChanDeadlock(t *testing.T) {
+	for r := 0; r < 20; r++ {
+		testControlChanDeadlock(t)
+	}
+}
+
+func testControlChanDeadlock(t *testing.T) {
+	var client struct {
+		Sub func(context.Context, int, int) (<-chan int, error)
+	}
+
+	n := 5000
+
+	serverHandler := &ChanHandler{
+		wait: make(chan struct{}, n),
+	}
+
+	rpcServer := NewServer()
+	rpcServer.Register("ChanHandler", serverHandler)
+
+	testServ := httptest.NewServer(rpcServer)
+	defer testServ.Close()
+
+	closer, err := NewClient("ws://"+testServ.Listener.Addr().String(), "ChanHandler", &client, nil)
+	require.NoError(t, err)
+
+	defer closer()
+
+	for i := 0; i < n; i++ {
+		serverHandler.wait <- struct{}{}
+	}
+
+	ctx, _ := context.WithCancel(context.Background())
+
+	sub, err := client.Sub(ctx, 1, -1)
+	require.NoError(t, err)
+
+	go func() {
+		for i := 0; i < n; i++ {
+			require.Equal(t, i+1, <-sub)
+		}
+	}()
+
+	_, err = client.Sub(ctx, 2, -1)
+	require.NoError(t, err)
 }

lib/jsonrpc/websocket.go

@@ -11,6 +11,7 @@ import (
 	"sync/atomic"
 
 	"github.com/gorilla/websocket"
+	"golang.org/x/xerrors"
 )
 
 const wsCancel = "xrpc.cancel"
@@ -33,8 +34,10 @@ type frame struct {
 }
 
 type outChanReg struct {
-	id uint64
-	ch reflect.Value
+	reqID int64
+
+	chID uint64
+	ch   reflect.Value
 }
 
 type wsConn struct {
@@ -134,51 +137,80 @@ func (c *wsConn) sendRequest(req request) {
 // (forwards channel messages to client)
 func (c *wsConn) handleOutChans() {
 	regV := reflect.ValueOf(c.registerCh)
+	exitV := reflect.ValueOf(c.exiting)
 
 	cases := []reflect.SelectCase{
 		{ // registration chan always 0
 			Dir:  reflect.SelectRecv,
 			Chan: regV,
 		},
+		{ // exit chan always 1
+			Dir:  reflect.SelectRecv,
+			Chan: exitV,
+		},
 	}
+	internal := len(cases)
 	var caseToID []uint64
 
 	for {
 		chosen, val, ok := reflect.Select(cases)
 
-		if chosen == 0 { // control channel
+		switch chosen {
+		case 0: // registration channel
 			if !ok {
 				// control channel closed - signals closed connection
+				// This shouldn't happen, instead the exiting channel should get closed
+				log.Warn("control channel closed")
+				return
+			}
+
+			registration := val.Interface().(outChanReg)
+
+			caseToID = append(caseToID, registration.chID)
+			cases = append(cases, reflect.SelectCase{
+				Dir:  reflect.SelectRecv,
+				Chan: registration.ch,
+			})
+
+			c.nextWriter(func(w io.Writer) {
+				resp := &response{
+					Jsonrpc: "2.0",
+					ID:      registration.reqID,
+					Result:  registration.chID,
+				}
+
+				if err := json.NewEncoder(w).Encode(resp); err != nil {
+					log.Error(err)
+					return
+				}
+			})
+
+			continue
+		case 1: // exiting channel
+			if !ok {
+				// exiting channel closed - signals closed connection
 				//
 				// We're not closing any channels as we're on receiving end.
 				// Also, context cancellation below should take care of any running
 				// requests
 				return
 			}
-
-			registration := val.Interface().(outChanReg)
-
-			caseToID = append(caseToID, registration.id)
-			cases = append(cases, reflect.SelectCase{
-				Dir:  reflect.SelectRecv,
-				Chan: registration.ch,
-			})
-
+			log.Warn("exiting channel received a message")
 			continue
 		}
 
 		if !ok {
 			// Output channel closed, cleanup, and tell remote that this happened
 
-			n := len(caseToID)
+			n := len(cases) - 1
 			if n > 0 {
 				cases[chosen] = cases[n]
-				caseToID[chosen-1] = caseToID[n-1]
+				caseToID[chosen-internal] = caseToID[n-internal]
 			}
 
-			id := caseToID[chosen-1]
+			id := caseToID[chosen-internal]
 			cases = cases[:n]
-			caseToID = caseToID[:n-1]
+			caseToID = caseToID[:n-internal]
 
 			c.sendRequest(request{
 				Jsonrpc: "2.0",
@@ -194,24 +226,29 @@ func (c *wsConn) handleOutChans() {
 			Jsonrpc: "2.0",
 			ID:      nil, // notification
 			Method:  chValue,
-			Params:  []param{{v: reflect.ValueOf(caseToID[chosen-1])}, {v: val}},
+			Params:  []param{{v: reflect.ValueOf(caseToID[chosen-internal])}, {v: val}},
 		})
 	}
 }
 
 // handleChanOut registers output channel for forwarding to client
-func (c *wsConn) handleChanOut(ch reflect.Value) interface{} {
+func (c *wsConn) handleChanOut(ch reflect.Value, req int64) error {
 	c.spawnOutChanHandlerOnce.Do(func() {
 		go c.handleOutChans()
 	})
 	id := atomic.AddUint64(&c.chanCtr, 1)
 
-	c.registerCh <- outChanReg{
-		id: id,
-		ch: ch,
-	}
+	select {
+	case c.registerCh <- outChanReg{
+		reqID: req,
 
-	return id
+		chID: id,
+		ch:   ch,
+	}:
+		return nil
+	case <-c.exiting:
+		return xerrors.New("connection closing")
+	}
 }
 
 //                          //
@@ -389,7 +426,6 @@ func (c *wsConn) handleWsConn(ctx context.Context) {
 	c.chanHandlers = map[uint64]func(m []byte, ok bool){}
 
 	c.registerCh = make(chan outChanReg)
-	defer close(c.registerCh)
 	defer close(c.exiting)
 
 	// ////

lib/lotuslog/levels.go

@@ -1,12 +1,18 @@
 package lotuslog
 
-import logging "github.com/ipfs/go-log/v2"
+import (
+	"os"
+
+	logging "github.com/ipfs/go-log/v2"
+)
 
 func SetupLogLevels() {
-	logging.SetLogLevel("*", "INFO")
-	logging.SetLogLevel("dht", "ERROR")
-	logging.SetLogLevel("swarm2", "WARN")
-	logging.SetLogLevel("bitswap", "WARN")
-	logging.SetLogLevel("pubsub", "WARN")
-	logging.SetLogLevel("connmgr", "WARN")
+	if _, set := os.LookupEnv("GOLOG_LOG_LEVEL"); !set {
+		logging.SetLogLevel("*", "INFO")
+		logging.SetLogLevel("dht", "ERROR")
+		logging.SetLogLevel("swarm2", "WARN")
+		logging.SetLogLevel("bitswap", "WARN")
+		logging.SetLogLevel("pubsub", "WARN")
+		logging.SetLogLevel("connmgr", "WARN")
+	}
 }

lib/statemachine/cbor_gen.go

@@ -66,61 +66,55 @@ func (t *TestState) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type map")
 	}
 
-	if extra != 2 {
-		return fmt.Errorf("cbor input had wrong number of fields")
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("TestState: map struct too large (%d)", extra)
 	}
 
 	var name string
+	n := extra
 
-	// t.A (uint64) (uint64)
+	for i := uint64(0); i < n; i++ {
 
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
+		{
+			sval, err := cbg.ReadString(br)
+			if err != nil {
+				return err
+			}
+
+			name = string(sval)
 		}
 
-		name = string(sval)
-	}
+		switch name {
+		// t.A (uint64) (uint64)
+		case "A":
 
-	if name != "A" {
-		return fmt.Errorf("expected struct map entry %s to be A", name)
-	}
+			maj, extra, err = cbg.CborReadHeader(br)
+			if err != nil {
+				return err
+			}
+			if maj != cbg.MajUnsignedInt {
+				return fmt.Errorf("wrong type for uint64 field")
+			}
+			t.A = uint64(extra)
+			// t.B (uint64) (uint64)
+		case "B":
 
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.A = uint64(extra)
-	// t.B (uint64) (uint64)
+			maj, extra, err = cbg.CborReadHeader(br)
+			if err != nil {
+				return err
+			}
+			if maj != cbg.MajUnsignedInt {
+				return fmt.Errorf("wrong type for uint64 field")
+			}
+			t.B = uint64(extra)
 
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
+		default:
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
-
-		name = string(sval)
 	}
 
-	if name != "B" {
-		return fmt.Errorf("expected struct map entry %s to be B", name)
-	}
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.B = uint64(extra)
 	return nil
 }
-
 func (t *TestEvent) MarshalCBOR(w io.Writer) error {
 	if t == nil {
 		_, err := w.Write(cbg.CborNull)
@@ -182,57 +176,52 @@ func (t *TestEvent) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type map")
 	}
 
-	if extra != 2 {
-		return fmt.Errorf("cbor input had wrong number of fields")
+	if extra > cbg.MaxLength {
+		return fmt.Errorf("TestEvent: map struct too large (%d)", extra)
 	}
 
 	var name string
+	n := extra
 
-	// t.A (string) (string)
+	for i := uint64(0); i < n; i++ {
 
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
+		{
+			sval, err := cbg.ReadString(br)
+			if err != nil {
+				return err
+			}
+
+			name = string(sval)
 		}
 
-		name = string(sval)
-	}
+		switch name {
+		// t.A (string) (string)
+		case "A":
 
-	if name != "A" {
-		return fmt.Errorf("expected struct map entry %s to be A", name)
-	}
+			{
+				sval, err := cbg.ReadString(br)
+				if err != nil {
+					return err
+				}
 
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
+				t.A = string(sval)
+			}
+			// t.Val (uint64) (uint64)
+		case "Val":
+
+			maj, extra, err = cbg.CborReadHeader(br)
+			if err != nil {
+				return err
+			}
+			if maj != cbg.MajUnsignedInt {
+				return fmt.Errorf("wrong type for uint64 field")
+			}
+			t.Val = uint64(extra)
+
+		default:
+			return fmt.Errorf("unknown struct field %d: '%s'", i, name)
 		}
-
-		t.A = string(sval)
-	}
-	// t.Val (uint64) (uint64)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		name = string(sval)
 	}
 
-	if name != "Val" {
-		return fmt.Errorf("expected struct map entry %s to be Val", name)
-	}
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.Val = uint64(extra)
 	return nil
 }

miner/miner.go

@@ -227,7 +227,16 @@ eventLoop:
 			}
 		} else {
 			nextRound := time.Unix(int64(base.ts.MinTimestamp()+uint64(build.BlockDelay*base.nullRounds)), 0)
-			time.Sleep(time.Until(nextRound))
+
+			select {
+			case <-time.After(time.Until(nextRound)):
+			case <-m.stop:
+				stopping := m.stopping
+				m.stop = nil
+				m.stopping = nil
+				close(stopping)
+				return
+			}
 		}
 	}
 }

node/impl/full/chain.go

@@ -3,6 +3,7 @@ package full
 import (
 	"context"
 	"fmt"
+	"io"
 	"strconv"
 	"strings"
 
@@ -18,10 +19,12 @@ import (
 	"github.com/ipfs/go-path"
 	"github.com/ipfs/go-path/resolver"
 	mh "github.com/multiformats/go-multihash"
+	"github.com/prometheus/common/log"
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
+
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -312,3 +315,34 @@ func (a *ChainAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Mess
 
 	return cm.VMMessage(), nil
 }
+
+func (a *ChainAPI) ChainExport(ctx context.Context, ts *types.TipSet) (<-chan []byte, error) {
+	r, w := io.Pipe()
+	out := make(chan []byte)
+	go func() {
+		defer w.Close()
+		if err := a.Chain.Export(ctx, ts, w); err != nil {
+			log.Errorf("chain export call failed: %s", err)
+			return
+		}
+	}()
+
+	go func() {
+		defer close(out)
+		for {
+			buf := make([]byte, 4096)
+			n, err := r.Read(buf)
+			if err != nil {
+				log.Errorf("chain export pipe read failed: %s", err)
+				return
+			}
+			select {
+			case out <- buf[:n]:
+			case <-ctx.Done():
+				log.Warnf("export writer failed: %s", ctx.Err())
+			}
+		}
+	}()
+
+	return out, nil
+}

node/impl/full/state.go

@@ -108,7 +108,7 @@ func (a *StateAPI) StatePledgeCollateral(ctx context.Context, ts *types.TipSet)
 	return types.BigFromBytes(ret.Return), nil
 }
 
-func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error) {
+func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.MethodCall, error) {
 	return a.StateManager.Call(ctx, msg, ts)
 }
 

node/impl/storminer.go

@@ -160,6 +160,16 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid uint64) (api.S
 		deals[i] = piece.DealID
 	}
 
+	log := make([]api.SectorLog, len(info.Log))
+	for i, l := range info.Log {
+		log[i] = api.SectorLog{
+			Kind:      l.Kind,
+			Timestamp: l.Timestamp,
+			Trace:     l.Trace,
+			Message:   l.Message,
+		}
+	}
+
 	return api.SectorInfo{
 		SectorID: sid,
 		State:    info.State,
@@ -172,6 +182,7 @@ func (sm *StorageMinerAPI) SectorsStatus(ctx context.Context, sid uint64) (api.S
 		Retries:  info.Nonce,
 
 		LastErr: info.LastErr,
+		Log:     log,
 	}, nil
 }
 

node/node_test.go

@@ -393,6 +393,12 @@ func TestAPIRPC(t *testing.T) {
 }
 
 func TestAPIDealFlow(t *testing.T) {
+	logging.SetLogLevel("miner", "ERROR")
+	logging.SetLogLevel("chainstore", "ERROR")
+	logging.SetLogLevel("chain", "ERROR")
+	logging.SetLogLevel("sub", "ERROR")
+	logging.SetLogLevel("storageminer", "ERROR")
+
 	test.TestDealFlow(t, mockSbBuilder, 10*time.Millisecond)
 }
 

peermgr/peermgr.go

@@ -42,7 +42,7 @@ type PeerMgr struct {
 	maxFilPeers int
 	minFilPeers int
 
-	expanding bool
+	expanding chan struct{}
 
 	h   host.Host
 	dht *dht.IpfsDHT
@@ -56,7 +56,8 @@ func NewPeerMgr(h host.Host, dht *dht.IpfsDHT, bootstrap dtypes.BootstrapPeers)
 		dht:           dht,
 		bootstrappers: bootstrap,
 
-		peers: make(map[peer.ID]time.Duration),
+		peers:     make(map[peer.ID]time.Duration),
+		expanding: make(chan struct{}, 1),
 
 		maxFilPeers: MaxFilPeers,
 		minFilPeers: MinFilPeers,
@@ -125,18 +126,19 @@ func (pmgr *PeerMgr) getPeerCount() int {
 }
 
 func (pmgr *PeerMgr) expandPeers() {
-	if pmgr.expanding {
+	select {
+	case pmgr.expanding <- struct{}{}:
+	default:
 		return
 	}
-	pmgr.expanding = true
+
 	go func() {
-		defer func() {
-			pmgr.expanding = false
-		}()
 		ctx, cancel := context.WithTimeout(context.TODO(), time.Second*30)
 		defer cancel()
 
 		pmgr.doExpand(ctx)
+
+		<-pmgr.expanding
 	}()
 }
 

storage/fpost_run.go

@@ -117,7 +117,10 @@ func (s *fpostScheduler) runPost(ctx context.Context, eps uint64, ts *types.TipS
 		return nil, xerrors.Errorf("getting sorted sector info: %w", err)
 	}
 
-	log.Infow("running fPoSt", "chain-random", rand, "eps", eps, "height", ts.Height())
+	log.Infow("running fPoSt",
+		"chain-random", rand,
+		"eps", eps,
+		"height", ts.Height())
 
 	faults, err := s.checkFaults(ctx, ssi)
 	if err != nil {
@@ -129,6 +132,10 @@ func (s *fpostScheduler) runPost(ctx context.Context, eps uint64, ts *types.TipS
 	var seed [32]byte
 	copy(seed[:], rand)
 
+	log.Infow("generating fPoSt",
+		"sectors", len(ssi.Values()),
+		"faults", len(faults))
+
 	scandidates, proof, err := s.sb.GenerateFallbackPoSt(ssi, seed, faults)
 	if err != nil {
 		return nil, xerrors.Errorf("running post failed: %w", err)

storage/fpost_sched.go

@@ -70,6 +70,9 @@ func (s *fpostScheduler) run(ctx context.Context) {
 			var lowest, highest *types.TipSet = s.cur, nil
 
 			for _, change := range changes {
+				if change.Val == nil {
+					log.Errorf("change.Val was nil")
+				}
 				switch change.Type {
 				case store.HCRevert:
 					lowest = change.Val
@@ -111,6 +114,9 @@ func (s *fpostScheduler) revert(ctx context.Context, newLowest *types.TipSet) er
 }
 
 func (s *fpostScheduler) update(ctx context.Context, new *types.TipSet) error {
+	if new == nil {
+		return xerrors.Errorf("no new tipset in fpostScheduler.update")
+	}
 	newEPS, start, err := s.shouldFallbackPost(ctx, new)
 	if err != nil {
 		return err

storage/miner.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/events"
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/store"
@@ -35,14 +36,11 @@ type Miner struct {
 	worker address.Address
 
 	sealing *sealing.Sealing
-
-	stop    chan struct{}
-	stopped chan struct{}
 }
 
 type storageMinerApi interface {
 	// Call a read only method on actors (no interaction with the chain required)
-	StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error)
+	StateCall(context.Context, *types.Message, *types.TipSet) (*api.MethodCall, error)
 	StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error)
 	StateMinerElectionPeriodStart(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error)
 	StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error)
@@ -51,6 +49,7 @@ type storageMinerApi interface {
 	StateWaitMsg(context.Context, cid.Cid) (*api.MsgWait, error) // TODO: removeme eventually
 	StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error)
 	StateGetReceipt(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error)
+	StateMarketStorageDeal(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error)
 
 	MpoolPushMessage(context.Context, *types.Message) (*types.SignedMessage, error)
 
@@ -59,6 +58,7 @@ type storageMinerApi interface {
 	ChainGetRandomness(context.Context, types.TipSetKey, int64) ([]byte, error)
 	ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet) (*types.TipSet, error)
 	ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
+	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
 
 	WalletSign(context.Context, address.Address, []byte) (*types.Signature, error)
 	WalletBalance(context.Context, address.Address) (types.BigInt, error)
@@ -74,9 +74,6 @@ func NewMiner(api storageMinerApi, addr address.Address, h host.Host, ds datasto
 		tktFn: tktFn,
 
 		maddr: addr,
-
-		stop:    make(chan struct{}),
-		stopped: make(chan struct{}),
 	}
 
 	return m, nil
@@ -100,19 +97,14 @@ func (m *Miner) Run(ctx context.Context) error {
 	evts := events.NewEvents(ctx, m.api)
 	m.sealing = sealing.New(m.api, evts, m.maddr, m.worker, m.ds, m.sb, m.tktFn)
 
+	go m.sealing.Run(ctx)
+
 	return nil
 }
 
 func (m *Miner) Stop(ctx context.Context) error {
 	defer m.sealing.Stop(ctx)
-
-	close(m.stop)
-	select {
-	case <-m.stopped:
-		return nil
-	case <-ctx.Done():
-		return ctx.Err()
-	}
+	return nil
 }
 
 func (m *Miner) runPreflightChecks(ctx context.Context) error {

storage/sbmock/sbmock.go

@@ -207,6 +207,10 @@ func (sb *SBMock) GetPath(string, string) (string, error) {
 	panic("nyi")
 }
 
+func (sb *SBMock) CanCommit(sectorID uint64) (bool, error) {
+	return true, nil
+}
+
 func (sb *SBMock) WorkerStats() sectorbuilder.WorkerStats {
 	panic("nyi")
 }

storage/sealing/checks.go

@@ -0,0 +1,105 @@
+package sealing
+
+import (
+	"context"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+// TODO: For now we handle this by halting state execution, when we get jsonrpc reconnecting
+//  We should implement some wait-for-api logic
+type ErrApi struct{ error }
+
+type ErrInvalidDeals struct{ error }
+type ErrExpiredDeals struct{ error }
+
+type ErrBadCommD struct{ error }
+type ErrExpiredTicket struct{ error }
+
+// checkPieces validates that:
+//  - Each piece han a corresponding on chain deal
+//  - Piece commitments match with on chain deals
+//  - Piece sizes match
+//  - Deals aren't expired
+func checkPieces(ctx context.Context, si SectorInfo, api sealingApi) error {
+	head, err := api.ChainHead(ctx)
+	if err != nil {
+		return &ErrApi{xerrors.Errorf("getting chain head: %w", err)}
+	}
+
+	for i, piece := range si.Pieces {
+		deal, err := api.StateMarketStorageDeal(ctx, piece.DealID, nil)
+		if err != nil {
+			return &ErrApi{xerrors.Errorf("getting deal %d for piece %d: %w", piece.DealID, i, err)}
+		}
+
+		if string(deal.PieceRef) != string(piece.CommP) {
+			return &ErrInvalidDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers deal %d with wrong CommP: %x != %x", i, len(si.Pieces), si.SectorID, piece.DealID, piece.CommP, deal.PieceRef)}
+		}
+
+		if piece.Size != deal.PieceSize {
+			return &ErrInvalidDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers deal %d with different size: %d != %d", i, len(si.Pieces), si.SectorID, piece.DealID, piece.Size, deal.PieceSize)}
+		}
+
+		if head.Height() >= deal.ProposalExpiration {
+			return &ErrExpiredDeals{xerrors.Errorf("piece %d (or %d) of sector %d refers expired deal %d - expires %d, head %d", i, len(si.Pieces), si.SectorID, piece.DealID, deal.ProposalExpiration, head.Height())}
+		}
+	}
+
+	return nil
+}
+
+// checkSeal checks that data commitment generated in the sealing process
+//  matches pieces, and that the seal ticket isn't expired
+func checkSeal(ctx context.Context, maddr address.Address, si SectorInfo, api sealingApi) (err error) {
+	head, err := api.ChainHead(ctx)
+	if err != nil {
+		return &ErrApi{xerrors.Errorf("getting chain head: %w", err)}
+	}
+
+	ssize, err := api.StateMinerSectorSize(ctx, maddr, head)
+	if err != nil {
+		return &ErrApi{err}
+	}
+
+	ccparams, err := actors.SerializeParams(&actors.ComputeDataCommitmentParams{
+		DealIDs:    si.deals(),
+		SectorSize: ssize,
+	})
+	if err != nil {
+		return xerrors.Errorf("computing params for ComputeDataCommitment: %w", err)
+	}
+
+	ccmt := &types.Message{
+		To:       actors.StorageMarketAddress,
+		From:     maddr,
+		Value:    types.NewInt(0),
+		GasPrice: types.NewInt(0),
+		GasLimit: types.NewInt(9999999999),
+		Method:   actors.SMAMethods.ComputeDataCommitment,
+		Params:   ccparams,
+	}
+	r, err := api.StateCall(ctx, ccmt, nil)
+	if err != nil {
+		return &ErrApi{xerrors.Errorf("calling ComputeDataCommitment: %w", err)}
+	}
+	if r.ExitCode != 0 {
+		return &ErrBadCommD{xerrors.Errorf("receipt for ComputeDataCommitment had exit code %d", r.ExitCode)}
+	}
+	if string(r.Return) != string(si.CommD) {
+		return &ErrBadCommD{xerrors.Errorf("on chain CommD differs from sector: %x != %x", r.Return, si.CommD)}
+	}
+
+	if int64(head.Height())-int64(si.Ticket.BlockHeight+build.SealRandomnessLookback) > build.SealRandomnessLookbackLimit {
+		return &ErrExpiredTicket{xerrors.Errorf("ticket expired: seal height: %d, head: %d", si.Ticket.BlockHeight+build.SealRandomnessLookback, head.Height())}
+	}
+
+	return nil
+
+}

storage/sealing/fsm.go

@@ -2,7 +2,9 @@ package sealing
 
 import (
 	"context"
+	"fmt"
 	"reflect"
+	"time"
 
 	"golang.org/x/xerrors"
 
@@ -19,9 +21,8 @@ func (m *Sealing) Plan(events []statemachine.Event, user interface{}) (interface
 	return func(ctx statemachine.Context, si SectorInfo) error {
 		err := next(ctx, si)
 		if err != nil {
-			if err := ctx.Send(SectorFatalError{error: err}); err != nil {
-				return xerrors.Errorf("error while sending error: reporting %+v: %w", err, err)
-			}
+			log.Errorf("unhandled sector error (%d): %+v", si.SectorID, err)
+			return nil
 		}
 
 		return nil
@@ -34,23 +35,37 @@ var fsmPlanners = []func(events []statemachine.Event, state *SectorInfo) error{
 	api.Unsealed: planOne(
 		on(SectorSealed{}, api.PreCommitting),
 		on(SectorSealFailed{}, api.SealFailed),
+		on(SectorPackingFailed{}, api.PackingFailed),
 	),
 	api.PreCommitting: planOne(
-		on(SectorPreCommitted{}, api.PreCommitted),
+		on(SectorSealFailed{}, api.SealFailed),
+		on(SectorPreCommitted{}, api.WaitSeed),
 		on(SectorPreCommitFailed{}, api.PreCommitFailed),
 	),
-	api.PreCommitted: planOne(
+	api.WaitSeed: planOne(
 		on(SectorSeedReady{}, api.Committing),
 		on(SectorPreCommitFailed{}, api.PreCommitFailed),
 	),
 	api.Committing: planCommitting,
-	api.CommitWait: planOne(on(SectorProving{}, api.Proving)),
+	api.CommitWait: planOne(
+		on(SectorProving{}, api.Proving),
+		on(SectorCommitFailed{}, api.CommitFailed),
+	),
 
 	api.Proving: planOne(
 		on(SectorFaultReported{}, api.FaultReported),
 		on(SectorFaulty{}, api.Faulty),
 	),
 
+	api.SealFailed: planOne(
+		on(SectorRetrySeal{}, api.Unsealed),
+	),
+	api.PreCommitFailed: planOne(
+		on(SectorRetryPreCommit{}, api.PreCommitting),
+		on(SectorRetryWaitSeed{}, api.WaitSeed),
+		on(SectorSealFailed{}, api.SealFailed),
+	),
+
 	api.Faulty: planOne(
 		on(SectorFaultReported{}, api.FaultReported),
 	),
@@ -61,9 +76,23 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 	/////
 	// First process all events
 
+	for _, event := range events {
+		l := Log{
+			Timestamp: uint64(time.Now().Unix()),
+			Message:   fmt.Sprintf("%+v", event),
+			Kind:      fmt.Sprintf("event;%T", event.User),
+		}
+
+		if err, iserr := event.User.(xerrors.Formatter); iserr {
+			l.Trace = fmt.Sprintf("%+v", err)
+		}
+
+		state.Log = append(state.Log, l)
+	}
+
 	p := fsmPlanners[state.State]
 	if p == nil {
-		return nil, xerrors.Errorf("planner for state %d not found", state.State)
+		return nil, xerrors.Errorf("planner for state %s not found", api.SectorStates[state.State])
 	}
 
 	if err := p(events, state); err != nil {
@@ -87,7 +116,7 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		*   PreCommitting <--> PreCommitFailed
 		|   |                  ^
 		|   v                  |
-		*<- PreCommitted ------/
+		*<- WaitSeed ----------/
 		|   |||
 		|   vvv      v--> SealCommitFailed
 		*<- Committing
@@ -115,8 +144,8 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 		return m.handleUnsealed, nil
 	case api.PreCommitting:
 		return m.handlePreCommitting, nil
-	case api.PreCommitted:
-		return m.handlePreCommitted, nil
+	case api.WaitSeed:
+		return m.handleWaitSeed, nil
 	case api.Committing:
 		return m.handleCommitting, nil
 	case api.CommitWait:
@@ -127,9 +156,9 @@ func (m *Sealing) plan(events []statemachine.Event, state *SectorInfo) (func(sta
 
 	// Handled failure modes
 	case api.SealFailed:
-		log.Warnf("sector %d entered unimplemented state 'SealFailed'", state.SectorID)
+		return m.handleSealFailed, nil
 	case api.PreCommitFailed:
-		log.Warnf("sector %d entered unimplemented state 'PreCommitFailed'", state.SectorID)
+		return m.handlePreCommitFailed, nil
 	case api.SealCommitFailed:
 		log.Warnf("sector %d entered unimplemented state 'SealCommitFailed'", state.SectorID)
 	case api.CommitFailed:
@@ -172,7 +201,7 @@ func planCommitting(events []statemachine.Event, state *SectorInfo) error {
 			e.apply(state)
 			state.State = api.Committing
 			return nil
-		case SectorSealCommitFailed:
+		case SectorComputeProofFailed:
 			state.State = api.SealCommitFailed
 		case SectorSealFailed:
 			state.State = api.CommitFailed
@@ -218,7 +247,7 @@ func planOne(ts ...func() (mut mutator, next api.SectorState)) func(events []sta
 	return func(events []statemachine.Event, state *SectorInfo) error {
 		if len(events) != 1 {
 			for _, event := range events {
-				if gm, ok := event.User.(globalMutator); !ok {
+				if gm, ok := event.User.(globalMutator); ok {
 					gm.applyGlobal(state)
 					return nil
 				}
@@ -226,6 +255,11 @@ func planOne(ts ...func() (mut mutator, next api.SectorState)) func(events []sta
 			return xerrors.Errorf("planner for state %s only has a plan for a single event only, got %+v", api.SectorStates[state.State], events)
 		}
 
+		if gm, ok := events[0].User.(globalMutator); ok {
+			gm.applyGlobal(state)
+			return nil
+		}
+
 		for _, t := range ts {
 			mut, next := t()
 
@@ -233,11 +267,15 @@ func planOne(ts ...func() (mut mutator, next api.SectorState)) func(events []sta
 				continue
 			}
 
+			if err, iserr := events[0].User.(error); iserr {
+				log.Warnf("sector %d got error event %T: %+v", state.SectorID, events[0].User, err)
+			}
+
 			events[0].User.(mutator).apply(state)
 			state.State = next
 			return nil
 		}
 
-		return xerrors.Errorf("planner for state %s received unexpected event %+v", events[0])
+		return xerrors.Errorf("planner for state %s received unexpected event %T (%+v)", api.SectorStates[state.State], events[0].User, events[0])
 	}
 }

storage/sealing/fsm_events.go

@@ -60,6 +60,10 @@ func (evt SectorPacked) apply(state *SectorInfo) {
 	state.Pieces = append(state.Pieces, evt.pieces...)
 }
 
+type SectorPackingFailed struct{ error }
+
+func (evt SectorPackingFailed) apply(*SectorInfo) {}
+
 type SectorSealed struct {
 	commR  []byte
 	commD  []byte
@@ -96,8 +100,12 @@ func (evt SectorSeedReady) apply(state *SectorInfo) {
 	state.Seed = evt.seed
 }
 
-type SectorSealCommitFailed struct{ error }
+type SectorComputeProofFailed struct{ error }
+
 type SectorCommitFailed struct{ error }
+
+func (evt SectorCommitFailed) apply(*SectorInfo) {}
+
 type SectorCommitted struct {
 	message cid.Cid
 	proof   []byte
@@ -112,6 +120,22 @@ type SectorProving struct{}
 
 func (evt SectorProving) apply(*SectorInfo) {}
 
+// Failed state recovery
+
+type SectorRetrySeal struct{}
+
+func (evt SectorRetrySeal) apply(state *SectorInfo) {}
+
+type SectorRetryPreCommit struct{}
+
+func (evt SectorRetryPreCommit) apply(state *SectorInfo) {}
+
+type SectorRetryWaitSeed struct{}
+
+func (evt SectorRetryWaitSeed) apply(state *SectorInfo) {}
+
+// Faults
+
 type SectorFaulty struct{}
 
 func (evt SectorFaulty) apply(state *SectorInfo) {}

storage/sealing/fsm_test.go

@@ -41,7 +41,7 @@ func TestHappyPath(t *testing.T) {
 	require.Equal(m.t, m.state.State, api.PreCommitting)
 
 	m.planSingle(SectorPreCommitted{})
-	require.Equal(m.t, m.state.State, api.PreCommitted)
+	require.Equal(m.t, m.state.State, api.WaitSeed)
 
 	m.planSingle(SectorSeedReady{})
 	require.Equal(m.t, m.state.State, api.Committing)
@@ -67,7 +67,7 @@ func TestSeedRevert(t *testing.T) {
 	require.Equal(m.t, m.state.State, api.PreCommitting)
 
 	m.planSingle(SectorPreCommitted{})
-	require.Equal(m.t, m.state.State, api.PreCommitted)
+	require.Equal(m.t, m.state.State, api.WaitSeed)
 
 	m.planSingle(SectorSeedReady{})
 	require.Equal(m.t, m.state.State, api.Committing)

storage/sealing/sealing.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-sectorbuilder"
-	"github.com/filecoin-project/lotus/lib/padreader"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
@@ -14,9 +13,11 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/events"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/lib/padreader"
 	"github.com/filecoin-project/lotus/lib/statemachine"
 )
 
@@ -28,7 +29,7 @@ type TicketFn func(context.Context) (*sectorbuilder.SealTicket, error)
 
 type sealingApi interface { // TODO: trim down
 	// Call a read only method on actors (no interaction with the chain required)
-	StateCall(ctx context.Context, msg *types.Message, ts *types.TipSet) (*types.MessageReceipt, error)
+	StateCall(context.Context, *types.Message, *types.TipSet) (*api.MethodCall, error)
 	StateMinerWorker(context.Context, address.Address, *types.TipSet) (address.Address, error)
 	StateMinerElectionPeriodStart(ctx context.Context, actor address.Address, ts *types.TipSet) (uint64, error)
 	StateMinerSectors(context.Context, address.Address, *types.TipSet) ([]*api.ChainSectorInfo, error)
@@ -37,6 +38,7 @@ type sealingApi interface { // TODO: trim down
 	StateWaitMsg(context.Context, cid.Cid) (*api.MsgWait, error) // TODO: removeme eventually
 	StateGetActor(ctx context.Context, actor address.Address, ts *types.TipSet) (*types.Actor, error)
 	StateGetReceipt(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error)
+	StateMarketStorageDeal(context.Context, uint64, *types.TipSet) (*actors.OnChainDeal, error)
 
 	MpoolPushMessage(context.Context, *types.Message) (*types.SignedMessage, error)
 
@@ -45,6 +47,7 @@ type sealingApi interface { // TODO: trim down
 	ChainGetRandomness(context.Context, types.TipSetKey, int64) ([]byte, error)
 	ChainGetTipSetByHeight(context.Context, uint64, *types.TipSet) (*types.TipSet, error)
 	ChainGetBlockMessages(context.Context, cid.Cid) (*api.BlockMessages, error)
+	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
 
 	WalletSign(context.Context, address.Address, []byte) (*types.Signature, error)
 	WalletBalance(context.Context, address.Address) (types.BigInt, error)
@@ -80,8 +83,6 @@ func New(api sealingApi, events *events.Events, maddr address.Address, worker ad
 }
 
 func (m *Sealing) Run(ctx context.Context) error {
-	m.events = events.NewEvents(ctx, m.api)
-
 	if err := m.restartSectors(ctx); err != nil {
 		log.Errorf("%+v", err)
 		return xerrors.Errorf("failed load sector states: %w", err)

storage/sealing/states.go

@@ -44,10 +44,24 @@ func (m *Sealing) handlePacking(ctx statemachine.Context, sector SectorInfo) err
 }
 
 func (m *Sealing) handleUnsealed(ctx statemachine.Context, sector SectorInfo) error {
+	if err := checkPieces(ctx.Context(), sector, m.api); err != nil { // Sanity check state
+		switch err.(type) {
+		case *ErrApi:
+			log.Errorf("handleUnsealed: api error, not proceeding: %+v", err)
+			return nil
+		case *ErrInvalidDeals:
+			return ctx.Send(SectorPackingFailed{xerrors.Errorf("invalid deals in sector: %w", err)})
+		case *ErrExpiredDeals: // Probably not much we can do here, maybe re-pack the sector?
+			return ctx.Send(SectorPackingFailed{xerrors.Errorf("expired deals in sector: %w", err)})
+		default:
+			return xerrors.Errorf("checkPieces sanity check error: %w", err)
+		}
+	}
+
 	log.Infow("performing sector replication...", "sector", sector.SectorID)
 	ticket, err := m.tktFn(ctx.Context())
 	if err != nil {
-		return err
+		return ctx.Send(SectorSealFailed{xerrors.Errorf("getting ticket failed: %w", err)})
 	}
 
 	rspco, err := m.sb.SealPreCommit(ctx.Context(), sector.SectorID, *ticket, sector.pieceInfos())
@@ -66,6 +80,20 @@ func (m *Sealing) handleUnsealed(ctx statemachine.Context, sector SectorInfo) er
 }
 
 func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInfo) error {
+	if err := checkSeal(ctx.Context(), m.maddr, sector, m.api); err != nil {
+		switch err.(type) {
+		case *ErrApi:
+			log.Errorf("handlePreCommitting: api error, not proceeding: %+v", err)
+			return nil
+		case *ErrBadCommD: // TODO: Should this just back to packing? (not really needed since handleUnsealed will do that too)
+			return ctx.Send(SectorSealFailed{xerrors.Errorf("bad CommD error: %w", err)})
+		case *ErrExpiredTicket:
+			return ctx.Send(SectorSealFailed{xerrors.Errorf("bad CommD error: %w", err)})
+		default:
+			return xerrors.Errorf("checkSeal sanity check error: %w", err)
+		}
+	}
+
 	params := &actors.SectorPreCommitInfo{
 		SectorNumber: sector.SectorID,
 
@@ -97,7 +125,7 @@ func (m *Sealing) handlePreCommitting(ctx statemachine.Context, sector SectorInf
 	return ctx.Send(SectorPreCommitted{message: smsg.Cid()})
 }
 
-func (m *Sealing) handlePreCommitted(ctx statemachine.Context, sector SectorInfo) error {
+func (m *Sealing) handleWaitSeed(ctx statemachine.Context, sector SectorInfo) error {
 	// would be ideal to just use the events.Called handler, but it wouldnt be able to handle individual message timeouts
 	log.Info("Sector precommitted: ", sector.SectorID)
 	mw, err := m.api.StateWaitMsg(ctx.Context(), *sector.PreCommitMessage)
@@ -147,7 +175,7 @@ func (m *Sealing) handleCommitting(ctx statemachine.Context, sector SectorInfo)
 
 	proof, err := m.sb.SealCommit(ctx.Context(), sector.SectorID, sector.Ticket.SB(), sector.Seed.SB(), sector.pieceInfos(), sector.rspco())
 	if err != nil {
-		return ctx.Send(SectorSealCommitFailed{xerrors.Errorf("computing seal proof failed: %w", err)})
+		return ctx.Send(SectorComputeProofFailed{xerrors.Errorf("computing seal proof failed: %w", err)})
 	}
 
 	// TODO: Consider splitting states and persist proof for faster recovery
@@ -198,8 +226,7 @@ func (m *Sealing) handleCommitWait(ctx statemachine.Context, sector SectorInfo)
 	}
 
 	if mw.Receipt.ExitCode != 0 {
-		log.Errorf("UNHANDLED: submitting sector proof failed (exit=%d, msg=%s) (t:%x; s:%x(%d); p:%x)", mw.Receipt.ExitCode, sector.CommitMessage, sector.Ticket.TicketBytes, sector.Seed.TicketBytes, sector.Seed.BlockHeight, sector.Proof)
-		return xerrors.Errorf("UNHANDLED: submitting sector proof failed (exit: %d)", mw.Receipt.ExitCode)
+		return ctx.Send(SectorCommitFailed{xerrors.Errorf("submitting sector proof failed (exit=%d, msg=%s) (t:%x; s:%x(%d); p:%x)", mw.Receipt.ExitCode, sector.CommitMessage, sector.Ticket.TicketBytes, sector.Seed.TicketBytes, sector.Seed.BlockHeight, sector.Proof)})
 	}
 
 	return ctx.Send(SectorProving{})

storage/sealing/states_failed.go

@@ -0,0 +1,119 @@
+package sealing
+
+import (
+	"bytes"
+	"fmt"
+	"time"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/lib/statemachine"
+)
+
+const minRetryTime = 1 * time.Minute
+
+func failedCooldown(ctx statemachine.Context, sector SectorInfo) error {
+	retryStart := time.Unix(int64(sector.Log[len(sector.Log)-1].Timestamp), 0).Add(minRetryTime)
+	if len(sector.Log) > 0 && !time.Now().After(retryStart) {
+		log.Infof("%s(%d), waiting %s before retrying", api.SectorStates[sector.State], sector.SectorID, time.Until(retryStart))
+		select {
+		case <-time.After(time.Until(retryStart)):
+		case <-ctx.Context().Done():
+			return ctx.Context().Err()
+		}
+	}
+
+	return nil
+}
+
+func (m *Sealing) checkPreCommitted(ctx statemachine.Context, sector SectorInfo) (*actors.PreCommittedSector, bool) {
+	act, err := m.api.StateGetActor(ctx.Context(), m.maddr, nil)
+	if err != nil {
+		log.Errorf("handleSealFailed(%d): temp error: %+v", sector.SectorID, err)
+		return nil, true
+	}
+
+	st, err := m.api.ChainReadObj(ctx.Context(), act.Head)
+	if err != nil {
+		log.Errorf("handleSealFailed(%d): temp error: %+v", sector.SectorID, err)
+		return nil, true
+	}
+
+	var state actors.StorageMinerActorState
+	if err := state.UnmarshalCBOR(bytes.NewReader(st)); err != nil {
+		log.Errorf("handleSealFailed(%d): temp error: unmarshaling miner state: %+v", sector.SectorID, err)
+		return nil, true
+	}
+
+	pci, found := state.PreCommittedSectors[fmt.Sprint(sector.SectorID)]
+	if found {
+		// TODO: If not expired yet, we can just try reusing sealticket
+		log.Warnf("sector %d found in miner preseal array", sector.SectorID)
+		return pci, true
+	}
+
+	return nil, false
+}
+
+func (m *Sealing) handleSealFailed(ctx statemachine.Context, sector SectorInfo) error {
+
+	if _, is := m.checkPreCommitted(ctx, sector); is {
+		// TODO: Remove this after we can re-precommit
+		return nil // noop, for now
+	}
+
+	if err := failedCooldown(ctx, sector); err != nil {
+		return err
+	}
+
+	return ctx.Send(SectorRetrySeal{})
+}
+
+func (m *Sealing) handlePreCommitFailed(ctx statemachine.Context, sector SectorInfo) error {
+	if err := checkSeal(ctx.Context(), m.maddr, sector, m.api); err != nil {
+		switch err.(type) {
+		case *ErrApi:
+			log.Errorf("handlePreCommitFailed: api error, not proceeding: %+v", err)
+			return nil
+		case *ErrBadCommD: // TODO: Should this just back to packing? (not really needed since handleUnsealed will do that too)
+			return ctx.Send(SectorSealFailed{xerrors.Errorf("bad CommD error: %w", err)})
+		case *ErrExpiredTicket:
+			return ctx.Send(SectorSealFailed{xerrors.Errorf("ticket expired error: %w", err)})
+		default:
+			return xerrors.Errorf("checkSeal sanity check error: %w", err)
+		}
+	}
+
+	if pci, is := m.checkPreCommitted(ctx, sector); is && pci != nil {
+		if sector.PreCommitMessage != nil {
+			log.Warn("sector %d is precommitted on chain, but we don't have precommit message", sector.SectorID)
+			return nil // TODO: SeedWait needs this currently
+		}
+
+		if string(pci.Info.CommR) != string(sector.CommR) {
+			log.Warn("sector %d is precommitted on chain, with different CommR: %x != %x", sector.SectorID, pci.Info.CommR, sector.CommR)
+			return nil // TODO: remove when the actor allows re-precommit
+		}
+
+		// TODO: we could compare more things, but I don't think we really need to
+		//  CommR tells us that CommD (and CommPs), and the ticket are all matching
+
+		if err := failedCooldown(ctx, sector); err != nil {
+			return err
+		}
+
+		return ctx.Send(SectorRetryWaitSeed{})
+	}
+
+	if sector.PreCommitMessage != nil {
+		log.Warn("retrying precommit even though the message failed to apply")
+	}
+
+	if err := failedCooldown(ctx, sector); err != nil {
+		return err
+	}
+
+	return ctx.Send(SectorRetryPreCommit{})
+}

storage/sealing/types.go

@@ -2,9 +2,8 @@ package sealing
 
 import (
 	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
-	"github.com/ipfs/go-cid"
-
 	"github.com/filecoin-project/lotus/api"
+	"github.com/ipfs/go-cid"
 )
 
 type SealTicket struct {
@@ -46,10 +45,20 @@ func (p *Piece) ppi() (out sectorbuilder.PublicPieceInfo) {
 	return out
 }
 
+type Log struct {
+	Timestamp uint64
+	Trace     string // for errors
+
+	Message string
+
+	// additional data (Event info)
+	Kind string
+}
+
 type SectorInfo struct {
 	State    api.SectorState
 	SectorID uint64
-	Nonce    uint64
+	Nonce    uint64 // TODO: remove
 
 	// Packing
 
@@ -63,7 +72,7 @@ type SectorInfo struct {
 
 	PreCommitMessage *cid.Cid
 
-	// PreCommitted
+	// WaitSeed
 	Seed SealSeed
 
 	// Committing
@@ -75,7 +84,7 @@ type SectorInfo struct {
 	// Debug
 	LastErr string
 
-	// TODO: Log []struct{ts, msg, trace string}
+	Log []Log
 }
 
 func (t *SectorInfo) pieceInfos() []sectorbuilder.PublicPieceInfo {

tools/stats/chain.dashboard.json

@@ -62,7 +62,7 @@
   "panels": [
     {
       "aliasColors": {},
-      "bars": false,
+      "bars": true,
       "dashLength": 10,
       "dashes": false,
       "datasource": "${DS_INFLUXDB}",
@@ -70,7 +70,7 @@
       "fill": 3,
       "fillGradient": 0,
       "gridPos": {
-        "h": 16,
+        "h": 9,
         "w": 24,
         "x": 0,
         "y": 0
@@ -80,18 +80,18 @@
       "interval": "",
       "legend": {
         "alignAsTable": true,
-        "avg": false,
-        "current": true,
+        "avg": true,
+        "current": false,
         "max": false,
         "min": false,
         "rightSide": true,
         "show": true,
-        "sort": "current",
+        "sort": "avg",
         "sortDesc": true,
         "total": false,
         "values": true
       },
-      "lines": true,
+      "lines": false,
       "linewidth": 1,
       "nullPointMode": "null",
       "options": {
@@ -131,7 +131,7 @@
           "measurement": "chain.election",
           "orderByTime": "ASC",
           "policy": "default",
-          "query": "SELECT moving_average(count(\"value\"), 10) FROM \"chain.election\" WHERE $timeFilter -10m GROUP BY time($__interval), \"miner\" fill(null)",
+          "query": "SELECT count(\"value\") FROM \"chain.election\" WHERE $timeFilter -10m GROUP BY time($__interval), \"miner\" fill(null)",
           "rawQuery": true,
           "refId": "A",
           "resultFormat": "time_series",
@@ -214,9 +214,10 @@
         "h": 4,
         "w": 8,
         "x": 0,
-        "y": 16
+        "y": 9
       },
       "id": 22,
+      "interval": "",
       "legend": {
         "avg": false,
         "current": false,
@@ -268,7 +269,7 @@
         }
       ],
       "thresholds": [],
-      "timeFrom": null,
+      "timeFrom": "4h",
       "timeRegions": [],
       "timeShift": null,
       "title": "Total Power",
@@ -287,9 +288,8 @@
       },
       "yaxes": [
         {
-          "decimals": 2,
           "format": "bytes",
-          "label": "",
+          "label": null,
           "logBase": 1,
           "max": null,
           "min": null,
@@ -329,9 +329,9 @@
       },
       "gridPos": {
         "h": 4,
-        "w": 8,
+        "w": 4,
         "x": 8,
-        "y": 16
+        "y": 9
       },
       "id": 12,
       "interval": null,
@@ -413,6 +413,122 @@
       ],
       "valueName": "avg"
     },
+    {
+      "cacheTimeout": null,
+      "colorBackground": false,
+      "colorValue": false,
+      "colors": [
+        "#299c46",
+        "rgba(237, 129, 40, 0.89)",
+        "#d44a3a"
+      ],
+      "datasource": "${DS_INFLUXDB}",
+      "format": "bytes",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 4,
+        "x": 12,
+        "y": 9
+      },
+      "id": 42,
+      "interval": "",
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "options": {},
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": true,
+        "ymax": null,
+        "ymin": 0
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "SELECT sum(\"value\") FROM \"chain.miner_power\" WHERE $timeFilter GROUP BY time(45s)",
+          "rawQuery": true,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "mean"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "thresholds": "",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Network Storage",
+      "type": "singlestat",
+      "valueFontSize": "80%",
+      "valueMaps": [
+        {
+          "op": "=",
+          "text": "N/A",
+          "value": "null"
+        }
+      ],
+      "valueName": "current"
+    },
     {
       "cacheTimeout": null,
       "colorBackground": false,
@@ -435,7 +551,7 @@
         "h": 4,
         "w": 8,
         "x": 16,
-        "y": 16
+        "y": 9
       },
       "id": 6,
       "interval": null,
@@ -477,8 +593,15 @@
       "tableColumn": "",
       "targets": [
         {
-          "groupBy": [],
-          "measurement": "chain.block_count",
+          "groupBy": [
+            {
+              "params": [
+                "$interval"
+              ],
+              "type": "time"
+            }
+          ],
+          "measurement": "chain.election",
           "orderByTime": "ASC",
           "policy": "default",
           "refId": "A",
@@ -490,6 +613,10 @@
                   "value"
                 ],
                 "type": "field"
+              },
+              {
+                "params": [],
+                "type": "sum"
               }
             ]
           ],
@@ -509,7 +636,7 @@
           "value": "null"
         }
       ],
-      "valueName": "current"
+      "valueName": "avg"
     },
     {
       "cacheTimeout": null,
@@ -534,7 +661,7 @@
         "h": 3,
         "w": 4,
         "x": 0,
-        "y": 20
+        "y": 13
       },
       "id": 4,
       "interval": null,
@@ -632,7 +759,7 @@
         "h": 3,
         "w": 4,
         "x": 4,
-        "y": 20
+        "y": 13
       },
       "id": 14,
       "interval": null,
@@ -734,7 +861,7 @@
         "h": 3,
         "w": 4,
         "x": 8,
-        "y": 20
+        "y": 13
       },
       "id": 32,
       "interval": null,
@@ -849,7 +976,7 @@
         "h": 3,
         "w": 4,
         "x": 12,
-        "y": 20
+        "y": 13
       },
       "id": 20,
       "interval": null,
@@ -964,7 +1091,7 @@
         "h": 3,
         "w": 4,
         "x": 16,
-        "y": 20
+        "y": 13
       },
       "id": 8,
       "interval": null,
@@ -1079,7 +1206,7 @@
         "h": 3,
         "w": 4,
         "x": 20,
-        "y": 20
+        "y": 13
       },
       "id": 10,
       "interval": null,
@@ -1131,7 +1258,7 @@
             },
             {
               "params": [
-                "0"
+                "null"
               ],
               "type": "fill"
             }
@@ -1139,8 +1266,8 @@
           "measurement": "chain.message_count",
           "orderByTime": "ASC",
           "policy": "default",
-          "query": "SELECT sum(\"value\") FROM (SELECT \"value\" FROM \"chain.message_count\" GROUP BY \"height\") WHERE $timeFilter GROUP BY time($__interval) fill(0)",
-          "rawQuery": true,
+          "query": "SELECT \"value\" FROM \"chain.message_count\" WHERE $timeFilter ",
+          "rawQuery": false,
           "refId": "A",
           "resultFormat": "time_series",
           "select": [
@@ -1153,7 +1280,7 @@
               },
               {
                 "params": [],
-                "type": "mean"
+                "type": "sum"
               }
             ]
           ],
@@ -1198,7 +1325,7 @@
         "h": 3,
         "w": 4,
         "x": 0,
-        "y": 23
+        "y": 16
       },
       "id": 16,
       "interval": "",
@@ -1293,16 +1420,16 @@
         "h": 3,
         "w": 16,
         "x": 4,
-        "y": 23
+        "y": 16
       },
       "id": 2,
       "legend": {
         "alignAsTable": true,
         "avg": true,
-        "current": false,
+        "current": true,
         "hideEmpty": false,
         "hideZero": false,
-        "max": false,
+        "max": true,
         "min": false,
         "rightSide": true,
         "show": true,
@@ -1321,10 +1448,6 @@
       "points": false,
       "renderer": "flot",
       "seriesOverrides": [
-        {
-          "alias": "chain.height.difference",
-          "yaxis": 2
-        },
         {
           "alias": "Null Blocks",
           "yaxis": 2
@@ -1399,7 +1522,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "",
+      "title": "Tipsets",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -1416,7 +1539,7 @@
       "yaxes": [
         {
           "format": "s",
-          "label": "Time delta",
+          "label": "Time between tipsets",
           "logBase": 1,
           "max": null,
           "min": null,
@@ -1425,7 +1548,7 @@
         {
           "decimals": 0,
           "format": "short",
-          "label": "# null blocks",
+          "label": "Number of Null blocks",
           "logBase": 1,
           "max": null,
           "min": "0",
@@ -1459,7 +1582,7 @@
         "h": 3,
         "w": 4,
         "x": 20,
-        "y": 23
+        "y": 16
       },
       "id": 30,
       "interval": null,
@@ -1540,10 +1663,10 @@
       "datasource": "${DS_INFLUXDB}",
       "fontSize": "100%",
       "gridPos": {
-        "h": 16,
+        "h": 21,
         "w": 4,
         "x": 0,
-        "y": 26
+        "y": 19
       },
       "id": 28,
       "options": {},
@@ -1627,13 +1750,13 @@
       "dashLength": 10,
       "dashes": false,
       "datasource": "${DS_INFLUXDB}",
-      "fill": 4,
+      "fill": 5,
       "fillGradient": 0,
       "gridPos": {
         "h": 8,
         "w": 12,
         "x": 4,
-        "y": 26
+        "y": 19
       },
       "id": 40,
       "interval": "",
@@ -1641,8 +1764,8 @@
         "alignAsTable": true,
         "avg": false,
         "current": true,
-        "hideEmpty": false,
-        "hideZero": false,
+        "hideEmpty": true,
+        "hideZero": true,
         "max": false,
         "min": false,
         "rightSide": true,
@@ -1765,10 +1888,10 @@
       "datasource": "${DS_INFLUXDB}",
       "fontSize": "100%",
       "gridPos": {
-        "h": 16,
+        "h": 21,
         "w": 8,
         "x": 16,
-        "y": 26
+        "y": 19
       },
       "id": 18,
       "options": {},
@@ -1862,10 +1985,10 @@
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
-        "h": 8,
+        "h": 6,
         "w": 12,
         "x": 4,
-        "y": 34
+        "y": 27
       },
       "id": 24,
       "legend": {
@@ -1981,6 +2104,128 @@
     {
       "aliasColors": {},
       "bars": false,
+      "cacheTimeout": null,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_INFLUXDB}",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 4,
+        "y": 33
+      },
+      "id": 44,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "previous"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "chain.miner_power",
+          "orderByTime": "ASC",
+          "policy": "default",
+          "query": "SELECT count(\"value\") FROM \"chain.miner_power\" WHERE $timeFilter GROUP BY time($__interval)",
+          "rawQuery": false,
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "count"
+              }
+            ]
+          ],
+          "tags": []
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Miners on Chain",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": 0,
+          "format": "none",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": true,
       "dashLength": 10,
       "dashes": false,
       "datasource": "${DS_INFLUXDB}",
@@ -1990,7 +2235,7 @@
         "h": 9,
         "w": 12,
         "x": 0,
-        "y": 42
+        "y": 40
       },
       "id": 34,
       "legend": {
@@ -2004,7 +2249,7 @@
         "total": false,
         "values": false
       },
-      "lines": true,
+      "lines": false,
       "linewidth": 1,
       "nullPointMode": "null",
       "options": {
@@ -2050,8 +2295,6 @@
           "measurement": "chain.message_count",
           "orderByTime": "ASC",
           "policy": "default",
-          "query": "SELECT moving_average(count(\"value\"), 2) FROM \"chain.message_count\" WHERE $timeFilter -4m GROUP BY time($__interval), \"actor\", \"method\" fill(null)",
-          "rawQuery": true,
           "refId": "A",
           "resultFormat": "time_series",
           "select": [
@@ -2064,7 +2307,7 @@
               },
               {
                 "params": [],
-                "type": "count"
+                "type": "sum"
               }
             ]
           ],
@@ -2114,35 +2357,35 @@
     },
     {
       "aliasColors": {},
-      "bars": false,
+      "bars": true,
       "dashLength": 10,
       "dashes": false,
       "datasource": "${DS_INFLUXDB}",
-      "decimals": 2,
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
         "h": 9,
         "w": 12,
         "x": 12,
-        "y": 42
+        "y": 40
       },
       "id": 36,
-      "interval": "",
       "legend": {
         "alignAsTable": true,
-        "avg": true,
+        "avg": false,
         "current": false,
-        "max": true,
+        "hideEmpty": true,
+        "hideZero": true,
+        "max": false,
         "min": false,
         "rightSide": true,
         "show": true,
         "sort": "avg",
-        "sortDesc": true,
+        "sortDesc": false,
         "total": false,
-        "values": true
+        "values": false
       },
-      "lines": true,
+      "lines": false,
       "linewidth": 1,
       "nullPointMode": "null",
       "options": {
@@ -2166,6 +2409,12 @@
               ],
               "type": "time"
             },
+            {
+              "params": [
+                "method"
+              ],
+              "type": "tag"
+            },
             {
               "params": [
                 "exitcode"
@@ -2178,12 +2427,6 @@
               ],
               "type": "tag"
             },
-            {
-              "params": [
-                "method"
-              ],
-              "type": "tag"
-            },
             {
               "params": [
                 "null"
@@ -2194,7 +2437,7 @@
           "measurement": "chain.message_count",
           "orderByTime": "ASC",
           "policy": "default",
-          "query": "SELECT moving_average(count(\"value\"), 10) FROM \"chain.message_count\" WHERE $timeFilter -10m GROUP BY time($__interval), \"exitcode\", \"actor\", \"method\" fill(null)",
+          "query": "SELECT sum(\"value\") FROM \"chain.message_count\" WHERE $timeFilter GROUP BY time($__interval), \"method\", \"exitcode\", \"actor\" fill(null)",
           "rawQuery": true,
           "refId": "A",
           "resultFormat": "time_series",
@@ -2209,12 +2452,6 @@
               {
                 "params": [],
                 "type": "count"
-              },
-              {
-                "params": [
-                  10
-                ],
-                "type": "moving_average"
               }
             ]
           ],
@@ -2241,7 +2478,6 @@
       },
       "yaxes": [
         {
-          "decimals": null,
           "format": "short",
           "label": "",
           "logBase": 1,
@@ -2264,7 +2500,7 @@
       }
     }
   ],
-  "refresh": "30s",
+  "refresh": "45s",
   "schemaVersion": 20,
   "style": "dark",
   "tags": [],
@@ -2280,6 +2516,7 @@
       "5s",
       "10s",
       "30s",
+      "45s",
       "1m",
       "5m",
       "15m",
@@ -2292,5 +2529,5 @@
   "timezone": "",
   "title": "Chain",
   "uid": "z6FtI92Zz",
-  "version": 8
-}
+  "version": 9
+}