Merge pull request #1116 from filecoin-project/fix/v0.2.5

Correctly update proving set & sector set with amt2

.circleci/config.yml

@@ -100,7 +100,7 @@ jobs:
         description: gotestsum format. https://github.com/gotestyourself/gotestsum#format
       coverage:
         type: string
-        default: -coverprofile=coverage.txt
+        default: -coverprofile=coverage.txt -coverpkg=github.com/filecoin-project/lotus/...
         description: Coverage flag. Set to the empty string to disable.
       codecov-upload:
         type: boolean
@@ -190,7 +190,7 @@ jobs:
       - install-deps
       - go/mod-download
       - run:
-          command: make buildall
+          command: make build
           no_output_timeout: 30m
       - store_artifacts:
           path: lotus
@@ -253,7 +253,8 @@ workflows:
     jobs:
       - lint-changes:
           args: "--new-from-rev origin/master"
-      - test
+      - test:
+          codecov-upload: true
       - test-short:
           go-test-flags: "--timeout 10m --short"
       - mod-tidy-check

.codecov.yml

@@ -1 +1,3 @@
 comment: off
+ignore:
+  - "cbor_gen.go"

Makefile

@@ -14,7 +14,7 @@ MODULES:=
 
 CLEAN:=
 BINS:=
-GOFLAGS+=-ldflags="-X "github.com/filecoin-project/lotus/build".CurrentCommit=+git$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null))"
+GOFLAGS+=-ldflags='-X="github.com/filecoin-project/lotus/build".CurrentCommit=+git$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null))'
 
 ## FFI
 
@@ -76,6 +76,7 @@ BINS+=lotus-seal-worker
 lotus-shed: $(BUILD_DEPS)
 	rm -f lotus-shed
 	go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-shed -i ./build
 .PHONY: lotus-seal-worker
 BINS+=lotus-seal-worker
 
@@ -123,14 +124,14 @@ BINS+=townhall
 fountain:
 	rm -f fountain
 	go build -o fountain ./cmd/lotus-fountain
-	go run github.com/GeertJohan/go.rice/rice append --exec fountain -i ./cmd/lotus-fountain
+	go run github.com/GeertJohan/go.rice/rice append --exec fountain -i ./cmd/lotus-fountain -i ./build
 .PHONY: fountain
 BINS+=fountain
 
 chainwatch:
 	rm -f chainwatch
 	go build -o chainwatch ./cmd/lotus-chainwatch
-	go run github.com/GeertJohan/go.rice/rice append --exec chainwatch -i ./cmd/lotus-chainwatch
+	go run github.com/GeertJohan/go.rice/rice append --exec chainwatch -i ./cmd/lotus-chainwatch -i ./build
 .PHONY: chainwatch
 BINS+=chainwatch
 
@@ -144,9 +145,18 @@ BINS+=bench
 stats:
 	rm -f stats
 	go build -o stats ./tools/stats
+	go run github.com/GeertJohan/go.rice/rice append --exec stats -i ./build
 .PHONY: stats
 BINS+=stats
 
+health:
+	rm -f lotus-health
+	go build -o lotus-health ./cmd/lotus-health
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-health -i ./build
+
+.PHONY: health
+BINS+=health
+
 # MISC
 
 buildall: $(BINS)

api/api_full.go

@@ -37,6 +37,7 @@ type FullNode interface {
 	ChainTipSetWeight(context.Context, *types.TipSet) (types.BigInt, error)
 	ChainGetNode(ctx context.Context, p string) (interface{}, error)
 	ChainGetMessage(context.Context, cid.Cid) (*types.Message, error)
+	ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*store.HeadChange, error)
 
 	// syncer
 	SyncState(context.Context) (*SyncState, error)
@@ -118,6 +119,7 @@ type FullNode interface {
 	StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error)
 	StateGetReceipt(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error)
 	StateMinerSectorCount(context.Context, address.Address, *types.TipSet) (MinerSectors, error)
+	StateCompute(context.Context, uint64, []*types.Message, *types.TipSet) (cid.Cid, error)
 
 	MarketEnsureAvailable(context.Context, address.Address, types.BigInt) error
 	// MarketFreeBalance

api/api_storage.go

@@ -13,7 +13,8 @@ type SectorState = uint64
 const (
 	UndefinedSectorState SectorState = iota
 
-	Empty   // TODO: Is this useful
+	// happy path
+	Empty
 	Packing // sector not in sealStore, and not on chain
 
 	Unsealed      // sealing / queued
@@ -22,13 +23,32 @@ const (
 	Committing
 	CommitWait // waiting for message to land on chain
 	Proving
+	_ // reserved
+	_
+	_
+	_
+
+	// recovery handling
+	// Reseal
+	_
+	_
+	_
+	_
+	_
+	_
+	_
+
+	// error modes
+	FailedUnrecoverable
 
 	SealFailed
 	PreCommitFailed
 	SealCommitFailed
 	CommitFailed
-
-	FailedUnrecoverable
+	_
+	_
+	_
+	_
 
 	Faulty        // sector is corrupted or gone for some reason
 	FaultReported // sector has been declared as a fault on chain

api/apistruct/struct.go

@@ -40,21 +40,22 @@ type FullNodeStruct struct {
 	CommonStruct
 
 	Internal struct {
-		ChainNotify            func(context.Context) (<-chan []*store.HeadChange, error)           `perm:"read"`
-		ChainHead              func(context.Context) (*types.TipSet, error)                        `perm:"read"`
-		ChainGetRandomness     func(context.Context, types.TipSetKey, int64) ([]byte, error)       `perm:"read"`
-		ChainGetBlock          func(context.Context, cid.Cid) (*types.BlockHeader, error)          `perm:"read"`
-		ChainGetTipSet         func(context.Context, types.TipSetKey) (*types.TipSet, error)       `perm:"read"`
-		ChainGetBlockMessages  func(context.Context, cid.Cid) (*api.BlockMessages, error)          `perm:"read"`
-		ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error)     `perm:"read"`
-		ChainGetParentMessages func(context.Context, cid.Cid) ([]api.Message, error)               `perm:"read"`
-		ChainGetTipSetByHeight func(context.Context, uint64, *types.TipSet) (*types.TipSet, error) `perm:"read"`
-		ChainReadObj           func(context.Context, cid.Cid) ([]byte, error)                      `perm:"read"`
-		ChainSetHead           func(context.Context, *types.TipSet) error                          `perm:"admin"`
-		ChainGetGenesis        func(context.Context) (*types.TipSet, error)                        `perm:"read"`
-		ChainTipSetWeight      func(context.Context, *types.TipSet) (types.BigInt, error)          `perm:"read"`
-		ChainGetNode           func(ctx context.Context, p string) (interface{}, error)            `perm:"read"`
-		ChainGetMessage        func(context.Context, cid.Cid) (*types.Message, error)              `perm:"read"`
+		ChainNotify            func(context.Context) (<-chan []*store.HeadChange, error)                            `perm:"read"`
+		ChainHead              func(context.Context) (*types.TipSet, error)                                         `perm:"read"`
+		ChainGetRandomness     func(context.Context, types.TipSetKey, int64) ([]byte, error)                        `perm:"read"`
+		ChainGetBlock          func(context.Context, cid.Cid) (*types.BlockHeader, error)                           `perm:"read"`
+		ChainGetTipSet         func(context.Context, types.TipSetKey) (*types.TipSet, error)                        `perm:"read"`
+		ChainGetBlockMessages  func(context.Context, cid.Cid) (*api.BlockMessages, error)                           `perm:"read"`
+		ChainGetParentReceipts func(context.Context, cid.Cid) ([]*types.MessageReceipt, error)                      `perm:"read"`
+		ChainGetParentMessages func(context.Context, cid.Cid) ([]api.Message, error)                                `perm:"read"`
+		ChainGetTipSetByHeight func(context.Context, uint64, *types.TipSet) (*types.TipSet, error)                  `perm:"read"`
+		ChainReadObj           func(context.Context, cid.Cid) ([]byte, error)                                       `perm:"read"`
+		ChainSetHead           func(context.Context, *types.TipSet) error                                           `perm:"admin"`
+		ChainGetGenesis        func(context.Context) (*types.TipSet, error)                                         `perm:"read"`
+		ChainTipSetWeight      func(context.Context, *types.TipSet) (types.BigInt, error)                           `perm:"read"`
+		ChainGetNode           func(ctx context.Context, p string) (interface{}, error)                             `perm:"read"`
+		ChainGetMessage        func(context.Context, cid.Cid) (*types.Message, error)                               `perm:"read"`
+		ChainGetPath           func(context.Context, types.TipSetKey, types.TipSetKey) ([]*store.HeadChange, error) `perm:"read"`
 
 		SyncState          func(context.Context) (*api.SyncState, error)                `perm:"read"`
 		SyncSubmitBlock    func(ctx context.Context, blk *types.BlockMsg) error         `perm:"write"`
@@ -114,6 +115,7 @@ type FullNodeStruct struct {
 		StateGetReceipt               func(context.Context, cid.Cid, *types.TipSet) (*types.MessageReceipt, error)                      `perm:"read"`
 		StateMinerSectorCount         func(context.Context, address.Address, *types.TipSet) (api.MinerSectors, error)                   `perm:"read"`
 		StateListMessages             func(ctx context.Context, match *types.Message, ts *types.TipSet, toht uint64) ([]cid.Cid, error) `perm:"read"`
+		StateCompute                  func(context.Context, uint64, []*types.Message, *types.TipSet) (cid.Cid, error)                   `perm:"read"`
 
 		MarketEnsureAvailable func(context.Context, address.Address, types.BigInt) error `perm:"sign"`
 
@@ -355,6 +357,10 @@ func (c *FullNodeStruct) ChainGetMessage(ctx context.Context, mc cid.Cid) (*type
 	return c.Internal.ChainGetMessage(ctx, mc)
 }
 
+func (c *FullNodeStruct) ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*store.HeadChange, error) {
+	return c.Internal.ChainGetPath(ctx, from, to)
+}
+
 func (c *FullNodeStruct) SyncState(ctx context.Context) (*api.SyncState, error) {
 	return c.Internal.SyncState(ctx)
 }
@@ -462,6 +468,10 @@ func (c *FullNodeStruct) StateListMessages(ctx context.Context, match *types.Mes
 	return c.Internal.StateListMessages(ctx, match, ts, toht)
 }
 
+func (c *FullNodeStruct) StateCompute(ctx context.Context, height uint64, msgs []*types.Message, ts *types.TipSet) (cid.Cid, error) {
+	return c.Internal.StateCompute(ctx, height, msgs, ts)
+}
+
 func (c *FullNodeStruct) MarketEnsureAvailable(ctx context.Context, addr address.Address, amt types.BigInt) error {
 	return c.Internal.MarketEnsureAvailable(ctx, addr, amt)
 }

api/cbor_gen.go

@@ -23,6 +23,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Channel (address.Address) (struct)
+	if len("Channel") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Channel\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Channel")))); err != nil {
 		return err
 	}
@@ -35,6 +39,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.ChannelMessage (cid.Cid) (struct)
+	if len("ChannelMessage") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"ChannelMessage\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("ChannelMessage")))); err != nil {
 		return err
 	}
@@ -53,6 +61,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Vouchers ([]*types.SignedVoucher) (slice)
+	if len("Vouchers") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Vouchers\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Vouchers")))); err != nil {
 		return err
 	}
@@ -60,6 +72,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
+	if len(t.Vouchers) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.Vouchers was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Vouchers)))); err != nil {
 		return err
 	}
@@ -200,6 +216,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.SectorID (uint64) (uint64)
+	if len("SectorID") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"SectorID\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("SectorID")))); err != nil {
 		return err
 	}
@@ -212,6 +232,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Offset (uint64) (uint64)
+	if len("Offset") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Offset\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Offset")))); err != nil {
 		return err
 	}
@@ -224,6 +248,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Size (uint64) (uint64)
+	if len("Size") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Size\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Size")))); err != nil {
 		return err
 	}
@@ -336,6 +364,10 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Refs ([]api.SealedRef) (slice)
+	if len("Refs") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Refs\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Refs")))); err != nil {
 		return err
 	}
@@ -343,6 +375,10 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
+	if len(t.Refs) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.Refs was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Refs)))); err != nil {
 		return err
 	}

api/test/deals.go

@@ -11,7 +11,7 @@ import (
 	"testing"
 	"time"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
@@ -24,7 +24,7 @@ func init() {
 	build.InsecurePoStValidation = true
 }
 
-func TestDealFlow(t *testing.T, b APIBuilder) {
+func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
@@ -64,7 +64,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) {
 	go func() {
 		defer close(done)
 		for mine {
-			time.Sleep(time.Second)
+			time.Sleep(blocktime)
 			fmt.Println("mining a block now")
 			if err := sn[0].MineOne(ctx); err != nil {
 				t.Error(err)

build/bootstrap/bootstrappers.pi

@@ -1,6 +1,6 @@
-/dns4/lotus-bootstrap-0.dfw.fil-test.net/tcp/1347/p2p/12D3KooWHwGBSiLR5ts7KW9MgH4BMzC2iXe18kwAQ8Ee3LUd1jeR
-/dns4/lotus-bootstrap-1.dfw.fil-test.net/tcp/1347/p2p/12D3KooWCLFaawdhLGcSpiqg43DtZ9QzPQ6HcB8Vvyu2Cnta8UWc
-/dns4/lotus-bootstrap-0.fra.fil-test.net/tcp/1347/p2p/12D3KooWMmaL7eaUCF6tVAghVmgozxz4uztbuFUQv6dyFpHRarHR
-/dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWLLpNYoKdf9NgcWudBhXLdTcXncqAsTzozw1scMMu6nS5
-/dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWCNL9vXaXwNs3Bu8uRAJK4pxpCyPeM7jZLSDpJma1wrV8
-/dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWNGGxFda1eC5U2YKAgs4ypoFHn3Z3xHCsjmFdrCcytoxm
+/dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWLZs8BWtEzRTYET4yR4jzDtPamaA1YsyPQJq6cf2RfxBD
+/dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWGvrgjWw4Yqo4AFWqYp4g37FpUvUCQBkNWudZVSwR9tY1
+/dns4/lotus-bootstrap-0.fra.fil-test.net/tcp/1347/p2p/12D3KooWSfNcrD1cs5Cj5eSHbK6nHCqJLffAuPqvRMBRgvUdqQhX
+/dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWNkXyVPspUnrHUiSC3VJPMcXvHuNdy3BTCLTPPnDgwwTT
+/dns4/lotus-bootstrap-0.dfw.fil-test.net/tcp/1347/p2p/12D3KooWSgJWJZK8LTRtCWzPa5FQheCFJjHpficVYgEQWeimcqCu
+/dns4/lotus-bootstrap-1.dfw.fil-test.net/tcp/1347/p2p/12D3KooWFPaC4dyGpbNXCpVHjZucdJnDwmv4ng9tponPx5GrzJkT

build/forks.go

@@ -1,4 +1,7 @@
 package build
 
-const ForkCCM = 1750
-const ForkNoPowerEPSUpdates = 16450
+const ForkBlizzardHeight = 6288
+
+const ForkFrigidHeight = 7950
+
+const ForkBootyBayHeight = 11000

build/genesis.go

@@ -2,7 +2,7 @@ package build
 
 import (
 	rice "github.com/GeertJohan/go.rice"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 )
 
 // moved from now-defunct build/paramfetch.go
@@ -11,12 +11,12 @@ var log = logging.Logger("build")
 func MaybeGenesis() []byte {
 	builtinGen, err := rice.FindBox("genesis")
 	if err != nil {
-		log.Warn("loading built-in genesis: %s", err)
+		log.Warnf("loading built-in genesis: %s", err)
 		return nil
 	}
 	genBytes, err := builtinGen.Bytes("devnet.car")
 	if err != nil {
-		log.Warn("loading built-in genesis: %s", err)
+		log.Warnf("loading built-in genesis: %s", err)
 	}
 
 	return genBytes

build/parameters.go

@@ -0,0 +1,5 @@
+package build
+
+import rice "github.com/GeertJohan/go.rice"
+
+var ParametersJson = rice.MustFindBox("proof-params").MustBytes("parameters.json")

build/params_debug.go

@@ -2,8 +2,6 @@
 
 package build
 
-import "os"
-
 var SectorSizes = []uint64{1024}
 
 // Seconds
@@ -31,7 +29,3 @@ const InteractivePoRepConfidence = 6
 
 // Bytes
 var MinimumMinerPower uint64 = 2 << 10 // 2KiB
-
-func init() {
-	os.Setenv("TRUST_PARAMS", "1")
-}

build/params_shared.go

@@ -56,6 +56,9 @@ const SealRandomnessLookback = Finality
 // Epochs
 const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000
 
+// Maximum lookback that randomness can be sourced from for a seal proof submission
+const MaxSealLookback = SealRandomnessLookbackLimit + 2000
+
 // /////
 // Mining
 
@@ -97,3 +100,9 @@ const BadBlockCacheSize = 1 << 15
 // assuming 4000 messages per round, this lets us not lose any messages across a
 // 10 block reorg.
 const BlsSignatureCacheSize = 40000
+
+// ///////
+// Limits
+
+const BlockMessageLimit = 512
+const MinerMaxSectors = 1 << 48

build/params_testnet.go

@@ -3,7 +3,6 @@
 package build
 
 var SectorSizes = []uint64{
-	1 << 30,
 	32 << 30,
 }
 

build/proof-params/parameters.json

@@ -0,0 +1,103 @@
+{
+  "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.params": {
+    "cid": "QmX7tYeNPWae2fjZ3Am6GB9dmHvLqvoz8dKo3PR98VYxH9",
+    "digest": "39a9edec3355516674f0d12b926be493",
+    "sector_size": 34359738368
+  },
+  "v20-proof-of-spacetime-election-5f585aca354eb68e411c8582ed0efd800792430e4e76d73468c4fc03f1a8d6d2.vk": {
+    "cid": "QmbNGx7pNbGiEr8ykoHxVXHW2LNSmGdsxKtj1onZCyguCX",
+    "digest": "0227ae7df4f2affe529ebafbbc7540ee",
+    "sector_size": 34359738368
+  },
+  "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.params": {
+    "cid": "QmRGZsNp4mp1cZshcXqt3VMuWscAEsiMa2iepF4CsWWoiv",
+    "digest": "991041a354b12c280542741f58c7f2ca",
+    "sector_size": 1024
+  },
+  "v20-proof-of-spacetime-election-a4e18190d4b4657ba1b4d08a341871b2a6f398e327cb9951b28ab141fbdbf49d.vk": {
+    "cid": "QmWpmrhCGVcfqLyqp5oGAnhPmCE5hGTPaauHi25mpQwRSU",
+    "digest": "91fac550e1f9bccab213830bb0c85bd6",
+    "sector_size": 1024
+  },
+  "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.params": {
+    "cid": "QmenSZXh1EsSyHiSRvA6wb8yaPhYBTjrKehJw96Px5HnN4",
+    "digest": "6322eacd2773163ddd51f9ca7d645fc4",
+    "sector_size": 1073741824
+  },
+  "v20-proof-of-spacetime-election-a9eb6d90b896a282ec2d3a875c6143e3fcff778f0da1460709e051833651559b.vk": {
+    "cid": "QmPvZoMKofw6eDhDg5ESJA2QAZP8HvM6qMQk7fw4pq9bQf",
+    "digest": "0df62745fceac922e3e70847cfc70b52",
+    "sector_size": 1073741824
+  },
+  "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.params": {
+    "cid": "QmVibFqzkZoL8cwQmzj8njPokCQGCCx4pBcUH77bzgJgV9",
+    "digest": "de9d71e672f286706a1673bd57abdaac",
+    "sector_size": 16777216
+  },
+  "v20-proof-of-spacetime-election-bf872523641b1de33553db2a177df13e412d7b3b0103e6696ae0a1cf5d525259.vk": {
+    "cid": "QmZa5FX27XyiEXQQLQpHqtMJKLzrcY8wMuj3pxzmSimSyu",
+    "digest": "7f796d3a0f13499181e44b5eee0cc744",
+    "sector_size": 16777216
+  },
+  "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.params": {
+    "cid": "Qmbt2SWWAmMcYoY3DAiRDXA8fAuqdqRLWucJMSxYmzBCmN",
+    "digest": "151ae0ae183fc141e8c2bebc28e5cc10",
+    "sector_size": 268435456
+  },
+  "v20-proof-of-spacetime-election-ffc3fb192364238b60977839d14e3154d4a98313e30d46694a12af54b6874975.vk": {
+    "cid": "QmUxvPu4xdVmjMFihUKoYyEdXBqxsXkvmxRweU7KouWHji",
+    "digest": "95eb89588e9d1832aca044c3a13178af",
+    "sector_size": 268435456
+  },
+  "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.params": {
+    "cid": "QmQZe8eLo2xXbhSDxtyYZNqEjqjdcWGdADywECRvNEZQdX",
+    "digest": "fcd50e2e08a8560a6bb3418e883567ed",
+    "sector_size": 268435456
+  },
+  "v20-stacked-proof-of-replication-117839dacd1ef31e5968a6fd13bcd6fa86638d85c40c9241a1d07c2a954eb89b.vk": {
+    "cid": "Qme1hn6QT1covfoUFGDZkqoE1pMTax9FNW3nWWmTNqFe7y",
+    "digest": "872e244d86499fd659082e3bcf3f13e7",
+    "sector_size": 268435456
+  },
+  "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.params": {
+    "cid": "QmSfrPDC9jwY4MKrjzhCqDBBAG44wSDM8oE5NuDwWSh2xN",
+    "digest": "0a338b941c5f17946340de5fc95cab30",
+    "sector_size": 34359738368
+  },
+  "v20-stacked-proof-of-replication-b46f3a1051afbb67f70aae7082da95def62eee943662f3e1bf69837fb08aaae4.vk": {
+    "cid": "QmTDGynCmnbaZNBP3Bv3F3duC3ecKRubCKeMUiQQZYbGpF",
+    "digest": "c752e070a6b7aa8b79aa661a6b600b55",
+    "sector_size": 34359738368
+  },
+  "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.params": {
+    "cid": "QmXjSSnMUnc7EjQBYtTHhvLU3kXJTbUyhVhJRSTRehh186",
+    "digest": "efa407fd09202dffd15799a8518e73d3",
+    "sector_size": 1024
+  },
+  "v20-stacked-proof-of-replication-e71093863cadc71de61f38311ee45816633973bbf34849316b147f8d2e66f199.vk": {
+    "cid": "QmYHW3zhQouDP4okFbXSsRMcZ8bokKGvzxqbv7ZrunPMiG",
+    "digest": "b2f09a0ccb62da28c890d5b881c8dcd2",
+    "sector_size": 1024
+  },
+  "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.params": {
+    "cid": "QmUhyfNeLb32LfSkjsUwTFYLXQGMj6JQ8daff4DdVMt79q",
+    "digest": "b53c1916a63839ec345aa2224e9198b7",
+    "sector_size": 1073741824
+  },
+  "v20-stacked-proof-of-replication-e99a585174b6a45b254ba4780d72c89ad808c305c6d11711009ade4f39dba8e9.vk": {
+    "cid": "QmWReGfbuoozNErbskmFvqV4q36BY6F2WWb4cVFc3zoYkA",
+    "digest": "20d58a3fae7343481f8298a2dd493dd7",
+    "sector_size": 1073741824
+  },
+  "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.params": {
+    "cid": "QmSAHu14Pe8iav6BYCt9XkpHJ73XM7tcpY4d9JK9BST9HU",
+    "digest": "7698426202c7e07b26ef056d31485b3a",
+    "sector_size": 16777216
+  },
+  "v20-stacked-proof-of-replication-f571ee2386f4c65a68e802747f2d78691006fc81a67971c4d9641403fffece16.vk": {
+    "cid": "QmaKtFLShnhMGVn7P9UsHjkgqtqRFSwCStqqykBN7u8dax",
+    "digest": "834408e5c3fce6ec5d1bf64e64cee94e",
+    "sector_size": 16777216
+  }
+}
+

build/version.go

@@ -5,7 +5,7 @@ import "fmt"
 var CurrentCommit string
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "0.1.6"
+const BuildVersion = "0.2.5"
 
 var UserVersion = BuildVersion + CurrentCommit
 

chain/actors/actor_init.go

@@ -15,7 +15,7 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-hamt-ipld"
 	cbor "github.com/ipfs/go-ipld-cbor"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	mh "github.com/multiformats/go-multihash"
 )
 
@@ -161,7 +161,7 @@ func (ia InitActor) Exec(act *types.Actor, vmctx types.VMContext, p *ExecParams)
 
 func IsBuiltinActor(code cid.Cid) bool {
 	switch code {
-	case StorageMarketCodeCid, StoragePowerCodeCid, StorageMinerCodeCid, AccountCodeCid, InitCodeCid, MultisigCodeCid, PaymentChannelCodeCid:
+	case StorageMarketCodeCid, StoragePowerCodeCid, StorageMinerCodeCid, StorageMiner2CodeCid, AccountCodeCid, InitCodeCid, MultisigCodeCid, PaymentChannelCodeCid:
 		return true
 	default:
 		return false

chain/actors/actor_miner.go

@@ -19,13 +19,9 @@ import (
 	cbor "github.com/ipfs/go-ipld-cbor"
 	"github.com/libp2p/go-libp2p-core/peer"
 	cbg "github.com/whyrusleeping/cbor-gen"
-	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
 )
 
-const MaxSectors = 1 << 48
-const RLEMax = 100e3
-
 type StorageMinerActor struct{}
 
 type StorageMinerActorState struct {
@@ -141,10 +137,7 @@ func (sma StorageMinerActor) Exports() []interface{} {
 	return []interface{}{
 		1: sma.StorageMinerConstructor,
 		2: sma.PreCommitSector,
-		3: withUpdates(
-			update{start: 0, method: sma.ProveCommitSectorV0},
-			update{start: build.ForkNoPowerEPSUpdates, method: sma.ProveCommitSectorV1},
-		),
+		3: sma.ProveCommitSector,
 		4: sma.SubmitFallbackPoSt,
 		//5: sma.SlashStorageFault,
 		//6: sma.GetCurrentProvingSet,
@@ -159,10 +152,7 @@ func (sma StorageMinerActor) Exports() []interface{} {
 		//15: sma.ChangeWorker,
 		16: sma.IsSlashed,
 		17: sma.CheckMiner,
-		18: withUpdates(
-			update{start: 0, method: sma.DeclareFaultsV0},
-			update{start: build.ForkNoPowerEPSUpdates, method: sma.DeclareFaultsV1},
-		),
+		18: sma.DeclareFaults,
 		19: sma.SlashConsensusFault,
 		20: sma.SubmitElectionPoSt,
 	}
@@ -298,110 +288,7 @@ type SectorProveCommitInfo struct {
 	DealIDs  []uint64
 }
 
-func (sma StorageMinerActor) ProveCommitSectorV0(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) {
-	ctx := vmctx.Context()
-	oldstate, self, err := loadState(vmctx)
-	if err != nil {
-		return nil, err
-	}
-
-	mi, err := loadMinerInfo(vmctx, self)
-	if err != nil {
-		return nil, err
-	}
-
-	us, ok := self.PreCommittedSectors[uintToStringKey(params.SectorID)]
-	if !ok {
-		return nil, aerrors.New(1, "no pre-commitment found for sector")
-	}
-
-	if us.ReceivedEpoch+build.InteractivePoRepDelay >= vmctx.BlockHeight() {
-		return nil, aerrors.New(2, "too early for proof submission")
-	}
-
-	delete(self.PreCommittedSectors, uintToStringKey(params.SectorID))
-
-	// TODO: ensure normalization to ID address
-	maddr := vmctx.Message().To
-
-	ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback)
-	if err != nil {
-		return nil, aerrors.Wrap(err, "failed to get ticket randomness")
-	}
-
-	seed, err := vmctx.GetRandomness(us.ReceivedEpoch + build.InteractivePoRepDelay)
-	if err != nil {
-		return nil, aerrors.Wrap(err, "failed to get randomness for prove sector commitment")
-	}
-
-	enc, err := SerializeParams(&ComputeDataCommitmentParams{
-		DealIDs:    params.DealIDs,
-		SectorSize: mi.SectorSize,
-	})
-	if err != nil {
-		return nil, aerrors.Wrap(err, "failed to serialize ComputeDataCommitmentParams")
-	}
-
-	commD, err := vmctx.Send(StorageMarketAddress, SMAMethods.ComputeDataCommitment, types.NewInt(0), enc)
-	if err != nil {
-		return nil, aerrors.Wrapf(err, "failed to compute data commitment (sector %d, deals: %v)", params.SectorID, params.DealIDs)
-	}
-
-	if ok, err := vmctx.Sys().ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil {
-		return nil, err
-	} else if !ok {
-		return nil, aerrors.Newf(2, "porep proof was invalid (t:%x; s:%x(%d); p:%s)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, truncateHexPrint(params.Proof))
-	}
-
-	// Note: There must exist a unique index in the miner's sector set for each
-	// sector ID. The `faults`, `recovered`, and `done` parameters of the
-	// SubmitPoSt method express indices into this sector set.
-	nssroot, err := AddToSectorSet(ctx, types.WrapStorage(vmctx.Storage()), self.Sectors, params.SectorID, us.Info.CommR, commD)
-	if err != nil {
-		return nil, err
-	}
-	self.Sectors = nssroot
-
-	// if miner is not mining, start their proving period now
-	// Note: As written here, every miners first PoSt will only be over one sector.
-	// We could set up a 'grace period' for starting mining that would allow miners
-	// to submit several sectors for their first proving period. Alternatively, we
-	// could simply make the 'PreCommitSector' call take multiple sectors at a time.
-	//
-	// Note: Proving period is a function of sector size; small sectors take less
-	// time to prove than large sectors do. Sector size is selected when pledging.
-	pss, lerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
-	if lerr != nil {
-		return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
-	}
-
-	if pss.Count == 0 {
-		self.ProvingSet = self.Sectors
-		// TODO: probably want to wait until the miner is above a certain
-		//  threshold before starting this
-		self.ElectionPeriodStart = vmctx.BlockHeight()
-	}
-
-	nstate, err := vmctx.Storage().Put(self)
-	if err != nil {
-		return nil, err
-	}
-	if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
-		return nil, err
-	}
-
-	activateParams, err := SerializeParams(&ActivateStorageDealsParams{
-		Deals: params.DealIDs,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	_, err = vmctx.Send(StorageMarketAddress, SMAMethods.ActivateStorageDeals, types.NewInt(0), activateParams)
-	return nil, aerrors.Wrapf(err, "calling ActivateStorageDeals failed")
-}
-
-func (sma StorageMinerActor) ProveCommitSectorV1(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) {
+func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) {
 	ctx := vmctx.Context()
 	oldstate, self, err := loadState(vmctx)
 	if err != nil {
@@ -431,6 +318,14 @@ func (sma StorageMinerActor) ProveCommitSectorV1(act *types.Actor, vmctx types.V
 	// TODO: ensure normalization to ID address
 	maddr := vmctx.Message().To
 
+	if vmctx.BlockHeight()-us.Info.SealEpoch > build.MaxSealLookback {
+		return nil, aerrors.Newf(5, "source randomness for sector SealEpoch too far in past (epoch %d)", us.Info.SealEpoch)
+	}
+
+	if vmctx.BlockHeight()-us.ReceivedEpoch > build.MaxSealLookback {
+		return nil, aerrors.Newf(6, "source randomness for sector ReceivedEpoch too far in past (epoch %d)", us.ReceivedEpoch)
+	}
+
 	ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback)
 	if err != nil {
 		return nil, aerrors.Wrap(err, "failed to get ticket randomness")
@@ -577,25 +472,21 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
 		return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
 	}
 
-	{
-		c, nerr := self.FaultSet.Count()
-		if nerr != nil {
-			return nil, aerrors.Absorb(nerr, 6, "invalid bitfield")
-		}
-
-		if c > RLEMax {
-			return nil, aerrors.Newf(7, "too many items in bitfield: %d", c)
-		}
+	ss, lerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
+	if lerr != nil {
+		return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
 	}
 
-	faults, nerr := self.FaultSet.AllMap()
+	faults, nerr := self.FaultSet.AllMap(2 * ss.Count)
 	if nerr != nil {
 		return nil, aerrors.Absorb(err, 5, "RLE+ invalid")
 	}
 
+	activeFaults := uint64(0)
 	var sectorInfos []ffi.PublicSectorInfo
 	if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error {
 		if faults[id] {
+			activeFaults++
 			return nil
 		}
 
@@ -632,8 +523,8 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
 		})
 	}
 
-	if ok, lerr := sectorbuilder.VerifyFallbackPost(vmctx.Context(), mi.SectorSize,
-		sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, 0); !ok || lerr != nil { // TODO: FORK - set faults to len(faults)
+	if ok, lerr := vmctx.Sys().VerifyFallbackPost(vmctx.Context(), mi.SectorSize,
+		sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, activeFaults); !ok || lerr != nil {
 		if lerr != nil {
 			// TODO: study PoST errors
 			return nil, aerrors.Absorb(lerr, 4, "PoST error")
@@ -644,7 +535,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
 	}
 
 	// Post submission is successful!
-	if err := onSuccessfulPoSt(self, vmctx); err != nil {
+	if err := onSuccessfulPoSt(self, vmctx, activeFaults); err != nil {
 		return nil, err
 	}
 
@@ -678,7 +569,7 @@ func SectorIsUnique(ctx context.Context, s types.Storage, sroot cid.Cid, sid uin
 }
 
 func AddToSectorSet(ctx context.Context, blks amt.Blocks, ss cid.Cid, sectorID uint64, commR, commD []byte) (cid.Cid, ActorError) {
-	if sectorID >= MaxSectors {
+	if sectorID >= build.MinerMaxSectors {
 		return cid.Undef, aerrors.Newf(25, "sector ID out of range: %d", sectorID)
 	}
 	ssr, err := amt.LoadAMT(blks, ss)
@@ -701,7 +592,7 @@ func AddToSectorSet(ctx context.Context, blks amt.Blocks, ss cid.Cid, sectorID u
 }
 
 func GetFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, sectorID uint64) (bool, []byte, []byte, ActorError) {
-	if sectorID >= MaxSectors {
+	if sectorID >= build.MinerMaxSectors {
 		return false, nil, nil, aerrors.Newf(25, "sector ID out of range: %d", sectorID)
 	}
 
@@ -747,17 +638,6 @@ func RemoveFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, ids [
 	return ncid, nil
 }
 
-func ValidatePoRep(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, ActorError) {
-	_, span := trace.StartSpan(ctx, "ValidatePoRep")
-	defer span.End()
-	ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof)
-	if err != nil {
-		return false, aerrors.Absorb(err, 25, "verify seal failed")
-	}
-
-	return ok, nil
-}
-
 func CollateralForPower(power types.BigInt) types.BigInt {
 	return types.BigMul(power, types.NewInt(10))
 	/* TODO: this
@@ -935,33 +815,7 @@ type DeclareFaultsParams struct {
 	Faults types.BitField
 }
 
-func (sma StorageMinerActor) DeclareFaultsV0(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) {
-	oldstate, self, aerr := loadState(vmctx)
-	if aerr != nil {
-		return nil, aerr
-	}
-
-	nfaults, err := types.MergeBitFields(params.Faults, self.FaultSet)
-	if err != nil {
-		return nil, aerrors.Absorb(err, 1, "failed to merge bitfields")
-	}
-
-	self.FaultSet = nfaults
-
-	self.LastFaultSubmission = vmctx.BlockHeight()
-
-	nstate, aerr := vmctx.Storage().Put(self)
-	if err != nil { // TODO: FORK: should be aerr
-		return nil, aerr
-	}
-	if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
-		return nil, err
-	}
-
-	return nil, nil
-}
-
-func (sma StorageMinerActor) DeclareFaultsV1(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) {
+func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) {
 	oldstate, self, aerr := loadState(vmctx)
 	if aerr != nil {
 		return nil, aerr
@@ -981,6 +835,20 @@ func (sma StorageMinerActor) DeclareFaultsV1(act *types.Actor, vmctx types.VMCon
 		return nil, aerrors.Absorb(err, 1, "failed to merge bitfields")
 	}
 
+	ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
+	if nerr != nil {
+		return nil, aerrors.HandleExternalError(nerr, "failed to load sector set")
+	}
+
+	cf, nerr := nfaults.Count()
+	if nerr != nil {
+		return nil, aerrors.Absorb(nerr, 2, "could not decode RLE+")
+	}
+
+	if cf > 2*ss.Count {
+		return nil, aerrors.Newf(3, "too many declared faults: %d > %d", cf, 2*ss.Count)
+	}
+
 	self.FaultSet = nfaults
 
 	self.LastFaultSubmission = vmctx.BlockHeight()
@@ -1056,7 +924,41 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM
 		return nil, aerrors.New(1, "slashed miners can't perform election PoSt")
 	}
 
-	if err := onSuccessfulPoSt(self, vmctx); err != nil {
+	pss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
+	if nerr != nil {
+		return nil, aerrors.HandleExternalError(nerr, "failed to load proving set")
+	}
+
+	ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
+	if nerr != nil {
+		return nil, aerrors.HandleExternalError(nerr, "failed to load proving set")
+	}
+
+	faults, nerr := self.FaultSet.AllMap(2 * ss.Count)
+	if nerr != nil {
+		return nil, aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)")
+	}
+
+	activeFaults := uint64(0)
+	for f := range faults {
+		if f > amt.MaxIndex {
+			continue
+		}
+
+		var comms [][]byte
+		err := pss.Get(f, &comms)
+		if err != nil {
+			var notfound *amt.ErrNotFound
+			if !xerrors.As(err, &notfound) {
+				return nil, aerrors.HandleExternalError(err, "failed to find sector in sector set")
+			}
+			continue
+		}
+
+		activeFaults++
+	}
+
+	if err := onSuccessfulPoSt(self, vmctx, activeFaults); err != nil { // TODO
 		return nil, err
 	}
 
@@ -1071,82 +973,7 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM
 	return nil, nil
 }
 
-func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError {
-	if vmctx.BlockHeight() >= build.ForkNoPowerEPSUpdates {
-		return onSuccessfulPoStV1(self, vmctx)
-	}
-
-	return onSuccessfulPoStV0(self, vmctx)
-}
-
-func onSuccessfulPoStV0(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError {
-	var mi MinerInfo
-	if err := vmctx.Storage().Get(self.Info, &mi); err != nil {
-		return err
-	}
-
-	pss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
-	if nerr != nil {
-		return aerrors.HandleExternalError(nerr, "failed to load proving set")
-	}
-
-	faults, nerr := self.FaultSet.All()
-	if nerr != nil {
-		return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)")
-	}
-
-	self.FaultSet = types.NewBitField()
-
-	oldPower := self.Power
-	newPower := types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), types.NewInt(mi.SectorSize))
-
-	// If below the minimum size requirement, miners have zero power
-	if newPower.LessThan(types.NewInt(build.MinimumMinerPower)) {
-		newPower = types.NewInt(0)
-	}
-
-	self.Power = newPower
-
-	delta := types.BigSub(self.Power, oldPower)
-	if self.SlashedAt != 0 {
-		self.SlashedAt = 0
-		delta = self.Power
-	}
-
-	prevSlashingDeadline := self.ElectionPeriodStart + build.SlashablePowerDelay
-	if !self.Active && newPower.GreaterThan(types.NewInt(0)) {
-		self.Active = true
-		prevSlashingDeadline = 0
-	}
-
-	if !(oldPower.IsZero() && newPower.IsZero()) {
-		enc, err := SerializeParams(&UpdateStorageParams{
-			Delta:                 delta,
-			NextSlashDeadline:     vmctx.BlockHeight() + build.SlashablePowerDelay,
-			PreviousSlashDeadline: prevSlashingDeadline,
-		})
-		if err != nil {
-			return err
-		}
-
-		_, err = vmctx.Send(StoragePowerAddress, SPAMethods.UpdateStorage, types.NewInt(0), enc)
-		if err != nil {
-			return aerrors.Wrap(err, "updating storage failed")
-		}
-	}
-
-	ncid, err := RemoveFromSectorSet(vmctx.Context(), vmctx.Storage(), self.Sectors, faults)
-	if err != nil {
-		return err
-	}
-
-	self.Sectors = ncid
-	self.ProvingSet = ncid
-	self.ElectionPeriodStart = vmctx.BlockHeight()
-	return nil
-}
-
-func onSuccessfulPoStV1(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError {
+func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext, activeFaults uint64) aerrors.ActorError {
 	// TODO: some sector upkeep stuff that is very haphazard and unclear in the spec
 
 	var mi MinerInfo
@@ -1159,17 +986,12 @@ func onSuccessfulPoStV1(self *StorageMinerActorState, vmctx types.VMContext) aer
 		return aerrors.HandleExternalError(nerr, "failed to load proving set")
 	}
 
-	{
-		c, nerr := self.FaultSet.Count()
-		if nerr != nil {
-			return aerrors.Absorb(nerr, 2, "invalid bitfield")
-		}
-		if c > RLEMax {
-			return aerrors.Newf(3, "too many items in bitfield: %d", c)
-		}
+	ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
+	if nerr != nil {
+		return aerrors.HandleExternalError(nerr, "failed to load sector set")
 	}
 
-	faults, nerr := self.FaultSet.All()
+	faults, nerr := self.FaultSet.All(2 * ss.Count)
 	if nerr != nil {
 		return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)")
 	}
@@ -1177,7 +999,7 @@ func onSuccessfulPoStV1(self *StorageMinerActorState, vmctx types.VMContext) aer
 	self.FaultSet = types.NewBitField()
 
 	oldPower := self.Power
-	newPower := types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), types.NewInt(mi.SectorSize))
+	newPower := types.BigMul(types.NewInt(pss.Count-activeFaults), types.NewInt(mi.SectorSize))
 
 	// If below the minimum size requirement, miners have zero power
 	if newPower.LessThan(types.NewInt(build.MinimumMinerPower)) {

chain/actors/actor_miner2.go

@@ -0,0 +1,865 @@
+package actors
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	ffi "github.com/filecoin-project/filecoin-ffi"
+	amt2 "github.com/filecoin-project/go-amt-ipld/v2"
+
+	"github.com/filecoin-project/go-sectorbuilder"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors/aerrors"
+	"github.com/filecoin-project/lotus/chain/types"
+
+	"github.com/ipfs/go-cid"
+	cbor "github.com/ipfs/go-ipld-cbor"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"golang.org/x/xerrors"
+)
+
+type StorageMinerActor2 struct{}
+
+func (sma StorageMinerActor2) Exports() []interface{} {
+	return []interface{}{
+		1: sma.StorageMinerConstructor,
+		2: sma.PreCommitSector,
+		3: sma.ProveCommitSector,
+		4: sma.SubmitFallbackPoSt,
+		//5: sma.SlashStorageFault,
+		//6: sma.GetCurrentProvingSet,
+		//7:  sma.ArbitrateDeal,
+		//8:  sma.DePledge,
+		9:  sma.GetOwner,
+		10: sma.GetWorkerAddr,
+		11: sma.GetPower, // TODO: Remove
+		12: sma.GetPeerID,
+		13: sma.GetSectorSize,
+		14: sma.UpdatePeerID,
+		//15: sma.ChangeWorker,
+		16: sma.IsSlashed,
+		17: sma.CheckMiner,
+		18: sma.DeclareFaults,
+		19: sma.SlashConsensusFault,
+		20: sma.SubmitElectionPoSt,
+	}
+}
+
+func (sma StorageMinerActor2) StorageMinerConstructor(act *types.Actor, vmctx types.VMContext, params *StorageMinerConstructorParams) ([]byte, ActorError) {
+	minerInfo := &MinerInfo{
+		Owner:      params.Owner,
+		Worker:     params.Worker,
+		PeerID:     params.PeerID,
+		SectorSize: params.SectorSize,
+	}
+
+	minfocid, err := vmctx.Storage().Put(minerInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	var self StorageMinerActorState
+	sectors := amt2.NewAMT(types.WrapStorage(vmctx.Storage()))
+	scid, serr := sectors.Flush()
+	if serr != nil {
+		return nil, aerrors.HandleExternalError(serr, "initializing AMT")
+	}
+
+	self.Sectors = scid
+	self.ProvingSet = scid
+	self.Info = minfocid
+
+	storage := vmctx.Storage()
+	c, err := storage.Put(&self)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := storage.Commit(EmptyCBOR, c); err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
+func (sma StorageMinerActor2) PreCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorPreCommitInfo) ([]byte, ActorError) {
+
+	ctx := vmctx.Context()
+	oldstate, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+
+	if params.SealEpoch >= vmctx.BlockHeight()+build.SealRandomnessLookback {
+		return nil, aerrors.Newf(1, "sector commitment must be based off past randomness (%d >= %d)", params.SealEpoch, vmctx.BlockHeight()+build.SealRandomnessLookback)
+	}
+
+	if vmctx.BlockHeight()-params.SealEpoch+build.SealRandomnessLookback > build.SealRandomnessLookbackLimit {
+		return nil, aerrors.Newf(2, "sector commitment must be recent enough (was %d)", vmctx.BlockHeight()-params.SealEpoch+build.SealRandomnessLookback)
+	}
+
+	mi, err := loadMinerInfo(vmctx, self)
+	if err != nil {
+		return nil, err
+	}
+
+	if vmctx.Message().From != mi.Worker {
+		return nil, aerrors.New(1, "not authorized to precommit sector for miner")
+	}
+
+	// make sure the miner isnt trying to submit a pre-existing sector
+	unique, err := SectorIsUnique(ctx, vmctx.Storage(), self.Sectors, params.SectorNumber)
+	if err != nil {
+		return nil, err
+	}
+	if !unique {
+		return nil, aerrors.New(3, "sector already committed!")
+	}
+
+	// Power of the miner after adding this sector
+	futurePower := types.BigAdd(self.Power, types.NewInt(mi.SectorSize))
+	collateralRequired := CollateralForPower(futurePower)
+
+	// TODO: grab from market?
+	if act.Balance.LessThan(collateralRequired) {
+		return nil, aerrors.New(4, "not enough collateral")
+	}
+
+	self.PreCommittedSectors[uintToStringKey(params.SectorNumber)] = &PreCommittedSector{
+		Info:          *params,
+		ReceivedEpoch: vmctx.BlockHeight(),
+	}
+
+	if len(self.PreCommittedSectors) > 4096 {
+		return nil, aerrors.New(5, "too many precommitted sectors")
+	}
+
+	nstate, err := vmctx.Storage().Put(self)
+	if err != nil {
+		return nil, err
+	}
+	if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
+func (sma StorageMinerActor2) ProveCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) {
+	ctx := vmctx.Context()
+	oldstate, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+
+	mi, err := loadMinerInfo(vmctx, self)
+	if err != nil {
+		return nil, err
+	}
+
+	if vmctx.Message().From != mi.Worker {
+		return nil, aerrors.New(1, "not authorized to submit sector proof for miner")
+	}
+
+	us, ok := self.PreCommittedSectors[uintToStringKey(params.SectorID)]
+	if !ok {
+		return nil, aerrors.New(1, "no pre-commitment found for sector")
+	}
+
+	if us.ReceivedEpoch+build.InteractivePoRepDelay >= vmctx.BlockHeight() {
+		return nil, aerrors.New(2, "too early for proof submission")
+	}
+
+	delete(self.PreCommittedSectors, uintToStringKey(params.SectorID))
+
+	// TODO: ensure normalization to ID address
+	maddr := vmctx.Message().To
+
+	if vmctx.BlockHeight()-us.Info.SealEpoch > build.MaxSealLookback {
+		return nil, aerrors.Newf(5, "source randomness for sector SealEpoch too far in past (epoch %d)", us.Info.SealEpoch)
+	}
+
+	if vmctx.BlockHeight()-us.ReceivedEpoch > build.MaxSealLookback {
+		return nil, aerrors.Newf(6, "source randomness for sector ReceivedEpoch too far in past (epoch %d)", us.ReceivedEpoch)
+	}
+
+	ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback)
+	if err != nil {
+		return nil, aerrors.Wrap(err, "failed to get ticket randomness")
+	}
+
+	seed, err := vmctx.GetRandomness(us.ReceivedEpoch + build.InteractivePoRepDelay)
+	if err != nil {
+		return nil, aerrors.Wrap(err, "failed to get randomness for prove sector commitment")
+	}
+
+	enc, err := SerializeParams(&ComputeDataCommitmentParams{
+		DealIDs:    params.DealIDs,
+		SectorSize: mi.SectorSize,
+	})
+	if err != nil {
+		return nil, aerrors.Wrap(err, "failed to serialize ComputeDataCommitmentParams")
+	}
+
+	commD, err := vmctx.Send(StorageMarketAddress, SMAMethods.ComputeDataCommitment, types.NewInt(0), enc)
+	if err != nil {
+		return nil, aerrors.Wrapf(err, "failed to compute data commitment (sector %d, deals: %v)", params.SectorID, params.DealIDs)
+	}
+
+	if ok, err := vmctx.Sys().ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil {
+		return nil, err
+	} else if !ok {
+		return nil, aerrors.Newf(2, "porep proof was invalid (t:%x; s:%x(%d); p:%s)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, truncateHexPrint(params.Proof))
+	}
+
+	// Note: There must exist a unique index in the miner's sector set for each
+	// sector ID. The `faults`, `recovered`, and `done` parameters of the
+	// SubmitPoSt method express indices into this sector set.
+	nssroot, err := AddToSectorSet2(ctx, types.WrapStorage(vmctx.Storage()), self.Sectors, params.SectorID, us.Info.CommR, commD)
+	if err != nil {
+		return nil, err
+	}
+	self.Sectors = nssroot
+
+	// if miner is not mining, start their proving period now
+	// Note: As written here, every miners first PoSt will only be over one sector.
+	// We could set up a 'grace period' for starting mining that would allow miners
+	// to submit several sectors for their first proving period. Alternatively, we
+	// could simply make the 'PreCommitSector' call take multiple sectors at a time.
+	//
+	// Note: Proving period is a function of sector size; small sectors take less
+	// time to prove than large sectors do. Sector size is selected when pledging.
+	pss, lerr := amt2.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
+	if lerr != nil {
+		return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
+	}
+
+	if pss.Count == 0 && !self.Active {
+		self.ProvingSet = self.Sectors
+		// TODO: probably want to wait until the miner is above a certain
+		//  threshold before starting this
+		self.ElectionPeriodStart = vmctx.BlockHeight()
+	}
+
+	nstate, err := vmctx.Storage().Put(self)
+	if err != nil {
+		return nil, err
+	}
+	if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
+		return nil, err
+	}
+
+	activateParams, err := SerializeParams(&ActivateStorageDealsParams{
+		Deals: params.DealIDs,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = vmctx.Send(StorageMarketAddress, SMAMethods.ActivateStorageDeals, types.NewInt(0), activateParams)
+	return nil, aerrors.Wrapf(err, "calling ActivateStorageDeals failed")
+}
+
+func (sma StorageMinerActor2) SubmitFallbackPoSt(act *types.Actor, vmctx types.VMContext, params *SubmitFallbackPoStParams) ([]byte, ActorError) {
+	oldstate, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+
+	mi, err := loadMinerInfo(vmctx, self)
+	if err != nil {
+		return nil, err
+	}
+
+	if vmctx.Message().From != mi.Worker {
+		return nil, aerrors.New(1, "not authorized to submit post for miner")
+	}
+
+	/*
+		// TODO: handle fees
+		msgVal := vmctx.Message().Value
+		if msgVal.LessThan(feesRequired) {
+			return nil, aerrors.New(2, "not enough funds to pay post submission fees")
+		}
+
+		if msgVal.GreaterThan(feesRequired) {
+			_, err := vmctx.Send(vmctx.Message().From, 0,
+				types.BigSub(msgVal, feesRequired), nil)
+			if err != nil {
+				return nil, aerrors.Wrap(err, "could not refund excess fees")
+			}
+		}
+	*/
+
+	var seed [sectorbuilder.CommLen]byte
+	{
+		randHeight := self.ElectionPeriodStart + build.FallbackPoStDelay
+		if vmctx.BlockHeight() <= randHeight {
+			// TODO: spec, retcode
+			return nil, aerrors.Newf(1, "submit fallback PoSt called too early (%d < %d)", vmctx.BlockHeight(), randHeight)
+		}
+
+		rand, err := vmctx.GetRandomness(randHeight)
+
+		if err != nil {
+			return nil, aerrors.Wrap(err, "could not get randomness for PoST")
+		}
+		if len(rand) < len(seed) {
+			return nil, aerrors.Escalate(fmt.Errorf("randomness too small (%d < %d)",
+				len(rand), len(seed)), "improper randomness")
+		}
+		copy(seed[:], rand)
+	}
+
+	pss, lerr := amt2.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
+	if lerr != nil {
+		return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
+	}
+
+	ss, lerr := amt2.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
+	if lerr != nil {
+		return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
+	}
+
+	faults, nerr := self.FaultSet.AllMap(2 * ss.Count)
+	if nerr != nil {
+		return nil, aerrors.Absorb(err, 5, "RLE+ invalid")
+	}
+
+	activeFaults := uint64(0)
+	var sectorInfos []ffi.PublicSectorInfo
+	if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error {
+		if faults[id] {
+			activeFaults++
+			return nil
+		}
+
+		var comms [][]byte
+		if err := cbor.DecodeInto(v.Raw, &comms); err != nil {
+			return xerrors.New("could not decode comms")
+		}
+		si := ffi.PublicSectorInfo{
+			SectorID: id,
+		}
+		commR := comms[0]
+		if len(commR) != len(si.CommR) {
+			return xerrors.Errorf("commR length is wrong: %d", len(commR))
+		}
+		copy(si.CommR[:], commR)
+
+		sectorInfos = append(sectorInfos, si)
+
+		return nil
+	}); err != nil {
+		return nil, aerrors.Absorb(err, 3, "could not decode sectorset")
+	}
+
+	proverID := vmctx.Message().To // TODO: normalize to ID address
+
+	var candidates []sectorbuilder.EPostCandidate
+	for _, t := range params.Candidates {
+		var partial [32]byte
+		copy(partial[:], t.Partial)
+		candidates = append(candidates, sectorbuilder.EPostCandidate{
+			PartialTicket:        partial,
+			SectorID:             t.SectorID,
+			SectorChallengeIndex: t.ChallengeIndex,
+		})
+	}
+
+	if ok, lerr := vmctx.Sys().VerifyFallbackPost(vmctx.Context(), mi.SectorSize,
+		sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, activeFaults); !ok || lerr != nil {
+		if lerr != nil {
+			// TODO: study PoST errors
+			return nil, aerrors.Absorb(lerr, 4, "PoST error")
+		}
+		if !ok {
+			return nil, aerrors.New(4, "PoST invalid")
+		}
+	}
+
+	// Post submission is successful!
+	if err := onSuccessfulPoSt2(self, vmctx, activeFaults); err != nil {
+		return nil, err
+	}
+
+	c, err := vmctx.Storage().Put(self)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := vmctx.Storage().Commit(oldstate, c); err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
+func (sma StorageMinerActor2) GetPower(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) {
+	_, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+	return self.Power.Bytes(), nil
+}
+
+func SectorIsUnique2(ctx context.Context, s types.Storage, sroot cid.Cid, sid uint64) (bool, ActorError) {
+	found, _, _, err := GetFromSectorSet2(ctx, s, sroot, sid)
+	if err != nil {
+		return false, err
+	}
+
+	return !found, nil
+}
+
+func AddToSectorSet2(ctx context.Context, blks amt2.Blocks, ss cid.Cid, sectorID uint64, commR, commD []byte) (cid.Cid, ActorError) {
+	if sectorID >= build.MinerMaxSectors {
+		return cid.Undef, aerrors.Newf(25, "sector ID out of range: %d", sectorID)
+	}
+	ssr, err := amt2.LoadAMT(blks, ss)
+	if err != nil {
+		return cid.Undef, aerrors.HandleExternalError(err, "could not load sector set node")
+	}
+
+	// TODO: Spec says to use SealCommitment, and construct commD from deals each time,
+	//  but that would make SubmitPoSt way, way more expensive
+	if err := ssr.Set(sectorID, [][]byte{commR, commD}); err != nil {
+		return cid.Undef, aerrors.HandleExternalError(err, "failed to set commitment in sector set")
+	}
+
+	ncid, err := ssr.Flush()
+	if err != nil {
+		return cid.Undef, aerrors.HandleExternalError(err, "failed to flush sector set")
+	}
+
+	return ncid, nil
+}
+
+func GetFromSectorSet2(ctx context.Context, s types.Storage, ss cid.Cid, sectorID uint64) (bool, []byte, []byte, ActorError) {
+	if sectorID >= build.MinerMaxSectors {
+		return false, nil, nil, aerrors.Newf(25, "sector ID out of range: %d", sectorID)
+	}
+
+	ssr, err := amt2.LoadAMT(types.WrapStorage(s), ss)
+	if err != nil {
+		return false, nil, nil, aerrors.HandleExternalError(err, "could not load sector set node")
+	}
+
+	var comms [][]byte
+	err = ssr.Get(sectorID, &comms)
+	if err != nil {
+		if _, ok := err.(*amt2.ErrNotFound); ok {
+			return false, nil, nil, nil
+		}
+		return false, nil, nil, aerrors.HandleExternalError(err, "failed to find sector in sector set")
+	}
+
+	if len(comms) != 2 {
+		return false, nil, nil, aerrors.Newf(20, "sector set entry should only have 2 elements")
+	}
+
+	return true, comms[0], comms[1], nil
+}
+
+func RemoveFromSectorSet2(ctx context.Context, s types.Storage, ss cid.Cid, ids []uint64) (cid.Cid, aerrors.ActorError) {
+
+	ssr, err := amt2.LoadAMT(types.WrapStorage(s), ss)
+	if err != nil {
+		return cid.Undef, aerrors.HandleExternalError(err, "could not load sector set node")
+	}
+
+	for _, id := range ids {
+		if err := ssr.Delete(id); err != nil {
+			log.Warnf("failed to delete sector %d from set: %s", id, err)
+		}
+	}
+
+	ncid, err := ssr.Flush()
+	if err != nil {
+		return cid.Undef, aerrors.HandleExternalError(err, "failed to flush sector set")
+	}
+
+	return ncid, nil
+}
+
+func (sma StorageMinerActor2) GetWorkerAddr(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) {
+	_, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+
+	mi, err := loadMinerInfo(vmctx, self)
+	if err != nil {
+		return nil, err
+	}
+
+	return mi.Worker.Bytes(), nil
+}
+
+func (sma StorageMinerActor2) GetOwner(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) {
+	_, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+
+	mi, err := loadMinerInfo(vmctx, self)
+	if err != nil {
+		return nil, err
+	}
+
+	return mi.Owner.Bytes(), nil
+}
+
+func (sma StorageMinerActor2) GetPeerID(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) {
+	_, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+
+	mi, err := loadMinerInfo(vmctx, self)
+	if err != nil {
+		return nil, err
+	}
+
+	return []byte(mi.PeerID), nil
+}
+
+func (sma StorageMinerActor2) UpdatePeerID(act *types.Actor, vmctx types.VMContext, params *UpdatePeerIDParams) ([]byte, ActorError) {
+	oldstate, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+
+	mi, err := loadMinerInfo(vmctx, self)
+	if err != nil {
+		return nil, err
+	}
+
+	if vmctx.Message().From != mi.Worker {
+		return nil, aerrors.New(2, "only the mine worker may update the peer ID")
+	}
+
+	mi.PeerID = params.PeerID
+
+	mic, err := vmctx.Storage().Put(mi)
+	if err != nil {
+		return nil, err
+	}
+
+	self.Info = mic
+
+	c, err := vmctx.Storage().Put(self)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := vmctx.Storage().Commit(oldstate, c); err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
+func (sma StorageMinerActor2) GetSectorSize(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) {
+	_, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+
+	mi, err := loadMinerInfo(vmctx, self)
+	if err != nil {
+		return nil, err
+	}
+
+	return types.NewInt(mi.SectorSize).Bytes(), nil
+}
+
+func (sma StorageMinerActor2) IsSlashed(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, ActorError) {
+	_, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return cbg.EncodeBool(self.SlashedAt != 0), nil
+}
+
+// TODO: better name
+func (sma StorageMinerActor2) CheckMiner(act *types.Actor, vmctx types.VMContext, params *CheckMinerParams) ([]byte, ActorError) {
+	if vmctx.Message().From != StoragePowerAddress {
+		return nil, aerrors.New(2, "only the storage power actor can check miner")
+	}
+
+	oldstate, self, err := loadState(vmctx)
+	if err != nil {
+		return nil, err
+	}
+
+	if !isLate(vmctx.BlockHeight(), self) {
+		// Everything's fine
+		return nil, nil
+	}
+
+	if self.SlashedAt != 0 {
+		// Don't slash more than necessary
+		return nil, nil
+	}
+
+	if params.NetworkPower.Equals(self.Power) {
+		// Don't break the network when there's only one miner left
+
+		log.Warnf("can't slash miner %s for missed PoSt, no power would be left in the network", vmctx.Message().To)
+		return nil, nil
+	}
+
+	// Slash for being late
+
+	self.SlashedAt = vmctx.BlockHeight()
+
+	nstate, err := vmctx.Storage().Put(self)
+	if err != nil {
+		return nil, err
+	}
+	if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
+		return nil, err
+	}
+
+	var out bytes.Buffer
+	if err := self.Power.MarshalCBOR(&out); err != nil {
+		return nil, aerrors.HandleExternalError(err, "marshaling return value")
+	}
+	return out.Bytes(), nil
+}
+
+func (sma StorageMinerActor2) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) {
+	oldstate, self, aerr := loadState(vmctx)
+	if aerr != nil {
+		return nil, aerr
+	}
+
+	mi, aerr := loadMinerInfo(vmctx, self)
+	if aerr != nil {
+		return nil, aerr
+	}
+
+	if vmctx.Message().From != mi.Worker {
+		return nil, aerrors.New(1, "not authorized to declare faults for miner")
+	}
+
+	nfaults, err := types.MergeBitFields(params.Faults, self.FaultSet)
+	if err != nil {
+		return nil, aerrors.Absorb(err, 1, "failed to merge bitfields")
+	}
+
+	ss, nerr := amt2.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
+	if nerr != nil {
+		return nil, aerrors.HandleExternalError(nerr, "failed to load sector set")
+	}
+
+	cf, nerr := nfaults.Count()
+	if nerr != nil {
+		return nil, aerrors.Absorb(nerr, 2, "could not decode RLE+")
+	}
+
+	if cf > 2*ss.Count {
+		return nil, aerrors.Newf(3, "too many declared faults: %d > %d", cf, 2*ss.Count)
+	}
+
+	self.FaultSet = nfaults
+
+	self.LastFaultSubmission = vmctx.BlockHeight()
+
+	nstate, aerr := vmctx.Storage().Put(self)
+	if aerr != nil {
+		return nil, aerr
+	}
+	if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
+func (sma StorageMinerActor2) SlashConsensusFault(act *types.Actor, vmctx types.VMContext, params *MinerSlashConsensusFault) ([]byte, ActorError) {
+	if vmctx.Message().From != StoragePowerAddress {
+		return nil, aerrors.New(1, "SlashConsensusFault may only be called by the storage market actor")
+	}
+
+	slashedCollateral := params.SlashedCollateral
+	if slashedCollateral.LessThan(act.Balance) {
+		slashedCollateral = act.Balance
+	}
+
+	// Some of the slashed collateral should be paid to the slasher
+	// GROWTH_RATE determines how fast the slasher share of slashed collateral will increase as block elapses
+	// current GROWTH_RATE results in SLASHER_SHARE reaches 1 after 30 blocks
+	// TODO: define arithmetic precision and rounding for this operation
+	blockElapsed := vmctx.BlockHeight() - params.AtHeight
+
+	slasherShare := slasherShare(params.SlashedCollateral, blockElapsed)
+
+	burnPortion := types.BigSub(slashedCollateral, slasherShare)
+
+	_, err := vmctx.Send(vmctx.Message().From, 0, slasherShare, nil)
+	if err != nil {
+		return nil, aerrors.Wrap(err, "failed to pay slasher")
+	}
+
+	_, err = vmctx.Send(BurntFundsAddress, 0, burnPortion, nil)
+	if err != nil {
+		return nil, aerrors.Wrap(err, "failed to burn funds")
+	}
+
+	// TODO: this still allows the miner to commit sectors and submit posts,
+	// their users could potentially be unaffected, but the miner will never be
+	// able to mine a block again
+	// One potential issue: the miner will have to pay back the slashed
+	// collateral to continue submitting PoSts, which includes pledge
+	// collateral that they no longer really 'need'
+
+	return nil, nil
+}
+
+func (sma StorageMinerActor2) SubmitElectionPoSt(act *types.Actor, vmctx types.VMContext, params *struct{}) ([]byte, aerrors.ActorError) {
+	if vmctx.Message().From != NetworkAddress {
+		return nil, aerrors.Newf(1, "submit election post can only be called by the storage power actor")
+	}
+
+	oldstate, self, aerr := loadState(vmctx)
+	if aerr != nil {
+		return nil, aerr
+	}
+
+	if self.SlashedAt != 0 {
+		return nil, aerrors.New(1, "slashed miners can't perform election PoSt")
+	}
+
+	pss, nerr := amt2.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
+	if nerr != nil {
+		return nil, aerrors.HandleExternalError(nerr, "failed to load proving set")
+	}
+
+	ss, nerr := amt2.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
+	if nerr != nil {
+		return nil, aerrors.HandleExternalError(nerr, "failed to load proving set")
+	}
+
+	faults, nerr := self.FaultSet.AllMap(2 * ss.Count)
+	if nerr != nil {
+		return nil, aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)")
+	}
+
+	activeFaults := uint64(0)
+	for f := range faults {
+		if f > amt2.MaxIndex {
+			continue
+		}
+
+		var comms [][]byte
+		err := pss.Get(f, &comms)
+		if err != nil {
+			var notfound *amt2.ErrNotFound
+			if !xerrors.As(err, &notfound) {
+				return nil, aerrors.HandleExternalError(err, "failed to find sector in sector set")
+			}
+			continue
+		}
+
+		activeFaults++
+	}
+
+	if err := onSuccessfulPoSt2(self, vmctx, activeFaults); err != nil { // TODO
+		return nil, err
+	}
+
+	ncid, err := vmctx.Storage().Put(self)
+	if err != nil {
+		return nil, err
+	}
+	if err := vmctx.Storage().Commit(oldstate, ncid); err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
+func onSuccessfulPoSt2(self *StorageMinerActorState, vmctx types.VMContext, activeFaults uint64) aerrors.ActorError {
+	// FORK
+	if vmctx.BlockHeight() < build.ForkBootyBayHeight {
+		return onSuccessfulPoSt(self, vmctx, activeFaults)
+	}
+
+	var mi MinerInfo
+	if err := vmctx.Storage().Get(self.Info, &mi); err != nil {
+		return err
+	}
+
+	pss, nerr := amt2.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
+	if nerr != nil {
+		return aerrors.HandleExternalError(nerr, "failed to load proving set")
+	}
+
+	ss, nerr := amt2.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
+	if nerr != nil {
+		return aerrors.HandleExternalError(nerr, "failed to load sector set")
+	}
+
+	faults, nerr := self.FaultSet.All(2 * ss.Count)
+	if nerr != nil {
+		return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)")
+	}
+
+	self.FaultSet = types.NewBitField()
+
+	oldPower := self.Power
+	newPower := types.BigMul(types.NewInt(pss.Count-activeFaults), types.NewInt(mi.SectorSize))
+
+	// If below the minimum size requirement, miners have zero power
+	if newPower.LessThan(types.NewInt(build.MinimumMinerPower)) {
+		newPower = types.NewInt(0)
+	}
+
+	self.Power = newPower
+
+	delta := types.BigSub(self.Power, oldPower)
+	if self.SlashedAt != 0 {
+		self.SlashedAt = 0
+		delta = self.Power
+	}
+
+	prevSlashingDeadline := self.ElectionPeriodStart + build.SlashablePowerDelay
+	if !self.Active && newPower.GreaterThan(types.NewInt(0)) {
+		self.Active = true
+		prevSlashingDeadline = 0
+	}
+
+	if !(oldPower.IsZero() && newPower.IsZero()) {
+		enc, err := SerializeParams(&UpdateStorageParams{
+			Delta:                 delta,
+			NextSlashDeadline:     vmctx.BlockHeight() + build.SlashablePowerDelay,
+			PreviousSlashDeadline: prevSlashingDeadline,
+		})
+		if err != nil {
+			return err
+		}
+
+		_, err = vmctx.Send(StoragePowerAddress, SPAMethods.UpdateStorage, types.NewInt(0), enc)
+		if err != nil {
+			return aerrors.Wrap(err, "updating storage failed")
+		}
+
+		self.ElectionPeriodStart = vmctx.BlockHeight()
+	}
+
+	var ncid cid.Cid
+	var err aerrors.ActorError
+
+	ncid, err = RemoveFromSectorSet2(vmctx.Context(), vmctx.Storage(), self.Sectors, faults)
+	if err != nil {
+		return err
+	}
+
+	self.Sectors = ncid
+	self.ProvingSet = ncid
+	return nil
+}

chain/actors/actor_miner_test.go

@@ -3,18 +3,22 @@ package actors_test
 import (
 	"bytes"
 	"context"
+	"math"
 	"math/rand"
 	"testing"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-sectorbuilder"
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/aerrors"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/lib/rlepluslazy"
 	hamt "github.com/ipfs/go-hamt-ipld"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	"github.com/stretchr/testify/assert"
 	cbg "github.com/whyrusleeping/cbor-gen"
 )
 
@@ -50,23 +54,37 @@ func TestMinerCommitSectors(t *testing.T) {
 	addSectorToMiner(h, t, minerAddr, worker, client, 1)
 
 	assertSectorIDs(h, t, minerAddr, []uint64{1})
+
 }
 
+type badRuns struct {
+	done bool
+}
+
+func (br *badRuns) HasNext() bool {
+	return !br.done
+}
+
+func (br *badRuns) NextRun() (rlepluslazy.Run, error) {
+	br.done = true
+	return rlepluslazy.Run{true, math.MaxInt64}, nil
+}
+
+var _ rlepluslazy.RunIterator = (*badRuns)(nil)
+
 func TestMinerSubmitBadFault(t *testing.T) {
+	oldSS, oldMin := build.SectorSizes, build.MinimumMinerPower
+	build.SectorSizes, build.MinimumMinerPower = []uint64{1024}, 1024
+	defer func() {
+		build.SectorSizes, build.MinimumMinerPower = oldSS, oldMin
+	}()
+
 	var worker, client address.Address
 	var minerAddr address.Address
 	opts := []HarnessOpt{
 		HarnessAddr(&worker, 1000000),
 		HarnessAddr(&client, 1000000),
-		HarnessActor(&minerAddr, &worker, actors.StorageMinerCodeCid,
-			func() cbg.CBORMarshaler {
-				return &actors.StorageMinerConstructorParams{
-					Owner:      worker,
-					Worker:     worker,
-					SectorSize: 1024,
-					PeerID:     "fakepeerid",
-				}
-			}),
+		HarnessAddMiner(&minerAddr, &worker),
 	}
 
 	h := NewHarness(t, opts...)
@@ -92,18 +110,52 @@ func TestMinerSubmitBadFault(t *testing.T) {
 
 	ret, _ = h.Invoke(t, actors.NetworkAddress, minerAddr, actors.MAMethods.SubmitElectionPoSt, nil)
 	ApplyOK(t, ret)
-
 	assertSectorIDs(h, t, minerAddr, []uint64{1})
 
+	st, err := getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr)
+	assert.NoError(t, err)
+	expectedPower := st.Power
+	if types.BigCmp(expectedPower, types.NewInt(1024)) != 0 {
+		t.Errorf("Expected power of 1024, got %s", expectedPower)
+	}
+
 	badnum := uint64(0)
 	badnum--
 	bf = types.NewBitField()
 	bf.Set(badnum)
+	bf.Set(badnum - 1)
 	ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf})
 	ApplyOK(t, ret)
 
 	ret, _ = h.Invoke(t, actors.NetworkAddress, minerAddr, actors.MAMethods.SubmitElectionPoSt, nil)
+
 	ApplyOK(t, ret)
+	assertSectorIDs(h, t, minerAddr, []uint64{1})
+
+	st, err = getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr)
+	assert.NoError(t, err)
+	currentPower := st.Power
+	if types.BigCmp(expectedPower, currentPower) != 0 {
+		t.Errorf("power changed and shouldn't have: %s != %s", expectedPower, currentPower)
+	}
+
+	bf.Set(badnum - 2)
+	ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf})
+	if ret.ExitCode != 3 {
+		t.Errorf("expected exit code 3, got %d: %+v", ret.ExitCode, ret.ActorErr)
+	}
+	assertSectorIDs(h, t, minerAddr, []uint64{1})
+
+	rle, err := rlepluslazy.EncodeRuns(&badRuns{}, []byte{})
+	assert.NoError(t, err)
+
+	bf, err = types.NewBitFieldFromBytes(rle)
+	assert.NoError(t, err)
+	ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf})
+	if ret.ExitCode != 3 {
+		t.Errorf("expected exit code 3, got %d: %+v", ret.ExitCode, ret.ActorErr)
+	}
+	assertSectorIDs(h, t, minerAddr, []uint64{1})
 
 	bf = types.NewBitField()
 	bf.Set(1)
@@ -175,7 +227,7 @@ func assertSectorIDs(h *Harness, t *testing.T, maddr address.Address, ids []uint
 	}
 }
 
-func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) {
+func getMinerState(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) (*actors.StorageMinerActorState, error) {
 	mact, err := st.GetActor(maddr)
 	if err != nil {
 		return nil, err
@@ -187,6 +239,14 @@ func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Bl
 	if err := cst.Get(ctx, mact.Head, &mstate); err != nil {
 		return nil, err
 	}
+	return &mstate, nil
+}
+
+func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) {
+	mstate, err := getMinerState(ctx, st, bs, maddr)
+	if err != nil {
+		return nil, err
+	}
 
 	return stmgr.LoadSectorsFromSet(ctx, bs, mstate.Sectors)
 }
@@ -202,7 +262,6 @@ func (h *Harness) makeFakeDeal(t *testing.T, miner, worker, client address.Addre
 	prop := actors.StorageDealProposal{
 		PieceRef:  commP[:],
 		PieceSize: size,
-		//PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid
 
 		Client:   client,
 		Provider: miner,

chain/actors/actor_paych.go

@@ -116,7 +116,7 @@ func (pca PaymentChannelActor) UpdateChannelState(act *types.Actor, vmctx types.
 
 	vb, nerr := sv.SigningBytes()
 	if nerr != nil {
-		return nil, aerrors.Escalate(nerr, "failed to serialize signedvoucher")
+		return nil, aerrors.Absorb(nerr, 1, "failed to serialize signedvoucher")
 	}
 
 	if err := vmctx.VerifySignature(sv.Signature, self.From, vb); err != nil {

chain/actors/actor_storagemarket.go

@@ -122,15 +122,21 @@ func (sdp *StorageDealProposal) Cid() (cid.Cid, error) {
 	return nd.Cid(), nil
 }
 
-func (sdp *StorageDealProposal) Verify() error {
-	unsigned := *sdp
-	unsigned.ProposerSignature = nil
-	var buf bytes.Buffer
-	if err := unsigned.MarshalCBOR(&buf); err != nil {
-		return err
+func (sdp *StorageDealProposal) Verify(worker address.Address) error {
+	if sdp.Client != worker || worker == address.Undef {
+		unsigned := *sdp
+		unsigned.ProposerSignature = nil
+		var buf bytes.Buffer
+		if err := unsigned.MarshalCBOR(&buf); err != nil {
+			return err
+		}
+
+		if err := sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()); err != nil {
+			return err
+		}
 	}
 
-	return sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes())
+	return nil
 }
 
 type OnChainDeal struct {
@@ -396,7 +402,7 @@ func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDe
 		return aerrors.New(2, "Deals must be submitted by the miner worker")
 	}
 
-	if err := deal.Verify(); err != nil {
+	if err := deal.Verify(providerWorker); err != nil {
 		return aerrors.Absorb(err, 3, "verifying proposer signature")
 	}
 

chain/actors/actor_storagepower.go

@@ -67,7 +67,7 @@ type CreateStorageMinerParams struct {
 
 func (spa StoragePowerActor) CreateStorageMiner(act *types.Actor, vmctx types.VMContext, params *CreateStorageMinerParams) ([]byte, ActorError) {
 	if !build.SupportedSectorSize(params.SectorSize) {
-		return nil, aerrors.New(1, "Unsupported sector size")
+		return nil, aerrors.Newf(1, "Unsupported sector size: %d", params.SectorSize)
 	}
 
 	var self StoragePowerState
@@ -85,7 +85,13 @@ func (spa StoragePowerActor) CreateStorageMiner(act *types.Actor, vmctx types.VM
 		return nil, aerrors.Newf(1, "not enough funds passed to cover required miner collateral (needed %s, got %s)", reqColl, vmctx.Message().Value)
 	}
 
-	encoded, err := CreateExecParams(StorageMinerCodeCid, &StorageMinerConstructorParams{
+	// FORK
+	minerCid := StorageMinerCodeCid
+	if vmctx.BlockHeight() > build.ForkFrigidHeight {
+		minerCid = StorageMiner2CodeCid
+	}
+
+	encoded, err := CreateExecParams(minerCid, &StorageMinerConstructorParams{
 		Owner:      params.Owner,
 		Worker:     params.Worker,
 		SectorSize: params.SectorSize,
@@ -130,10 +136,29 @@ type ArbitrateConsensusFaultParams struct {
 }
 
 func (spa StoragePowerActor) ArbitrateConsensusFault(act *types.Actor, vmctx types.VMContext, params *ArbitrateConsensusFaultParams) ([]byte, ActorError) {
+	if params == nil || params.Block1 == nil || params.Block2 == nil {
+		return nil, aerrors.New(1, "failed to parse params")
+	}
+
 	if params.Block1.Miner != params.Block2.Miner {
 		return nil, aerrors.New(2, "blocks must be from the same miner")
 	}
 
+	// FORK
+	if vmctx.BlockHeight() > build.ForkBlizzardHeight {
+		if params.Block1.Height <= build.ForkBlizzardHeight {
+			return nil, aerrors.New(10, "cannot slash miners with blocks from before blizzard")
+		}
+
+		if params.Block2.Height <= build.ForkBlizzardHeight {
+			return nil, aerrors.New(11, "cannot slash miners with blocks from before blizzard")
+		}
+
+		if params.Block1.Cid() == params.Block2.Cid() {
+			return nil, aerrors.New(3, "blocks must be different")
+		}
+	}
+
 	rval, err := vmctx.Send(params.Block1.Miner, MAMethods.GetWorkerAddr, types.NewInt(0), nil)
 	if err != nil {
 		return nil, aerrors.Wrap(err, "failed to get miner worker")

chain/actors/actor_storagepower_test.go

@@ -93,7 +93,15 @@ func TestStorageMarketCreateAndSlashMiner(t *testing.T) {
 		signBlock(t, h.w, workerAddr, b1)
 		signBlock(t, h.w, workerAddr, b2)
 
+		h.BlockHeight = build.ForkBlizzardHeight + 1
 		ret, _ := h.Invoke(t, ownerAddr, StoragePowerAddress, SPAMethods.ArbitrateConsensusFault,
+			&ArbitrateConsensusFaultParams{
+				Block1: b1,
+				Block2: b1,
+			})
+		assert.Equal(t, uint8(3), ret.ExitCode, "should have failed with exit 3")
+
+		ret, _ = h.Invoke(t, ownerAddr, StoragePowerAddress, SPAMethods.ArbitrateConsensusFault,
 			&ArbitrateConsensusFaultParams{
 				Block1: b1,
 				Block2: b2,
@@ -145,7 +153,7 @@ func cheatStorageMarketTotal(t *testing.T, vm *vm.VM, bs bstore.Blockstore) {
 
 func fakeBlock(t *testing.T, minerAddr address.Address, ts uint64) *types.BlockHeader {
 	c := fakeCid(t, 1)
-	return &types.BlockHeader{Height: 5, Miner: minerAddr, Timestamp: ts, ParentStateRoot: c, Messages: c, ParentMessageReceipts: c, BLSAggregate: types.Signature{Type: types.KTBLS}}
+	return &types.BlockHeader{Height: 8000, Miner: minerAddr, Timestamp: ts, ParentStateRoot: c, Messages: c, ParentMessageReceipts: c, BLSAggregate: types.Signature{Type: types.KTBLS}}
 }
 
 func fakeCid(t *testing.T, s int) cid.Cid {

chain/actors/actors.go

@@ -12,6 +12,7 @@ var CronCodeCid cid.Cid
 var StoragePowerCodeCid cid.Cid
 var StorageMarketCodeCid cid.Cid
 var StorageMinerCodeCid cid.Cid
+var StorageMiner2CodeCid cid.Cid
 var MultisigCodeCid cid.Cid
 var InitCodeCid cid.Cid
 var PaymentChannelCodeCid cid.Cid
@@ -46,6 +47,7 @@ func init() {
 	StoragePowerCodeCid = mustSum("fil/1/power")
 	StorageMarketCodeCid = mustSum("fil/1/market")
 	StorageMinerCodeCid = mustSum("fil/1/miner")
+	StorageMiner2CodeCid = mustSum("fil/1/miner/2")
 	MultisigCodeCid = mustSum("fil/1/multisig")
 	InitCodeCid = mustSum("fil/1/init")
 	PaymentChannelCodeCid = mustSum("fil/1/paych")

chain/actors/actors_test.go

@@ -50,10 +50,10 @@ func setupVMTestEnv(t *testing.T) (*vm.VM, []address.Address, bstore.Blockstore)
 		t.Fatal(err)
 	}
 
-	cs := store.NewChainStore(bs, nil)
+	cs := store.NewChainStore(bs, nil, nil)
 
 	// TODO: should probabaly mock out the randomness bit, nil works for now
-	vm, err := vm.NewVM(stateroot, 1, nil, maddr, cs.Blockstore())
+	vm, err := vm.NewVM(stateroot, 1, nil, maddr, cs.Blockstore(), cs.VMSys())
 	if err != nil {
 		t.Fatal(err)
 	}

chain/actors/aerrors/wrap.go

@@ -171,7 +171,8 @@ func HandleExternalError(err error, msg string) ActorError {
 	}
 
 	return &actorError{
-		fatal: true,
+		fatal:   false,
+		retCode: 219,
 
 		msg:   msg,
 		frame: xerrors.Caller(1),

chain/actors/harness2_test.go

@@ -3,6 +3,7 @@ package actors_test
 import (
 	"bytes"
 	"context"
+	"github.com/filecoin-project/go-sectorbuilder"
 	"math/rand"
 	"testing"
 
@@ -115,6 +116,32 @@ func HarnessActor(actor *address.Address, creator *address.Address, code cid.Cid
 
 }
 
+func HarnessAddMiner(addr *address.Address, creator *address.Address) HarnessOpt {
+	return func(t testing.TB, h *Harness) error {
+		if h.Stage != HarnessPostInit {
+			return nil
+		}
+		if !addr.Empty() {
+			return xerrors.New("actor address should be empty")
+		}
+		ret, _ := h.InvokeWithValue(t, *creator, actors.StoragePowerAddress,
+			actors.SPAMethods.CreateStorageMiner, types.NewInt(3000), &actors.StorageMinerConstructorParams{
+				Owner:      *creator,
+				Worker:     *creator,
+				SectorSize: 1024,
+				PeerID:     "fakepeerid",
+			})
+
+		if ret.ExitCode != 0 {
+			return xerrors.Errorf("creating actor: %w", ret.ActorErr)
+		}
+		var err error
+		*addr, err = address.NewFromBytes(ret.Return)
+		return err
+
+	}
+}
+
 func HarnessCtx(ctx context.Context) HarnessOpt {
 	return func(t testing.TB, h *Harness) error {
 		h.ctx = ctx
@@ -168,8 +195,8 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness {
 		t.Fatal(err)
 	}
 
-	h.cs = store.NewChainStore(h.bs, nil)
-	h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore())
+	h.cs = store.NewChainStore(h.bs, nil, vm.Syscalls(sectorbuilder.ProofVerifier))
+	h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore(), h.cs.VMSys())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -177,7 +204,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness {
 	for _, opt := range options {
 		err := opt(t, h)
 		if err != nil {
-			t.Fatalf("Applying options: %v", err)
+			t.Fatalf("Applying options: %+v", err)
 		}
 	}
 

chain/blocksync/blocksync.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	inet "github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 )

chain/blocksync/blocksync_client.go

@@ -455,7 +455,7 @@ func (bpt *bsPeerTracker) prefSortedPeers() []peer.ID {
 		getPeerInitLat := func(p peer.ID) float64 {
 			var res float64
 			if bpt.pmgr != nil {
-				if lat, ok := bpt.pmgr.GetPeerLatency(out[i]); ok {
+				if lat, ok := bpt.pmgr.GetPeerLatency(p); ok {
 					res = float64(lat)
 				}
 			}
@@ -476,7 +476,7 @@ func (bpt *bsPeerTracker) prefSortedPeers() []peer.ID {
 			failRateJ := float64(pj.failures) / float64(pj.failures+pj.successes)
 			costJ = float64(pj.averageTime) + failRateJ*float64(bpt.avgGlobalTime)
 		} else {
-			costI = getPeerInitLat(out[i])
+			costJ = getPeerInitLat(out[j])
 		}
 
 		return costI < costJ

chain/deals/cbor_gen.go

@@ -1,883 +0,0 @@
-package deals
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/libp2p/go-libp2p-core/peer"
-	cbg "github.com/whyrusleeping/cbor-gen"
-	xerrors "golang.org/x/xerrors"
-)
-
-// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
-
-var _ = xerrors.Errorf
-
-func (t *AskRequest) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{129}); err != nil {
-		return err
-	}
-
-	// t.Miner (address.Address) (struct)
-	if err := t.Miner.MarshalCBOR(w); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *AskRequest) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 1 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Miner (address.Address) (struct)
-
-	{
-
-		if err := t.Miner.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	return nil
-}
-
-func (t *AskResponse) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{129}); err != nil {
-		return err
-	}
-
-	// t.Ask (types.SignedStorageAsk) (struct)
-	if err := t.Ask.MarshalCBOR(w); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *AskResponse) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 1 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Ask (types.SignedStorageAsk) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-			t.Ask = new(types.SignedStorageAsk)
-			if err := t.Ask.UnmarshalCBOR(br); err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
-func (t *Proposal) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{130}); err != nil {
-		return err
-	}
-
-	// t.DealProposal (actors.StorageDealProposal) (struct)
-	if err := t.DealProposal.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.Piece (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.Piece); err != nil {
-		return xerrors.Errorf("failed to write cid field t.Piece: %w", err)
-	}
-
-	return nil
-}
-
-func (t *Proposal) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 2 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.DealProposal (actors.StorageDealProposal) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-			t.DealProposal = new(actors.StorageDealProposal)
-			if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
-				return err
-			}
-		}
-
-	}
-	// t.Piece (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.Piece: %w", err)
-		}
-
-		t.Piece = c
-
-	}
-	return nil
-}
-
-func (t *Response) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{132}); err != nil {
-		return err
-	}
-
-	// t.State (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
-		return err
-	}
-
-	// t.Message (string) (string)
-	if len(t.Message) > cbg.MaxLength {
-		return xerrors.Errorf("Value in field t.Message was too long")
-	}
-
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil {
-		return err
-	}
-	if _, err := w.Write([]byte(t.Message)); err != nil {
-		return err
-	}
-
-	// t.Proposal (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.Proposal); err != nil {
-		return xerrors.Errorf("failed to write cid field t.Proposal: %w", err)
-	}
-
-	// t.StorageDealSubmission (types.SignedMessage) (struct)
-	if err := t.StorageDealSubmission.MarshalCBOR(w); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *Response) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 4 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.State (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.State = uint64(extra)
-	// t.Message (string) (string)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		t.Message = string(sval)
-	}
-	// t.Proposal (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.Proposal: %w", err)
-		}
-
-		t.Proposal = c
-
-	}
-	// t.StorageDealSubmission (types.SignedMessage) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-			t.StorageDealSubmission = new(types.SignedMessage)
-			if err := t.StorageDealSubmission.UnmarshalCBOR(br); err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
-func (t *SignedResponse) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{130}); err != nil {
-		return err
-	}
-
-	// t.Response (deals.Response) (struct)
-	if err := t.Response.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.Signature (types.Signature) (struct)
-	if err := t.Signature.MarshalCBOR(w); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 2 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Response (deals.Response) (struct)
-
-	{
-
-		if err := t.Response.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.Signature (types.Signature) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-			t.Signature = new(types.Signature)
-			if err := t.Signature.UnmarshalCBOR(br); err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
-func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{136}); err != nil {
-		return err
-	}
-
-	// t.Data (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.Data); err != nil {
-		return xerrors.Errorf("failed to write cid field t.Data: %w", err)
-	}
-
-	// t.PricePerEpoch (types.BigInt) (struct)
-	if err := t.PricePerEpoch.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.ProposalExpiration (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil {
-		return err
-	}
-
-	// t.Duration (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil {
-		return err
-	}
-
-	// t.ProviderAddress (address.Address) (struct)
-	if err := t.ProviderAddress.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.Client (address.Address) (struct)
-	if err := t.Client.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.MinerWorker (address.Address) (struct)
-	if err := t.MinerWorker.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.MinerID (peer.ID) (string)
-	if len(t.MinerID) > cbg.MaxLength {
-		return xerrors.Errorf("Value in field t.MinerID was too long")
-	}
-
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil {
-		return err
-	}
-	if _, err := w.Write([]byte(t.MinerID)); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 8 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Data (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.Data: %w", err)
-		}
-
-		t.Data = c
-
-	}
-	// t.PricePerEpoch (types.BigInt) (struct)
-
-	{
-
-		if err := t.PricePerEpoch.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.ProposalExpiration (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.ProposalExpiration = uint64(extra)
-	// t.Duration (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.Duration = uint64(extra)
-	// t.ProviderAddress (address.Address) (struct)
-
-	{
-
-		if err := t.ProviderAddress.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.Client (address.Address) (struct)
-
-	{
-
-		if err := t.Client.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.MinerWorker (address.Address) (struct)
-
-	{
-
-		if err := t.MinerWorker.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.MinerID (peer.ID) (string)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		t.MinerID = peer.ID(sval)
-	}
-	return nil
-}
-
-func (t *ClientDeal) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{135}); err != nil {
-		return err
-	}
-
-	// t.ProposalCid (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.ProposalCid); err != nil {
-		return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err)
-	}
-
-	// t.Proposal (actors.StorageDealProposal) (struct)
-	if err := t.Proposal.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.State (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
-		return err
-	}
-
-	// t.Miner (peer.ID) (string)
-	if len(t.Miner) > cbg.MaxLength {
-		return xerrors.Errorf("Value in field t.Miner was too long")
-	}
-
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil {
-		return err
-	}
-	if _, err := w.Write([]byte(t.Miner)); err != nil {
-		return err
-	}
-
-	// t.MinerWorker (address.Address) (struct)
-	if err := t.MinerWorker.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.DealID (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
-		return err
-	}
-
-	// t.PublishMessage (types.SignedMessage) (struct)
-	if err := t.PublishMessage.MarshalCBOR(w); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 7 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.ProposalCid (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err)
-		}
-
-		t.ProposalCid = c
-
-	}
-	// t.Proposal (actors.StorageDealProposal) (struct)
-
-	{
-
-		if err := t.Proposal.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.State (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.State = uint64(extra)
-	// t.Miner (peer.ID) (string)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		t.Miner = peer.ID(sval)
-	}
-	// t.MinerWorker (address.Address) (struct)
-
-	{
-
-		if err := t.MinerWorker.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.DealID (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.DealID = uint64(extra)
-	// t.PublishMessage (types.SignedMessage) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-			t.PublishMessage = new(types.SignedMessage)
-			if err := t.PublishMessage.UnmarshalCBOR(br); err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
-func (t *MinerDeal) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{135}); err != nil {
-		return err
-	}
-
-	// t.Client (peer.ID) (string)
-	if len(t.Client) > cbg.MaxLength {
-		return xerrors.Errorf("Value in field t.Client was too long")
-	}
-
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil {
-		return err
-	}
-	if _, err := w.Write([]byte(t.Client)); err != nil {
-		return err
-	}
-
-	// t.Proposal (actors.StorageDealProposal) (struct)
-	if err := t.Proposal.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.ProposalCid (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.ProposalCid); err != nil {
-		return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err)
-	}
-
-	// t.State (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
-		return err
-	}
-
-	// t.Ref (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.Ref); err != nil {
-		return xerrors.Errorf("failed to write cid field t.Ref: %w", err)
-	}
-
-	// t.DealID (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
-		return err
-	}
-
-	// t.SectorID (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 7 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Client (peer.ID) (string)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		t.Client = peer.ID(sval)
-	}
-	// t.Proposal (actors.StorageDealProposal) (struct)
-
-	{
-
-		if err := t.Proposal.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.ProposalCid (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err)
-		}
-
-		t.ProposalCid = c
-
-	}
-	// t.State (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.State = uint64(extra)
-	// t.Ref (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.Ref: %w", err)
-		}
-
-		t.Ref = c
-
-	}
-	// t.DealID (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.DealID = uint64(extra)
-	// t.SectorID (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.SectorID = uint64(extra)
-	return nil
-}
-
-func (t *StorageDataTransferVoucher) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{130}); err != nil {
-		return err
-	}
-
-	// t.Proposal (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.Proposal); err != nil {
-		return xerrors.Errorf("failed to write cid field t.Proposal: %w", err)
-	}
-
-	// t.DealID (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *StorageDataTransferVoucher) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 2 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Proposal (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.Proposal: %w", err)
-		}
-
-		t.Proposal = c
-
-	}
-	// t.DealID (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.DealID = uint64(extra)
-	return nil
-}

chain/deals/client.go

@@ -1,311 +0,0 @@
-package deals
-
-import (
-	"context"
-
-	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
-	"github.com/libp2p/go-libp2p-core/host"
-	inet "github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/go-statestore"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/events"
-	"github.com/filecoin-project/lotus/chain/market"
-	"github.com/filecoin-project/lotus/chain/stmgr"
-	"github.com/filecoin-project/lotus/chain/store"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/wallet"
-	"github.com/filecoin-project/lotus/node/impl/full"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/filecoin-project/lotus/retrieval/discovery"
-)
-
-var log = logging.Logger("deals")
-
-type ClientDeal struct {
-	ProposalCid cid.Cid
-	Proposal    actors.StorageDealProposal
-	State       api.DealState
-	Miner       peer.ID
-	MinerWorker address.Address
-	DealID      uint64
-
-	PublishMessage *types.SignedMessage
-
-	s inet.Stream
-}
-
-type Client struct {
-	sm    *stmgr.StateManager
-	chain *store.ChainStore
-	h     host.Host
-	w     *wallet.Wallet
-	// dataTransfer
-	// TODO: once the data transfer module is complete, the
-	// client will listen to events on the data transfer module
-	// Because we are using only a fake DAGService
-	// implementation, there's no validation or events on the client side
-	dataTransfer dtypes.ClientDataTransfer
-	dag          dtypes.ClientDAG
-	discovery    *discovery.Local
-	events       *events.Events
-	fm           *market.FundMgr
-
-	deals *statestore.StateStore
-	conns map[cid.Cid]inet.Stream
-
-	incoming chan *ClientDeal
-	updated  chan clientDealUpdate
-
-	stop    chan struct{}
-	stopped chan struct{}
-}
-
-type clientDealUpdate struct {
-	newState api.DealState
-	id       cid.Cid
-	err      error
-	mut      func(*ClientDeal)
-}
-
-type clientApi struct {
-	full.ChainAPI
-	full.StateAPI
-}
-
-func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, fm *market.FundMgr, deals dtypes.ClientDealStore, chainapi full.ChainAPI, stateapi full.StateAPI) *Client {
-	c := &Client{
-		sm:           sm,
-		chain:        chain,
-		h:            h,
-		w:            w,
-		dataTransfer: dataTransfer,
-		dag:          dag,
-		discovery:    discovery,
-		fm:           fm,
-		events:       events.NewEvents(context.TODO(), &clientApi{chainapi, stateapi}),
-
-		deals: deals,
-		conns: map[cid.Cid]inet.Stream{},
-
-		incoming: make(chan *ClientDeal, 16),
-		updated:  make(chan clientDealUpdate, 16),
-
-		stop:    make(chan struct{}),
-		stopped: make(chan struct{}),
-	}
-
-	return c
-}
-
-func (c *Client) Run(ctx context.Context) {
-	go func() {
-		defer close(c.stopped)
-
-		for {
-			select {
-			case deal := <-c.incoming:
-				c.onIncoming(deal)
-			case update := <-c.updated:
-				c.onUpdated(ctx, update)
-			case <-c.stop:
-				return
-			}
-		}
-	}()
-}
-
-func (c *Client) onIncoming(deal *ClientDeal) {
-	log.Info("incoming deal")
-
-	if _, ok := c.conns[deal.ProposalCid]; ok {
-		log.Errorf("tracking deal connection: already tracking connection for deal %s", deal.ProposalCid)
-		return
-	}
-	c.conns[deal.ProposalCid] = deal.s
-
-	if err := c.deals.Begin(deal.ProposalCid, deal); err != nil {
-		// We may have re-sent the proposal
-		log.Errorf("deal tracking failed: %s", err)
-		c.failDeal(deal.ProposalCid, err)
-		return
-	}
-
-	go func() {
-		c.updated <- clientDealUpdate{
-			newState: api.DealUnknown,
-			id:       deal.ProposalCid,
-			err:      nil,
-		}
-	}()
-}
-
-func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) {
-	log.Infof("Client deal %s updated state to %s", update.id, api.DealStates[update.newState])
-	var deal ClientDeal
-	err := c.deals.Mutate(update.id, func(d *ClientDeal) error {
-		d.State = update.newState
-		if update.mut != nil {
-			update.mut(d)
-		}
-		deal = *d
-		return nil
-	})
-	if update.err != nil {
-		log.Errorf("deal %s failed: %s", update.id, update.err)
-		c.failDeal(update.id, update.err)
-		return
-	}
-	if err != nil {
-		c.failDeal(update.id, err)
-		return
-	}
-
-	switch update.newState {
-	case api.DealUnknown: // new
-		c.handle(ctx, deal, c.new, api.DealAccepted)
-	case api.DealAccepted:
-		c.handle(ctx, deal, c.accepted, api.DealStaged)
-	case api.DealStaged:
-		c.handle(ctx, deal, c.staged, api.DealSealing)
-	case api.DealSealing:
-		c.handle(ctx, deal, c.sealing, api.DealNoUpdate)
-		// TODO: DealComplete -> watch for faults, expiration, etc.
-	}
-}
-
-type ClientDealProposal struct {
-	Data cid.Cid
-
-	PricePerEpoch      types.BigInt
-	ProposalExpiration uint64
-	Duration           uint64
-
-	ProviderAddress address.Address
-	Client          address.Address
-	MinerWorker     address.Address
-	MinerID         peer.ID
-}
-
-func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) {
-	if err := c.fm.EnsureAvailable(ctx, p.Client, types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))); err != nil {
-		return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err)
-	}
-
-	commP, pieceSize, err := c.commP(ctx, p.Data)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("computing commP failed: %w", err)
-	}
-
-	dealProposal := &actors.StorageDealProposal{
-		PieceRef:             commP,
-		PieceSize:            uint64(pieceSize),
-		Client:               p.Client,
-		Provider:             p.ProviderAddress,
-		ProposalExpiration:   p.ProposalExpiration,
-		Duration:             p.Duration,
-		StoragePricePerEpoch: p.PricePerEpoch,
-		StorageCollateral:    types.NewInt(uint64(pieceSize)), // TODO: real calc
-	}
-
-	if err := api.SignWith(ctx, c.w.Sign, p.Client, dealProposal); err != nil {
-		return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err)
-	}
-
-	proposalNd, err := cborutil.AsIpld(dealProposal)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err)
-	}
-
-	s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err)
-	}
-
-	proposal := &Proposal{
-		DealProposal: dealProposal,
-		Piece:        p.Data,
-	}
-
-	if err := cborutil.WriteCborRPC(s, proposal); err != nil {
-		s.Reset()
-		return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err)
-	}
-
-	deal := &ClientDeal{
-		ProposalCid: proposalNd.Cid(),
-		Proposal:    *dealProposal,
-		State:       api.DealUnknown,
-		Miner:       p.MinerID,
-		MinerWorker: p.MinerWorker,
-
-		s: s,
-	}
-
-	c.incoming <- deal
-
-	return deal.ProposalCid, c.discovery.AddPeer(p.Data, discovery.RetrievalPeer{
-		Address: dealProposal.Provider,
-		ID:      deal.Miner,
-	})
-}
-
-func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*types.SignedStorageAsk, error) {
-	s, err := c.h.NewStream(ctx, p, AskProtocolID)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to open stream to miner: %w", err)
-	}
-
-	req := &AskRequest{
-		Miner: a,
-	}
-	if err := cborutil.WriteCborRPC(s, req); err != nil {
-		return nil, xerrors.Errorf("failed to send ask request: %w", err)
-	}
-
-	var out AskResponse
-	if err := cborutil.ReadCborRPC(s, &out); err != nil {
-		return nil, xerrors.Errorf("failed to read ask response: %w", err)
-	}
-
-	if out.Ask == nil {
-		return nil, xerrors.Errorf("got no ask back")
-	}
-
-	if out.Ask.Ask.Miner != a {
-		return nil, xerrors.Errorf("got back ask for wrong miner")
-	}
-
-	if err := c.checkAskSignature(out.Ask); err != nil {
-		return nil, xerrors.Errorf("ask was not properly signed")
-	}
-
-	return out.Ask, nil
-}
-
-func (c *Client) List() ([]ClientDeal, error) {
-	var out []ClientDeal
-	if err := c.deals.List(&out); err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) {
-	var out ClientDeal
-	if err := c.deals.Get(d, &out); err != nil {
-		return nil, err
-	}
-	return &out, nil
-}
-
-func (c *Client) Stop() {
-	close(c.stop)
-	<-c.stopped
-}

chain/deals/client_states.go

@@ -1,233 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"context"
-
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/stmgr"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-type clientHandlerFunc func(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error)
-
-func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFunc, next api.DealState) {
-	go func() {
-		mut, err := cb(ctx, deal)
-		if err != nil {
-			next = api.DealError
-		}
-
-		if err == nil && next == api.DealNoUpdate {
-			return
-		}
-
-		select {
-		case c.updated <- clientDealUpdate{
-			newState: next,
-			id:       deal.ProposalCid,
-			err:      err,
-			mut:      mut,
-		}:
-		case <-c.stop:
-		}
-	}()
-}
-
-func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
-	resp, err := c.readStorageDealResp(deal)
-	if err != nil {
-		return nil, err
-	}
-
-	// TODO: verify StorageDealSubmission
-
-	if err := c.disconnect(deal); err != nil {
-		return nil, err
-	}
-
-	/* data transfer happens */
-	if resp.State != api.DealAccepted {
-		return nil, xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State)
-	}
-
-	return func(info *ClientDeal) {
-		info.PublishMessage = resp.StorageDealSubmission
-	}, nil
-}
-
-func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
-	log.Infow("DEAL ACCEPTED!")
-
-	pubmsg := deal.PublishMessage.Message
-	pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider)
-	if err != nil {
-		return nil, xerrors.Errorf("getting miner worker failed: %w", err)
-	}
-
-	if pubmsg.From != pw {
-		return nil, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider)
-	}
-
-	if pubmsg.To != actors.StorageMarketAddress {
-		return nil, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To)
-	}
-
-	if pubmsg.Method != actors.SMAMethods.PublishStorageDeals {
-		return nil, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method)
-	}
-
-	var params actors.PublishStorageDealsParams
-	if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil {
-		return nil, err
-	}
-
-	dealIdx := -1
-	for i, storageDeal := range params.Deals {
-		// TODO: make it less hacky
-		sd := storageDeal
-		eq, err := cborutil.Equals(&deal.Proposal, &sd)
-		if err != nil {
-			return nil, err
-		}
-		if eq {
-			dealIdx = i
-			break
-		}
-	}
-
-	if dealIdx == -1 {
-		return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage.Cid())
-	}
-
-	// TODO: timeout
-	_, ret, err := c.sm.WaitForMessage(ctx, deal.PublishMessage.Cid())
-	if err != nil {
-		return nil, xerrors.Errorf("waiting for deal publish message: %w", err)
-	}
-	if ret.ExitCode != 0 {
-		return nil, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode)
-	}
-
-	var res actors.PublishStorageDealResponse
-	if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil {
-		return nil, err
-	}
-
-	return func(info *ClientDeal) {
-		info.DealID = res.DealIDs[dealIdx]
-	}, nil
-}
-
-func (c *Client) staged(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
-	// TODO: Maybe wait for pre-commit
-
-	return nil, nil
-}
-
-func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
-	checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) {
-		sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts)
-		if err != nil {
-			// TODO: This may be fine for some errors
-			return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err)
-		}
-
-		if sd.ActivationEpoch > 0 {
-			select {
-			case c.updated <- clientDealUpdate{
-				newState: api.DealComplete,
-				id:       deal.ProposalCid,
-			}:
-			case <-c.stop:
-			}
-
-			return true, false, nil
-		}
-
-		return false, true, nil
-	}
-
-	called := func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) {
-		defer func() {
-			if err != nil {
-				select {
-				case c.updated <- clientDealUpdate{
-					newState: api.DealComplete,
-					id:       deal.ProposalCid,
-					err:      xerrors.Errorf("handling applied event: %w", err),
-				}:
-				case <-c.stop:
-				}
-			}
-		}()
-
-		if msg == nil {
-			log.Error("timed out waiting for deal activation... what now?")
-			return false, nil
-		}
-
-		sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts)
-		if err != nil {
-			return false, xerrors.Errorf("failed to look up deal on chain: %w", err)
-		}
-
-		if sd.ActivationEpoch == 0 {
-			return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", deal.DealID, ts.ParentState(), ts.Height())
-		}
-
-		log.Infof("Storage deal %d activated at epoch %d", deal.DealID, sd.ActivationEpoch)
-
-		select {
-		case c.updated <- clientDealUpdate{
-			newState: api.DealComplete,
-			id:       deal.ProposalCid,
-		}:
-		case <-c.stop:
-		}
-
-		return false, nil
-	}
-
-	revert := func(ctx context.Context, ts *types.TipSet) error {
-		log.Warn("deal activation reverted; TODO: actually handle this!")
-		// TODO: Just go back to DealSealing?
-		return nil
-	}
-
-	matchEvent := func(msg *types.Message) (bool, error) {
-		if msg.To != deal.Proposal.Provider {
-			return false, nil
-		}
-
-		if msg.Method != actors.MAMethods.ProveCommitSector {
-			return false, nil
-		}
-
-		var params actors.SectorProveCommitInfo
-		if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil {
-			return false, err
-		}
-
-		var found bool
-		for _, dealID := range params.DealIDs {
-			if dealID == deal.DealID {
-				found = true
-				break
-			}
-		}
-
-		return found, nil
-	}
-
-	if err := c.events.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, matchEvent); err != nil {
-		return nil, xerrors.Errorf("failed to set up called handler")
-	}
-
-	return nil, nil
-}

chain/deals/client_utils.go

@@ -1,171 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"context"
-	"runtime"
-
-	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
-	"github.com/ipfs/go-cid"
-	files "github.com/ipfs/go-ipfs-files"
-	unixfile "github.com/ipfs/go-unixfs/file"
-	"github.com/ipld/go-ipld-prime"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"golang.org/x/xerrors"
-
-	cborutil "github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/go-statestore"
-
-	"github.com/filecoin-project/lotus/datatransfer"
-	"github.com/filecoin-project/lotus/lib/padreader"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-)
-
-func (c *Client) failDeal(id cid.Cid, cerr error) {
-	if cerr == nil {
-		_, f, l, _ := runtime.Caller(1)
-		cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l)
-	}
-
-	s, ok := c.conns[id]
-	if ok {
-		_ = s.Reset()
-		delete(c.conns, id)
-	}
-
-	// TODO: store in some sort of audit log
-	log.Errorf("deal %s failed: %+v", id, cerr)
-}
-
-func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, uint64, error) {
-	root, err := c.dag.Get(ctx, data)
-	if err != nil {
-		log.Errorf("failed to get file root for deal: %s", err)
-		return nil, 0, err
-	}
-
-	n, err := unixfile.NewUnixfsFile(ctx, c.dag, root)
-	if err != nil {
-		log.Errorf("cannot open unixfs file: %s", err)
-		return nil, 0, err
-	}
-
-	uf, ok := n.(files.File)
-	if !ok {
-		// TODO: we probably got directory, how should we handle this in unixfs mode?
-		return nil, 0, xerrors.New("unsupported unixfs type")
-	}
-
-	s, err := uf.Size()
-	if err != nil {
-		return nil, 0, err
-	}
-
-	pr, psize := padreader.New(uf, uint64(s))
-
-	commp, err := sectorbuilder.GeneratePieceCommitment(pr, psize)
-	if err != nil {
-		return nil, 0, xerrors.Errorf("generating CommP: %w", err)
-	}
-
-	return commp[:], psize, nil
-}
-
-func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) {
-	s, ok := c.conns[deal.ProposalCid]
-	if !ok {
-		// TODO: Try to re-establish the connection using query protocol
-		return nil, xerrors.Errorf("no connection to miner")
-	}
-
-	var resp SignedResponse
-	if err := cborutil.ReadCborRPC(s, &resp); err != nil {
-		log.Errorw("failed to read Response message", "error", err)
-		return nil, err
-	}
-
-	if err := resp.Verify(deal.MinerWorker); err != nil {
-		return nil, xerrors.Errorf("verifying response signature failed", err)
-	}
-
-	if resp.Response.Proposal != deal.ProposalCid {
-		return nil, xerrors.Errorf("miner responded to a wrong proposal: %s != %s", resp.Response.Proposal, deal.ProposalCid)
-	}
-
-	return &resp.Response, nil
-}
-
-func (c *Client) disconnect(deal ClientDeal) error {
-	s, ok := c.conns[deal.ProposalCid]
-	if !ok {
-		return nil
-	}
-
-	err := s.Close()
-	delete(c.conns, deal.ProposalCid)
-	return err
-}
-
-var _ datatransfer.RequestValidator = &ClientRequestValidator{}
-
-// ClientRequestValidator validates data transfer requests for the client
-// in a storage market
-type ClientRequestValidator struct {
-	deals *statestore.StateStore
-}
-
-// NewClientRequestValidator returns a new client request validator for the
-// given datastore
-func NewClientRequestValidator(deals dtypes.ClientDealStore) *ClientRequestValidator {
-	crv := &ClientRequestValidator{
-		deals: deals,
-	}
-	return crv
-}
-
-// ValidatePush validates a push request received from the peer that will send data
-// Will always error because clients should not accept push requests from a provider
-// in a storage deal (i.e. send data to client).
-func (c *ClientRequestValidator) ValidatePush(
-	sender peer.ID,
-	voucher datatransfer.Voucher,
-	baseCid cid.Cid,
-	Selector ipld.Node) error {
-	return ErrNoPushAccepted
-}
-
-// ValidatePull validates a pull request received from the peer that will receive data
-// Will succeed only if:
-// - voucher has correct type
-// - voucher references an active deal
-// - referenced deal matches the receiver (miner)
-// - referenced deal matches the given base CID
-// - referenced deal is in an acceptable state
-func (c *ClientRequestValidator) ValidatePull(
-	receiver peer.ID,
-	voucher datatransfer.Voucher,
-	baseCid cid.Cid,
-	Selector ipld.Node) error {
-	dealVoucher, ok := voucher.(*StorageDataTransferVoucher)
-	if !ok {
-		return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType)
-	}
-
-	var deal ClientDeal
-	err := c.deals.Get(dealVoucher.Proposal, &deal)
-	if err != nil {
-		return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal)
-	}
-	if deal.Miner != receiver {
-		return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer)
-	}
-	if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) {
-		return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece)
-	}
-	for _, state := range DataTransferStates {
-		if deal.State == state {
-			return nil
-		}
-	}
-	return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState)
-}

chain/deals/provider.go

@@ -1,294 +0,0 @@
-package deals
-
-import (
-	"context"
-	"errors"
-	"sync"
-
-	cid "github.com/ipfs/go-cid"
-	datastore "github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
-	inet "github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/go-statestore"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/datatransfer"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/filecoin-project/lotus/storage"
-	"github.com/filecoin-project/lotus/storage/sectorblocks"
-)
-
-var ProviderDsPrefix = "/deals/provider"
-
-type MinerDeal struct {
-	Client      peer.ID
-	Proposal    actors.StorageDealProposal
-	ProposalCid cid.Cid
-	State       api.DealState
-
-	Ref cid.Cid
-
-	DealID   uint64
-	SectorID uint64 // Set when State >= DealStaged
-
-	s inet.Stream
-}
-
-type Provider struct {
-	pricePerByteBlock types.BigInt // how much we want for storing one byte for one block
-	minPieceSize      uint64
-
-	ask   *types.SignedStorageAsk
-	askLk sync.Mutex
-
-	secb   *sectorblocks.SectorBlocks
-	sminer *storage.Miner
-	full   api.FullNode
-
-	// TODO: This will go away once storage market module + CAR
-	// is implemented
-	dag dtypes.StagingDAG
-
-	// dataTransfer is the manager of data transfers used by this storage provider
-	dataTransfer dtypes.ProviderDataTransfer
-
-	deals *statestore.StateStore
-	ds    dtypes.MetadataDS
-
-	conns map[cid.Cid]inet.Stream
-
-	actor address.Address
-
-	incoming chan MinerDeal
-	updated  chan minerDealUpdate
-	stop     chan struct{}
-	stopped  chan struct{}
-}
-
-type minerDealUpdate struct {
-	newState api.DealState
-	id       cid.Cid
-	err      error
-	mut      func(*MinerDeal)
-}
-
-var (
-	// ErrDataTransferFailed means a data transfer for a deal failed
-	ErrDataTransferFailed = errors.New("deal data transfer failed")
-)
-
-func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, fullNode api.FullNode) (*Provider, error) {
-	addr, err := ds.Get(datastore.NewKey("miner-address"))
-	if err != nil {
-		return nil, err
-	}
-	minerAddress, err := address.NewFromBytes(addr)
-	if err != nil {
-		return nil, err
-	}
-
-	h := &Provider{
-		sminer:       sminer,
-		dag:          dag,
-		dataTransfer: dataTransfer,
-		full:         fullNode,
-		secb:         secb,
-
-		pricePerByteBlock: types.NewInt(3), // TODO: allow setting
-		minPieceSize:      256,             // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up))
-
-		conns: map[cid.Cid]inet.Stream{},
-
-		incoming: make(chan MinerDeal),
-		updated:  make(chan minerDealUpdate),
-		stop:     make(chan struct{}),
-		stopped:  make(chan struct{}),
-
-		actor: minerAddress,
-
-		deals: statestore.New(namespace.Wrap(ds, datastore.NewKey(ProviderDsPrefix))),
-		ds:    ds,
-	}
-
-	if err := h.tryLoadAsk(); err != nil {
-		return nil, err
-	}
-
-	if h.ask == nil {
-		// TODO: we should be fine with this state, and just say it means 'not actively accepting deals'
-		// for now... lets just set a price
-		if err := h.SetPrice(types.NewInt(500_000_000), 1000000); err != nil {
-			return nil, xerrors.Errorf("failed setting a default price: %w", err)
-		}
-	}
-
-	// register a data transfer event handler -- this will move deals from
-	// accepted to staged
-	h.dataTransfer.SubscribeToEvents(h.onDataTransferEvent)
-
-	return h, nil
-}
-
-func (p *Provider) Run(ctx context.Context) {
-	// TODO: restore state
-
-	go func() {
-		defer log.Warn("quitting deal provider loop")
-		defer close(p.stopped)
-
-		for {
-			select {
-			case deal := <-p.incoming: // DealAccepted
-				p.onIncoming(deal)
-			case update := <-p.updated: // DealStaged
-				p.onUpdated(ctx, update)
-			case <-p.stop:
-				return
-			}
-		}
-	}()
-}
-
-func (p *Provider) onIncoming(deal MinerDeal) {
-	log.Info("incoming deal")
-
-	p.conns[deal.ProposalCid] = deal.s
-
-	if err := p.deals.Begin(deal.ProposalCid, &deal); err != nil {
-		// This can happen when client re-sends proposal
-		p.failDeal(deal.ProposalCid, err)
-		log.Errorf("deal tracking failed: %s", err)
-		return
-	}
-
-	go func() {
-		p.updated <- minerDealUpdate{
-			newState: api.DealAccepted,
-			id:       deal.ProposalCid,
-			err:      nil,
-		}
-	}()
-}
-
-func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) {
-	log.Infof("Deal %s updated state to %s", update.id, api.DealStates[update.newState])
-	if update.err != nil {
-		log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err)
-		p.failDeal(update.id, update.err)
-		return
-	}
-	var deal MinerDeal
-	err := p.deals.Mutate(update.id, func(d *MinerDeal) error {
-		d.State = update.newState
-		if update.mut != nil {
-			update.mut(d)
-		}
-		deal = *d
-		return nil
-	})
-	if err != nil {
-		p.failDeal(update.id, err)
-		return
-	}
-
-	switch update.newState {
-	case api.DealAccepted:
-		p.handle(ctx, deal, p.accept, api.DealNoUpdate)
-	case api.DealStaged:
-		p.handle(ctx, deal, p.staged, api.DealSealing)
-	case api.DealSealing:
-		p.handle(ctx, deal, p.sealing, api.DealComplete)
-	case api.DealComplete:
-		p.handle(ctx, deal, p.complete, api.DealNoUpdate)
-	}
-}
-
-// onDataTransferEvent is the function called when an event occurs in a data
-// transfer -- it reads the voucher to verify this even occurred in a storage
-// market deal, then, based on the data transfer event that occurred, it generates
-// and update message for the deal -- either moving to staged for a completion
-// event or moving to error if a data transfer error occurs
-func (p *Provider) onDataTransferEvent(event datatransfer.Event, channelState datatransfer.ChannelState) {
-	voucher, ok := channelState.Voucher().(*StorageDataTransferVoucher)
-	// if this event is for a transfer not related to storage, ignore
-	if !ok {
-		return
-	}
-
-	// data transfer events for opening and progress do not affect deal state
-	var next api.DealState
-	var err error
-	var mut func(*MinerDeal)
-	switch event {
-	case datatransfer.Complete:
-		next = api.DealStaged
-		mut = func(deal *MinerDeal) {
-			deal.DealID = voucher.DealID
-		}
-	case datatransfer.Error:
-		next = api.DealFailed
-		err = ErrDataTransferFailed
-	default:
-		// the only events we care about are complete and error
-		return
-	}
-
-	select {
-	case p.updated <- minerDealUpdate{
-		newState: next,
-		id:       voucher.Proposal,
-		err:      err,
-		mut:      mut,
-	}:
-	case <-p.stop:
-	}
-}
-
-func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) {
-	proposalNd, err := cborutil.AsIpld(proposal.DealProposal)
-	if err != nil {
-		return MinerDeal{}, err
-	}
-
-	return MinerDeal{
-		Client:      s.Conn().RemotePeer(),
-		Proposal:    *proposal.DealProposal,
-		ProposalCid: proposalNd.Cid(),
-		State:       api.DealUnknown,
-
-		Ref: proposal.Piece,
-
-		s: s,
-	}, nil
-}
-
-func (p *Provider) HandleStream(s inet.Stream) {
-	log.Info("Handling storage deal proposal!")
-
-	proposal, err := p.readProposal(s)
-	if err != nil {
-		log.Error(err)
-		s.Close()
-		return
-	}
-
-	deal, err := p.newDeal(s, proposal)
-	if err != nil {
-		log.Errorf("%+v", err)
-		s.Close()
-		return
-	}
-
-	p.incoming <- deal
-}
-
-func (p *Provider) Stop() {
-	close(p.stop)
-	<-p.stopped
-}

chain/deals/provider_asks.go

@@ -1,160 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"context"
-	"time"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/lotus/chain/stmgr"
-	"github.com/filecoin-project/lotus/chain/types"
-	datastore "github.com/ipfs/go-datastore"
-	inet "github.com/libp2p/go-libp2p-core/network"
-	"golang.org/x/xerrors"
-)
-
-func (p *Provider) SetPrice(price types.BigInt, ttlsecs int64) error {
-	p.askLk.Lock()
-	defer p.askLk.Unlock()
-
-	var seqno uint64
-	if p.ask != nil {
-		seqno = p.ask.Ask.SeqNo + 1
-	}
-
-	now := time.Now().Unix()
-	ask := &types.StorageAsk{
-		Price:        price,
-		Timestamp:    uint64(now),
-		Expiry:       uint64(now + ttlsecs),
-		Miner:        p.actor,
-		SeqNo:        seqno,
-		MinPieceSize: p.minPieceSize,
-	}
-
-	ssa, err := p.signAsk(ask)
-	if err != nil {
-		return err
-	}
-
-	return p.saveAsk(ssa)
-}
-
-func (p *Provider) getAsk(m address.Address) *types.SignedStorageAsk {
-	p.askLk.Lock()
-	defer p.askLk.Unlock()
-	if m != p.actor {
-		return nil
-	}
-
-	return p.ask
-}
-
-func (p *Provider) HandleAskStream(s inet.Stream) {
-	defer s.Close()
-	var ar AskRequest
-	if err := cborutil.ReadCborRPC(s, &ar); err != nil {
-		log.Errorf("failed to read AskRequest from incoming stream: %s", err)
-		return
-	}
-
-	resp := p.processAskRequest(&ar)
-
-	if err := cborutil.WriteCborRPC(s, resp); err != nil {
-		log.Errorf("failed to write ask response: %s", err)
-		return
-	}
-}
-
-func (p *Provider) processAskRequest(ar *AskRequest) *AskResponse {
-	return &AskResponse{
-		Ask: p.getAsk(ar.Miner),
-	}
-}
-
-var bestAskKey = datastore.NewKey("latest-ask")
-
-func (p *Provider) tryLoadAsk() error {
-	p.askLk.Lock()
-	defer p.askLk.Unlock()
-
-	err := p.loadAsk()
-	if err != nil {
-		if xerrors.Is(err, datastore.ErrNotFound) {
-			log.Warn("no previous ask found, miner will not accept deals until a price is set")
-			return nil
-		}
-		return err
-	}
-
-	return nil
-}
-
-func (p *Provider) loadAsk() error {
-	askb, err := p.ds.Get(datastore.NewKey("latest-ask"))
-	if err != nil {
-		return xerrors.Errorf("failed to load most recent ask from disk: %w", err)
-	}
-
-	var ssa types.SignedStorageAsk
-	if err := cborutil.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil {
-		return err
-	}
-
-	p.ask = &ssa
-	return nil
-}
-
-func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) {
-	b, err := cborutil.Dump(a)
-	if err != nil {
-		return nil, err
-	}
-
-	worker, err := p.getWorker(p.actor)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to get worker to sign ask: %w", err)
-	}
-
-	sig, err := p.full.WalletSign(context.TODO(), worker, b)
-	if err != nil {
-		return nil, err
-	}
-
-	return &types.SignedStorageAsk{
-		Ask:       a,
-		Signature: sig,
-	}, nil
-}
-
-func (p *Provider) saveAsk(a *types.SignedStorageAsk) error {
-	b, err := cborutil.Dump(a)
-	if err != nil {
-		return err
-	}
-
-	if err := p.ds.Put(bestAskKey, b); err != nil {
-		return err
-	}
-
-	p.ask = a
-	return nil
-}
-
-func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error {
-	tss := c.sm.ChainStore().GetHeaviestTipSet().ParentState()
-
-	w, err := stmgr.GetMinerWorkerRaw(context.TODO(), c.sm, tss, ask.Ask.Miner)
-	if err != nil {
-		return xerrors.Errorf("failed to get worker for miner in ask", err)
-	}
-
-	sigb, err := cborutil.Dump(ask.Ask)
-	if err != nil {
-		return xerrors.Errorf("failed to re-serialize ask")
-	}
-
-	return ask.Signature.Verify(w, sigb)
-
-}

chain/deals/provider_states.go

@@ -1,215 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"context"
-
-	ipldfree "github.com/ipld/go-ipld-prime/impl/free"
-	"github.com/ipld/go-ipld-prime/traversal/selector"
-	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
-
-	unixfile "github.com/ipfs/go-unixfs/file"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/padreader"
-	"github.com/filecoin-project/lotus/storage/sectorblocks"
-)
-
-type providerHandlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error)
-
-func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandlerFunc, next api.DealState) {
-	go func() {
-		mut, err := cb(ctx, deal)
-
-		if err == nil && next == api.DealNoUpdate {
-			return
-		}
-
-		select {
-		case p.updated <- minerDealUpdate{
-			newState: next,
-			id:       deal.ProposalCid,
-			err:      err,
-			mut:      mut,
-		}:
-		case <-p.stop:
-		}
-	}()
-}
-
-// ACCEPTED
-func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
-
-	head, err := p.full.ChainHead(ctx)
-	if err != nil {
-		return nil, err
-	}
-	if head.Height() >= deal.Proposal.ProposalExpiration {
-		return nil, xerrors.Errorf("deal proposal already expired")
-	}
-
-	// TODO: check StorageCollateral
-
-	minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1<<30))
-	if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) {
-		return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice)
-	}
-
-	if deal.Proposal.PieceSize < p.ask.Ask.MinPieceSize {
-		return nil, xerrors.Errorf("piece size less than minimum required size: %d < %d", deal.Proposal.PieceSize, p.ask.Ask.MinPieceSize)
-	}
-
-	// check market funds
-	clientMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client, nil)
-	if err != nil {
-		return nil, xerrors.Errorf("getting client market balance failed: %w", err)
-	}
-
-	// This doesn't guarantee that the client won't withdraw / lock those funds
-	// but it's a decent first filter
-	if clientMarketBalance.Available.LessThan(deal.Proposal.TotalStoragePrice()) {
-		return nil, xerrors.New("clientMarketBalance.Available too small")
-	}
-
-	waddr, err := p.full.StateMinerWorker(ctx, deal.Proposal.Provider, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	// TODO: check StorageCollateral (may be too large (or too small))
-	if err := p.full.MarketEnsureAvailable(ctx, waddr, deal.Proposal.StorageCollateral); err != nil {
-		return nil, err
-	}
-
-	log.Info("publishing deal")
-
-	params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{
-		Deals: []actors.StorageDealProposal{deal.Proposal},
-	})
-	if err != nil {
-		return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err)
-	}
-
-	// TODO: We may want this to happen after fetching data
-	smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{
-		To:       actors.StorageMarketAddress,
-		From:     waddr,
-		Value:    types.NewInt(0),
-		GasPrice: types.NewInt(0),
-		GasLimit: types.NewInt(1000000),
-		Method:   actors.SMAMethods.PublishStorageDeals,
-		Params:   params,
-	})
-	if err != nil {
-		return nil, err
-	}
-	r, err := p.full.StateWaitMsg(ctx, smsg.Cid())
-	if err != nil {
-		return nil, err
-	}
-	if r.Receipt.ExitCode != 0 {
-		return nil, xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode)
-	}
-	var resp actors.PublishStorageDealResponse
-	if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil {
-		return nil, err
-	}
-	if len(resp.DealIDs) != 1 {
-		return nil, xerrors.Errorf("got unexpected number of DealIDs from SMA")
-	}
-
-	log.Infof("fetching data for a deal %d", resp.DealIDs[0])
-	err = p.sendSignedResponse(&Response{
-		State: api.DealAccepted,
-
-		Proposal:              deal.ProposalCid,
-		StorageDealSubmission: smsg,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	if err := p.disconnect(deal); err != nil {
-		log.Warnf("closing client connection: %+v", err)
-	}
-
-	ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder())
-
-	// this is the selector for "get the whole DAG"
-	// TODO: support storage deals with custom payload selectors
-	allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(),
-		ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
-
-	// initiate a pull data transfer. This will complete asynchronously and the
-	// completion of the data transfer will trigger a change in deal state
-	// (see onDataTransferEvent)
-	_, err = p.dataTransfer.OpenPullDataChannel(ctx,
-		deal.Client,
-		&StorageDataTransferVoucher{Proposal: deal.ProposalCid, DealID: resp.DealIDs[0]},
-		deal.Ref,
-		allSelector,
-	)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to open pull data channel: %w", err)
-	}
-
-	return nil, nil
-}
-
-// STAGED
-
-func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
-	root, err := p.dag.Get(ctx, deal.Ref)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to get file root for deal: %s", err)
-	}
-
-	// TODO: abstract this away into ReadSizeCloser + implement different modes
-	n, err := unixfile.NewUnixfsFile(ctx, p.dag, root)
-	if err != nil {
-		return nil, xerrors.Errorf("cannot open unixfs file: %s", err)
-	}
-
-	uf, ok := n.(sectorblocks.UnixfsReader)
-	if !ok {
-		// we probably got directory, unsupported for now
-		return nil, xerrors.Errorf("unsupported unixfs file type")
-	}
-
-	// TODO: uf.Size() is user input, not trusted
-	// This won't be useful / here after we migrate to putting CARs into sectors
-	size, err := uf.Size()
-	if err != nil {
-		return nil, xerrors.Errorf("getting unixfs file size: %w", err)
-	}
-	if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize {
-		return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size")
-	}
-
-	sectorID, err := p.secb.AddUnixfsPiece(ctx, uf, deal.DealID)
-	if err != nil {
-		return nil, xerrors.Errorf("AddPiece failed: %s", err)
-	}
-	log.Warnf("New Sector: %d (deal %d)", sectorID, deal.DealID)
-
-	return func(deal *MinerDeal) {
-		deal.SectorID = sectorID
-	}, nil
-}
-
-// SEALING
-
-func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
-	// TODO: consider waiting for seal to happen
-
-	return nil, nil
-}
-
-func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
-	// TODO: observe sector lifecycle, status, expiration..
-
-	return nil, nil
-}

chain/deals/provider_utils.go

@@ -1,198 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"context"
-	"runtime"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/datatransfer"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/ipld/go-ipld-prime"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/go-statestore"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/types"
-
-	"github.com/ipfs/go-cid"
-	inet "github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"golang.org/x/xerrors"
-)
-
-func (p *Provider) failDeal(id cid.Cid, cerr error) {
-	if err := p.deals.End(id); err != nil {
-		log.Warnf("deals.End: %s", err)
-	}
-
-	if cerr == nil {
-		_, f, l, _ := runtime.Caller(1)
-		cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l)
-	}
-
-	log.Warnf("deal %s failed: %s", id, cerr)
-
-	err := p.sendSignedResponse(&Response{
-		State:    api.DealFailed,
-		Message:  cerr.Error(),
-		Proposal: id,
-	})
-
-	s, ok := p.conns[id]
-	if ok {
-		_ = s.Reset()
-		delete(p.conns, id)
-	}
-
-	if err != nil {
-		log.Warnf("notifying client about deal failure: %s", err)
-	}
-}
-
-func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) {
-	if err := cborutil.ReadCborRPC(s, &proposal); err != nil {
-		log.Errorw("failed to read proposal message", "error", err)
-		return proposal, err
-	}
-
-	if err := proposal.DealProposal.Verify(); err != nil {
-		return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err)
-	}
-
-	if proposal.DealProposal.Provider != p.actor {
-		log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider)
-		return proposal, err
-	}
-
-	return
-}
-
-func (p *Provider) sendSignedResponse(resp *Response) error {
-	s, ok := p.conns[resp.Proposal]
-	if !ok {
-		return xerrors.New("couldn't send response: not connected")
-	}
-
-	msg, err := cborutil.Dump(resp)
-	if err != nil {
-		return xerrors.Errorf("serializing response: %w", err)
-	}
-
-	worker, err := p.getWorker(p.actor)
-	if err != nil {
-		return err
-	}
-
-	sig, err := p.full.WalletSign(context.TODO(), worker, msg)
-	if err != nil {
-		return xerrors.Errorf("failed to sign response message: %w", err)
-	}
-
-	signedResponse := &SignedResponse{
-		Response:  *resp,
-		Signature: sig,
-	}
-
-	err = cborutil.WriteCborRPC(s, signedResponse)
-	if err != nil {
-		// Assume client disconnected
-		s.Close()
-		delete(p.conns, resp.Proposal)
-	}
-	return err
-}
-
-func (p *Provider) disconnect(deal MinerDeal) error {
-	s, ok := p.conns[deal.ProposalCid]
-	if !ok {
-		return nil
-	}
-
-	err := s.Close()
-	delete(p.conns, deal.ProposalCid)
-	return err
-}
-
-func (p *Provider) getWorker(miner address.Address) (address.Address, error) {
-	getworker := &types.Message{
-		To:     miner,
-		From:   miner,
-		Method: actors.MAMethods.GetWorkerAddr,
-	}
-	r, err := p.full.StateCall(context.TODO(), getworker, nil)
-	if err != nil {
-		return address.Undef, xerrors.Errorf("getting worker address: %w", err)
-	}
-
-	if r.ExitCode != 0 {
-		return address.Undef, xerrors.Errorf("getWorker call failed: %d", r.ExitCode)
-	}
-
-	return address.NewFromBytes(r.Return)
-}
-
-var _ datatransfer.RequestValidator = &ProviderRequestValidator{}
-
-// ProviderRequestValidator validates data transfer requests for the provider
-// in a storage market
-type ProviderRequestValidator struct {
-	deals *statestore.StateStore
-}
-
-// NewProviderRequestValidator returns a new client request validator for the
-// given datastore
-func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *ProviderRequestValidator {
-	return &ProviderRequestValidator{
-		deals: deals,
-	}
-}
-
-// ValidatePush validates a push request received from the peer that will send data
-// Will succeed only if:
-// - voucher has correct type
-// - voucher references an active deal
-// - referenced deal matches the client
-// - referenced deal matches the given base CID
-// - referenced deal is in an acceptable state
-func (m *ProviderRequestValidator) ValidatePush(
-	sender peer.ID,
-	voucher datatransfer.Voucher,
-	baseCid cid.Cid,
-	Selector ipld.Node) error {
-	dealVoucher, ok := voucher.(*StorageDataTransferVoucher)
-	if !ok {
-		return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType)
-	}
-
-	var deal MinerDeal
-	err := m.deals.Get(dealVoucher.Proposal, &deal)
-	if err != nil {
-		return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal)
-	}
-	if deal.Client != sender {
-		return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Client.String(), sender.String(), ErrWrongPeer)
-	}
-
-	if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) {
-		return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece)
-	}
-	for _, state := range DataTransferStates {
-		if deal.State == state {
-			return nil
-		}
-	}
-	return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState)
-}
-
-// ValidatePull validates a pull request received from the peer that will receive data.
-// Will always error because providers should not accept pull requests from a client
-// in a storage deal (i.e. send data to client).
-func (m *ProviderRequestValidator) ValidatePull(
-	receiver peer.ID,
-	voucher datatransfer.Voucher,
-	baseCid cid.Cid,
-	Selector ipld.Node) error {
-	return ErrNoPullAccepted
-}

chain/deals/request_validation_test.go

@@ -1,291 +0,0 @@
-package deals_test
-
-import (
-	"fmt"
-	"math/rand"
-	"testing"
-
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
-	dss "github.com/ipfs/go-datastore/sync"
-	blocksutil "github.com/ipfs/go-ipfs-blocksutil"
-	"github.com/libp2p/go-libp2p-core/peer"
-	xerrors "golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/go-statestore"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/deals"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-var blockGenerator = blocksutil.NewBlockGenerator()
-
-type wrongDTType struct {
-}
-
-func (wrongDTType) ToBytes() ([]byte, error) {
-	return []byte{}, nil
-}
-
-func (wrongDTType) FromBytes([]byte) error {
-	return fmt.Errorf("not implemented")
-}
-
-func (wrongDTType) Identifier() string {
-	return "WrongDTTYPE"
-}
-
-func uniqueStorageDealProposal() (actors.StorageDealProposal, error) {
-	clientAddr, err := address.NewIDAddress(uint64(rand.Int()))
-	if err != nil {
-		return actors.StorageDealProposal{}, err
-	}
-	providerAddr, err := address.NewIDAddress(uint64(rand.Int()))
-	if err != nil {
-		return actors.StorageDealProposal{}, err
-	}
-	return actors.StorageDealProposal{
-		PieceRef: blockGenerator.Next().Cid().Bytes(),
-		Client:   clientAddr,
-		Provider: providerAddr,
-		ProposerSignature: &types.Signature{
-			Data: []byte("foo bar cat dog"),
-			Type: types.KTBLS,
-		},
-	}, nil
-}
-
-func newClientDeal(minerID peer.ID, state api.DealState) (deals.ClientDeal, error) {
-	newProposal, err := uniqueStorageDealProposal()
-	if err != nil {
-		return deals.ClientDeal{}, err
-	}
-	proposalNd, err := cborutil.AsIpld(&newProposal)
-	if err != nil {
-		return deals.ClientDeal{}, err
-	}
-	minerAddr, err := address.NewIDAddress(uint64(rand.Int()))
-	if err != nil {
-		return deals.ClientDeal{}, err
-	}
-
-	return deals.ClientDeal{
-		Proposal:    newProposal,
-		ProposalCid: proposalNd.Cid(),
-		Miner:       minerID,
-		MinerWorker: minerAddr,
-		State:       state,
-	}, nil
-}
-
-func newMinerDeal(clientID peer.ID, state api.DealState) (deals.MinerDeal, error) {
-	newProposal, err := uniqueStorageDealProposal()
-	if err != nil {
-		return deals.MinerDeal{}, err
-	}
-	proposalNd, err := cborutil.AsIpld(&newProposal)
-	if err != nil {
-		return deals.MinerDeal{}, err
-	}
-	ref, err := cid.Cast(newProposal.PieceRef)
-	if err != nil {
-		return deals.MinerDeal{}, err
-	}
-
-	return deals.MinerDeal{
-		Proposal:    newProposal,
-		ProposalCid: proposalNd.Cid(),
-		Client:      clientID,
-		State:       state,
-		Ref:         ref,
-	}, nil
-}
-
-func TestClientRequestValidation(t *testing.T) {
-	ds := dss.MutexWrap(datastore.NewMapDatastore())
-	state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))
-
-	crv := deals.NewClientRequestValidator(state)
-	minerID := peer.ID("fakepeerid")
-	block := blockGenerator.Next()
-	t.Run("ValidatePush fails", func(t *testing.T) {
-		if !xerrors.Is(crv.ValidatePush(minerID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPushAccepted) {
-			t.Fatal("Push should fail for the client request validator for storage deals")
-		}
-	})
-	t.Run("ValidatePull fails deal not found", func(t *testing.T) {
-		proposal, err := uniqueStorageDealProposal()
-		if err != nil {
-			t.Fatal("error creating proposal")
-		}
-		proposalNd, err := cborutil.AsIpld(&proposal)
-		if err != nil {
-			t.Fatal("error serializing proposal")
-		}
-		pieceRef, err := cid.Cast(proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) {
-			t.Fatal("Pull should fail if there is no deal stored")
-		}
-	})
-	t.Run("ValidatePull fails wrong client", func(t *testing.T) {
-		otherMiner := peer.ID("otherminer")
-		clientDeal, err := newClientDeal(otherMiner, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) {
-			t.Fatal("Pull should fail if miner address is incorrect")
-		}
-	})
-	t.Run("ValidatePull fails wrong piece ref", func(t *testing.T) {
-		clientDeal, err := newClientDeal(minerID, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
-			t.Fatal("Pull should fail if piece ref is incorrect")
-		}
-	})
-	t.Run("ValidatePull fails wrong deal state", func(t *testing.T) {
-		clientDeal, err := newClientDeal(minerID, api.DealComplete)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) {
-			t.Fatal("Pull should fail if deal is in a state that cannot be data transferred")
-		}
-	})
-	t.Run("ValidatePull succeeds", func(t *testing.T) {
-		clientDeal, err := newClientDeal(minerID, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil) != nil {
-			t.Fatal("Pull should should succeed when all parameters are correct")
-		}
-	})
-}
-
-func TestProviderRequestValidation(t *testing.T) {
-	ds := dss.MutexWrap(datastore.NewMapDatastore())
-	state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))
-
-	mrv := deals.NewProviderRequestValidator(state)
-	clientID := peer.ID("fakepeerid")
-	block := blockGenerator.Next()
-	t.Run("ValidatePull fails", func(t *testing.T) {
-		if !xerrors.Is(mrv.ValidatePull(clientID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPullAccepted) {
-			t.Fatal("Pull should fail for the provider request validator for storage deals")
-		}
-	})
-
-	t.Run("ValidatePush fails deal not found", func(t *testing.T) {
-		proposal, err := uniqueStorageDealProposal()
-		if err != nil {
-			t.Fatal("error creating proposal")
-		}
-		proposalNd, err := cborutil.AsIpld(&proposal)
-		if err != nil {
-			t.Fatal("error serializing proposal")
-		}
-		pieceRef, err := cid.Cast(proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) {
-			t.Fatal("Push should fail if there is no deal stored")
-		}
-	})
-	t.Run("ValidatePush fails wrong miner", func(t *testing.T) {
-		otherClient := peer.ID("otherclient")
-		minerDeal, err := newMinerDeal(otherClient, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) {
-			t.Fatal("Push should fail if miner address is incorrect")
-		}
-	})
-	t.Run("ValidatePush fails wrong piece ref", func(t *testing.T) {
-		minerDeal, err := newMinerDeal(clientID, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
-			t.Fatal("Push should fail if piece ref is incorrect")
-		}
-	})
-	t.Run("ValidatePush fails wrong deal state", func(t *testing.T) {
-		minerDeal, err := newMinerDeal(clientID, api.DealComplete)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) {
-			t.Fatal("Push should fail if deal is in a state that cannot be data transferred")
-		}
-	})
-	t.Run("ValidatePush succeeds", func(t *testing.T) {
-		minerDeal, err := newMinerDeal(clientID, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil) != nil {
-			t.Fatal("Push should should succeed when all parameters are correct")
-		}
-	})
-}

chain/deals/types.go

@@ -1,114 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"errors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/ipfs/go-cid"
-)
-
-var (
-	// ErrWrongVoucherType means the voucher was not the correct type can validate against
-	ErrWrongVoucherType = errors.New("cannot validate voucher type.")
-
-	// ErrNoPushAccepted just means clients do not accept pushes for storage deals
-	ErrNoPushAccepted = errors.New("client should not receive data for a storage deal.")
-
-	// ErrNoPullAccepted just means providers do not accept pulls for storage deals
-	ErrNoPullAccepted = errors.New("provider should not send data for a storage deal.")
-
-	// ErrNoDeal means no active deal was found for this vouchers proposal cid
-	ErrNoDeal = errors.New("no deal found for this proposal.")
-
-	// ErrWrongPeer means that the other peer for this data transfer request does not match
-	// the other peer for the deal
-	ErrWrongPeer = errors.New("data Transfer peer id and Deal peer id do not match.")
-
-	// ErrWrongPiece means that the pieceref for this data transfer request does not match
-	// the one specified in the deal
-	ErrWrongPiece = errors.New("base CID for deal does not match CID for piece.")
-
-	// ErrInacceptableDealState means the deal for this transfer is not in a deal state
-	// where transfer can be performed
-	ErrInacceptableDealState = errors.New("deal is not a in a state where deals are accepted.")
-
-	// DataTransferStates are the states in which it would make sense to actually start a data transfer
-	DataTransferStates = []api.DealState{api.DealAccepted, api.DealUnknown}
-)
-
-const DealProtocolID = "/fil/storage/mk/1.0.1"
-const AskProtocolID = "/fil/storage/ask/1.0.1"
-
-type Proposal struct {
-	DealProposal *actors.StorageDealProposal
-
-	Piece cid.Cid // Used for retrieving from the client
-}
-
-type Response struct {
-	State api.DealState
-
-	// DealProposalRejected
-	Message  string
-	Proposal cid.Cid
-
-	// DealAccepted
-	StorageDealSubmission *types.SignedMessage
-}
-
-// TODO: Do we actually need this to be signed?
-type SignedResponse struct {
-	Response Response
-
-	Signature *types.Signature
-}
-
-func (r *SignedResponse) Verify(addr address.Address) error {
-	b, err := cborutil.Dump(&r.Response)
-	if err != nil {
-		return err
-	}
-
-	return r.Signature.Verify(addr, b)
-}
-
-type AskRequest struct {
-	Miner address.Address
-}
-
-type AskResponse struct {
-	Ask *types.SignedStorageAsk
-}
-
-// StorageDataTransferVoucher is the voucher type for data transfers
-// used by the storage market
-type StorageDataTransferVoucher struct {
-	Proposal cid.Cid
-	DealID   uint64
-}
-
-// ToBytes converts the StorageDataTransferVoucher to raw bytes
-func (dv *StorageDataTransferVoucher) ToBytes() ([]byte, error) {
-	var buf bytes.Buffer
-	err := dv.MarshalCBOR(&buf)
-	if err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-// FromBytes converts the StorageDataTransferVoucher to raw bytes
-func (dv *StorageDataTransferVoucher) FromBytes(raw []byte) error {
-	r := bytes.NewReader(raw)
-	return dv.UnmarshalCBOR(r)
-}
-
-// Identifier is the unique string identifier for a StorageDataTransferVoucher
-func (dv *StorageDataTransferVoucher) Identifier() string {
-	return "StorageDataTransferVoucher"
-}

chain/events/events.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"

chain/gen/gen.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/binary"
 	"fmt"
+	"github.com/filecoin-project/lotus/chain/vm"
 	"io/ioutil"
 	"sync/atomic"
 
@@ -34,7 +35,7 @@ import (
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 )
 
 var log = logging.Logger("gen")
@@ -170,7 +171,9 @@ func NewGenerator() (*ChainGen, error) {
 		MinerAddrs: []address.Address{maddr1, maddr2},
 	}
 
-	genb, err := MakeGenesisBlock(bs, map[address.Address]types.BigInt{
+	sys := vm.Syscalls(sectorbuilder.ProofVerifier)
+
+	genb, err := MakeGenesisBlock(bs, sys, map[address.Address]types.BigInt{
 		mk1:    types.FromFil(40000),
 		mk2:    types.FromFil(40000),
 		banker: types.FromFil(50000),
@@ -179,7 +182,7 @@ func NewGenerator() (*ChainGen, error) {
 		return nil, xerrors.Errorf("make genesis block failed: %w", err)
 	}
 
-	cs := store.NewChainStore(bs, ds)
+	cs := store.NewChainStore(bs, ds, sys)
 
 	genfb := &types.FullBlock{Header: genb.Genesis}
 	gents := store.NewFullTipSet([]*types.FullBlock{genfb})

chain/gen/utils.go

@@ -275,7 +275,7 @@ func mustEnc(i cbg.CBORMarshaler) []byte {
 }
 
 func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDealProposal, error) {
-	vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore())
+	vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore(), cs.VMSys())
 	if err != nil {
 		return cid.Undef, nil, xerrors.Errorf("failed to create NewVM: %w", err)
 	}
@@ -555,7 +555,7 @@ func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value
 	return ret.Return, nil
 }
 
-func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.BigInt, gmcfg *GenMinerCfg, ts uint64) (*GenesisBootstrap, error) {
+func MakeGenesisBlock(bs bstore.Blockstore, sys *types.VMSyscalls, balances map[address.Address]types.BigInt, gmcfg *GenMinerCfg, ts uint64) (*GenesisBootstrap, error) {
 	ctx := context.Background()
 
 	state, err := MakeInitialStateTree(bs, balances)
@@ -569,7 +569,7 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B
 	}
 
 	// temp chainstore
-	cs := store.NewChainStore(bs, datastore.NewMapDatastore())
+	cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys)
 	stateroot, deals, err := SetupStorageMiners(ctx, cs, stateroot, gmcfg)
 	if err != nil {
 		return nil, xerrors.Errorf("setup storage miners failed: %w", err)

chain/messagepool/messagepool.go

@@ -13,7 +13,7 @@ import (
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
 	"github.com/ipfs/go-datastore/query"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	lps "github.com/whyrusleeping/pubsub"
 	"go.uber.org/multierr"
@@ -94,7 +94,7 @@ func (ms *msgSet) add(m *types.SignedMessage) error {
 	if _, has := ms.msgs[m.Message.Nonce]; has {
 		if m.Cid() != ms.msgs[m.Message.Nonce].Cid() {
 			log.Error("Add with duplicate nonce")
-			return xerrors.Errorf("message to %s with nonce %d already in mpool")
+			return xerrors.Errorf("message to %s with nonce %d already in mpool", m.Message.To, m.Message.Nonce)
 		}
 	}
 	ms.msgs[m.Message.Nonce] = m

chain/metrics/consensus.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"go.uber.org/fx"
 

chain/state/statetree.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/ipfs/go-cid"
 	hamt "github.com/ipfs/go-hamt-ipld"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"

chain/stmgr/call.go

@@ -18,7 +18,7 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate
 	ctx, span := trace.StartSpan(ctx, "statemanager.CallRaw")
 	defer span.End()
 
-	vmi, err := vm.NewVM(bstate, bheight, r, actors.NetworkAddress, sm.cs.Blockstore())
+	vmi, err := vm.NewVM(bstate, bheight, r, actors.NetworkAddress, sm.cs.Blockstore(), sm.cs.VMSys())
 	if err != nil {
 		return nil, xerrors.Errorf("failed to set up vm: %w", err)
 	}

chain/stmgr/fork_no_p_eps.go

@@ -1,127 +0,0 @@
-package stmgr
-
-import (
-	"context"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-amt-ipld"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-hamt-ipld"
-	cbor "github.com/ipfs/go-ipld-cbor"
-	"github.com/whyrusleeping/cbor-gen"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/state"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-func (sm *StateManager) forkNoPowerEPS(ctx context.Context, pstate cid.Cid) (cid.Cid, error) {
-	cst := hamt.CSTFromBstore(sm.cs.Blockstore())
-	st, err := state.LoadStateTree(cst, pstate)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("loading parent state tree: %w", err)
-	}
-
-	if err := st.MutateActor(actors.StoragePowerAddress, func(spa *types.Actor) error {
-		var head actors.StoragePowerState
-		if err := cst.Get(ctx, spa.Head, &head); err != nil {
-			return xerrors.Errorf("reading StoragePower state: %w", err)
-		}
-
-		buckets, err := amt.LoadAMT(amt.WrapBlockstore(sm.cs.Blockstore()), head.ProvingBuckets)
-		if err != nil {
-			return xerrors.Errorf("opening proving buckets AMT: %w", err)
-		}
-
-		fixedBuckets := map[uint64]map[address.Address]struct{}{}
-
-		if err := buckets.ForEach(func(bucketId uint64, ent *typegen.Deferred) error {
-			var bcid cid.Cid
-			if err := cbor.DecodeInto(ent.Raw, &bcid); err != nil {
-				return xerrors.Errorf("decoding bucket cid: %w", err)
-			}
-
-			bucket, err := hamt.LoadNode(ctx, cst, bcid)
-			if err != nil {
-				return xerrors.Errorf("loading bucket hamt: %w", err)
-			}
-
-			return bucket.ForEach(ctx, func(abytes string, _ interface{}) error {
-				addr, err := address.NewFromBytes([]byte(abytes))
-				if err != nil {
-					return xerrors.Errorf("parsing address in proving bucket: %w", err)
-				}
-
-				// now find the correct bucket
-				miner, err := st.GetActor(addr)
-				if err != nil {
-					return xerrors.Errorf("getting miner %s: %w", addr, err)
-				}
-
-				var minerHead actors.StorageMinerActorState
-				if err := cst.Get(ctx, miner.Head, &minerHead); err != nil {
-					return xerrors.Errorf("reading miner %s state: %w", addr, err)
-				}
-
-				correctBucket := minerHead.ElectionPeriodStart % build.SlashablePowerDelay
-				if correctBucket != bucketId {
-					log.Warnf("miner %s was in wrong proving bucket %d, putting in %d (eps: %d)", addr, bucketId, correctBucket, minerHead.ElectionPeriodStart)
-				}
-
-				if _, ok := fixedBuckets[correctBucket]; !ok {
-					fixedBuckets[correctBucket] = map[address.Address]struct{}{}
-				}
-				fixedBuckets[correctBucket][addr] = struct{}{}
-
-				return nil
-			})
-		}); err != nil {
-			return err
-		}
-
-		// /////
-		// Write fixed buckets
-
-		fixed := amt.NewAMT(amt.WrapBlockstore(sm.cs.Blockstore()))
-
-		for bucketId, addrss := range fixedBuckets {
-			bucket := hamt.NewNode(cst)
-			for addr := range addrss {
-				if err := bucket.Set(ctx, string(addr.Bytes()), actors.CborNull); err != nil {
-					return xerrors.Errorf("setting address in bucket: %w", err)
-				}
-			}
-
-			if err := bucket.Flush(ctx); err != nil {
-				return xerrors.Errorf("flushing bucket amt: %w", err)
-			}
-
-			bcid, err := cst.Put(context.TODO(), bucket)
-			if err != nil {
-				return xerrors.Errorf("put bucket: %w", err)
-			}
-
-			if err := fixed.Set(bucketId, bcid); err != nil {
-				return xerrors.Errorf("set bucket: %w", err)
-			}
-		}
-
-		head.ProvingBuckets, err = fixed.Flush()
-		if err != nil {
-			return xerrors.Errorf("flushing bucket amt: %w", err)
-		}
-
-		spa.Head, err = cst.Put(ctx, &head)
-		if err != nil {
-			return xerrors.Errorf("putting actor head: %w", err)
-		}
-
-		return nil
-	}); err != nil {
-		return cid.Undef, err
-	}
-
-	return st.Flush()
-}

chain/stmgr/forks.go

@@ -2,24 +2,150 @@ package stmgr
 
 import (
 	"context"
-	"github.com/ipfs/go-cid"
-	"golang.org/x/xerrors"
 
+	amt "github.com/filecoin-project/go-amt-ipld/v2"
 	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/state"
+	"github.com/ipfs/go-cid"
+	hamt "github.com/ipfs/go-hamt-ipld"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"golang.org/x/xerrors"
 )
 
 func (sm *StateManager) handleStateForks(ctx context.Context, pstate cid.Cid, height, parentH uint64) (_ cid.Cid, err error) {
 	for i := parentH; i < height; i++ {
 		switch i {
-		case build.ForkNoPowerEPSUpdates:
-			pstate, err = sm.forkNoPowerEPS(ctx, pstate)
+		case build.ForkBlizzardHeight:
+			log.Warnw("Executing blizzard fork logic", "height", i)
+			pstate, err = fixBlizzardAMTBug(ctx, sm, pstate)
 			if err != nil {
-				return cid.Undef, xerrors.Errorf("executing state fork in epoch %d: %w", i, err)
+				return cid.Undef, xerrors.Errorf("blizzard bug fix failed: %w", err)
+			}
+		case build.ForkFrigidHeight:
+			log.Warnw("Executing frigid fork logic", "height", i)
+			pstate, err = fixBlizzardAMTBug(ctx, sm, pstate)
+			if err != nil {
+				return cid.Undef, xerrors.Errorf("frigid bug fix failed: %w", err)
+			}
+		case build.ForkBootyBayHeight:
+			log.Warnw("Executing booty bay fork logic", "height", i)
+			pstate, err = fixBlizzardAMTBug(ctx, sm, pstate)
+			if err != nil {
+				return cid.Undef, xerrors.Errorf("booty bay bug fix failed: %w", err)
 			}
-
-			log.Infof("forkNoPowerEPS state: %s", pstate)
 		}
 	}
 
 	return pstate, nil
 }
+
+/*
+1) Iterate through each miner in the chain:
+	1.1) Fixup their sector set and proving set
+	1.2) Change their code cid to point to the new miner actor code
+*/
+func fixBlizzardAMTBug(ctx context.Context, sm *StateManager, pstate cid.Cid) (cid.Cid, error) {
+	cst := hamt.CSTFromBstore(sm.cs.Blockstore())
+	st, err := state.LoadStateTree(cst, pstate)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	spa, err := st.GetActor(actors.StoragePowerAddress)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to get storage power actor: %w", err)
+	}
+
+	var spast actors.StoragePowerState
+	if err := cst.Get(ctx, spa.Head, &spast); err != nil {
+		return cid.Undef, err
+	}
+
+	miners, err := actors.MinerSetList(ctx, cst, spast.Miners)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	for _, m := range miners {
+		mact, err := st.GetActor(m)
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("getting miner actor to fix: %w", err)
+		}
+
+		nhead, err := fixMiner(ctx, cst, sm.cs.Blockstore(), mact.Head)
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("fixing miner: %w", err)
+		}
+
+		if nhead != mact.Head {
+			log.Warnf("Miner %s had changes", m)
+		}
+
+		mact.Head = nhead
+		mact.Code = actors.StorageMiner2CodeCid
+
+		if err := st.SetActor(m, mact); err != nil {
+			return cid.Undef, err
+		}
+	}
+
+	return st.Flush()
+}
+
+func fixMiner(ctx context.Context, cst *hamt.CborIpldStore, bs blockstore.Blockstore, mscid cid.Cid) (cid.Cid, error) {
+	var mstate actors.StorageMinerActorState
+	if err := cst.Get(ctx, mscid, &mstate); err != nil {
+		return cid.Undef, xerrors.Errorf("failed to load miner actor state: %w", err)
+	}
+
+	amts := amt.WrapBlockstore(bs)
+
+	nsectors, err := amtFsck(amts, mstate.Sectors)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error fsck'ing sector set: %w", err)
+	}
+	mstate.Sectors = nsectors
+
+	nproving, err := amtFsck(amts, mstate.ProvingSet)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("error fsck'ing proving set: %w", err)
+	}
+	mstate.ProvingSet = nproving
+
+	nmcid, err := cst.Put(ctx, &mstate)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("failed to put modified miner state: %w", err)
+	}
+
+	return nmcid, nil
+}
+
+func amtFsck(s amt.Blocks, ss cid.Cid) (cid.Cid, error) {
+	a, err := amt.LoadAMT(s, ss)
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("could not load AMT: %w", a)
+	}
+
+	b := amt.NewAMT(s)
+
+	err = a.ForEach(func(id uint64, data *cbg.Deferred) error {
+		err := b.Set(id, data)
+		if err != nil {
+			return xerrors.Errorf("could not copy at idx (%d): %w", id, err)
+		}
+		return nil
+	})
+
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("could not copy: %w", err)
+	}
+
+	nss, err := b.Flush()
+	if err != nil {
+		return cid.Undef, xerrors.Errorf("could not flush: %w", err)
+	}
+
+	return nss, nil
+}

chain/stmgr/stmgr.go

@@ -18,7 +18,7 @@ import (
 	bls "github.com/filecoin-project/filecoin-ffi"
 	"github.com/ipfs/go-cid"
 	hamt "github.com/ipfs/go-hamt-ipld"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"go.opencensus.io/trace"
 )
 
@@ -139,7 +139,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
 
 	r := store.NewChainRand(sm.cs, cids, blks[0].Height)
 
-	vmi, err := vm.NewVM(pstate, blks[0].Height, r, address.Undef, sm.cs.Blockstore())
+	vmi, err := vm.NewVM(pstate, blks[0].Height, r, address.Undef, sm.cs.Blockstore(), sm.cs.VMSys())
 	if err != nil {
 		return cid.Undef, cid.Undef, xerrors.Errorf("instantiating VM failed: %w", err)
 	}

chain/stmgr/utils.go

@@ -10,7 +10,9 @@ import (
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
 
 	amt "github.com/filecoin-project/go-amt-ipld"
 	cid "github.com/ipfs/go-cid"
@@ -311,3 +313,37 @@ func LoadSectorsFromSet(ctx context.Context, bs blockstore.Blockstore, ssc cid.C
 
 	return sset, nil
 }
+
+func ComputeState(ctx context.Context, sm *StateManager, height uint64, msgs []*types.Message, ts *types.TipSet) (cid.Cid, error) {
+	if ts == nil {
+		ts = sm.cs.GetHeaviestTipSet()
+	}
+
+	base, _, err := sm.TipSetState(ctx, ts)
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	fstate, err := sm.handleStateForks(ctx, base, height, ts.Height())
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	r := store.NewChainRand(sm.cs, ts.Cids(), height)
+	vmi, err := vm.NewVM(fstate, height, r, actors.NetworkAddress, sm.cs.Blockstore(), sm.cs.VMSys())
+	if err != nil {
+		return cid.Undef, err
+	}
+
+	for i, msg := range msgs {
+		ret, err := vmi.ApplyMessage(ctx, msg)
+		if err != nil {
+			return cid.Undef, xerrors.Errorf("applying message %s: %w", msg.Cid(), err)
+		}
+		if ret.ExitCode != 0 {
+			log.Infof("compute state apply message %d failed (exit: %d): %s", i, ret.ExitCode, ret.ActorErr)
+		}
+	}
+
+	return vmi.Flush(ctx)
+}

chain/store/store.go

@@ -23,7 +23,7 @@ import (
 	dstore "github.com/ipfs/go-datastore"
 	hamt "github.com/ipfs/go-hamt-ipld"
 	bstore "github.com/ipfs/go-ipfs-blockstore"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	pubsub "github.com/whyrusleeping/pubsub"
 	"golang.org/x/xerrors"
@@ -51,9 +51,11 @@ type ChainStore struct {
 
 	mmCache *lru.ARCCache
 	tsCache *lru.ARCCache
+
+	vmcalls *types.VMSyscalls
 }
 
-func NewChainStore(bs bstore.Blockstore, ds dstore.Batching) *ChainStore {
+func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls *types.VMSyscalls) *ChainStore {
 	c, _ := lru.NewARC(2048)
 	tsc, _ := lru.NewARC(4096)
 	cs := &ChainStore{
@@ -63,6 +65,7 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching) *ChainStore {
 		tipsets:  make(map[uint64][]cid.Cid),
 		mmCache:  c,
 		tsCache:  tsc,
+		vmcalls:  vmcalls,
 	}
 
 	cs.reorgCh = cs.reorgWorker(context.TODO())
@@ -727,6 +730,30 @@ func (cs *ChainStore) readMsgMetaCids(mmc cid.Cid) ([]cid.Cid, []cid.Cid, error)
 	return blscids, secpkcids, nil
 }
 
+func (cs *ChainStore) GetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error) {
+	fts, err := cs.LoadTipSet(from)
+	if err != nil {
+		return nil, xerrors.Errorf("loading from tipset %s: %w", from, err)
+	}
+	tts, err := cs.LoadTipSet(to)
+	if err != nil {
+		return nil, xerrors.Errorf("loading to tipset %s: %w", to, err)
+	}
+	revert, apply, err := cs.ReorgOps(fts, tts)
+	if err != nil {
+		return nil, xerrors.Errorf("error getting tipset branches: %w", err)
+	}
+
+	path := make([]*HeadChange, len(revert)+len(apply))
+	for i, r := range revert {
+		path[i] = &HeadChange{Type: HCRevert, Val: r}
+	}
+	for j, i := 0, len(apply)-1; i >= 0; j, i = j+1, i-1 {
+		path[j+len(revert)] = &HeadChange{Type: HCApply, Val: apply[i]}
+	}
+	return path, nil
+}
+
 func (cs *ChainStore) MessagesForBlock(b *types.BlockHeader) ([]*types.Message, []*types.SignedMessage, error) {
 	blscids, secpkcids, err := cs.readMsgMetaCids(b.Messages)
 	if err != nil {
@@ -793,6 +820,10 @@ func (cs *ChainStore) Blockstore() bstore.Blockstore {
 	return cs.bs
 }
 
+func (cs *ChainStore) VMSys() *types.VMSyscalls {
+	return cs.vmcalls
+}
+
 func (cs *ChainStore) TryFillTipSet(ts *types.TipSet) (*FullTipSet, error) {
 	var out []*types.FullBlock
 

chain/store/store_test.go

@@ -55,7 +55,7 @@ func BenchmarkGetRandomness(b *testing.B) {
 
 	bs := blockstore.NewBlockstore(bds)
 
-	cs := store.NewChainStore(bs, mds)
+	cs := store.NewChainStore(bs, mds, nil)
 
 	b.ResetTimer()
 

chain/store/weight.go

@@ -60,7 +60,7 @@ func (cs *ChainStore) call(ctx context.Context, msg *types.Message, ts *types.Ti
 
 	r := NewChainRand(cs, ts.Cids(), ts.Height())
 
-	vmi, err := vm.NewVM(bstate, ts.Height(), r, actors.NetworkAddress, cs.bs)
+	vmi, err := vm.NewVM(bstate, ts.Height(), r, actors.NetworkAddress, cs.bs, cs.vmcalls)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to set up vm: %w", err)
 	}

chain/sub/incoming.go

@@ -4,10 +4,11 @@ import (
 	"context"
 	"time"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	connmgr "github.com/libp2p/go-libp2p-core/connmgr"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -33,6 +34,11 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
 			continue
 		}
 
+		if len(blk.BlsMessages)+len(blk.SecpkMessages) > build.BlockMessageLimit {
+			log.Warnf("received block with too many messages over pubsub")
+			continue
+		}
+
 		go func() {
 			log.Infof("New block over pubsub: %s", blk.Cid())
 

chain/sync.go

@@ -18,7 +18,7 @@ import (
 	dstore "github.com/ipfs/go-datastore"
 	hamt "github.com/ipfs/go-hamt-ipld"
 	bstore "github.com/ipfs/go-ipfs-blockstore"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p-core/connmgr"
 	"github.com/libp2p/go-libp2p-core/peer"
 	cbg "github.com/whyrusleeping/cbor-gen"
@@ -189,6 +189,10 @@ func (syncer *Syncer) IncomingBlocks(ctx context.Context) (<-chan *types.BlockHe
 }
 
 func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
+	if msgc := len(fblk.BlsMessages) + len(fblk.SecpkMessages); msgc > build.BlockMessageLimit {
+		return xerrors.Errorf("block %s has too many messages (%d)", fblk.Header.Cid(), msgc)
+	}
+
 	var bcids, scids []cbg.CBORMarshaler
 	for _, m := range fblk.BlsMessages {
 		c := cbg.CborCid(m.Cid())
@@ -295,6 +299,10 @@ func zipTipSetAndMessages(bs amt.Blocks, ts *types.TipSet, allbmsgs []*types.Mes
 			bmsgCids = append(bmsgCids, &c)
 		}
 
+		if msgc := len(bmsgCids) + len(smsgCids); msgc > build.BlockMessageLimit {
+			return nil, fmt.Errorf("block %q has too many messages (%d)", b.Cid(), msgc)
+		}
+
 		mrcid, err := computeMsgMeta(bs, bmsgCids, smsgCids)
 		if err != nil {
 			return nil, err
@@ -533,15 +541,17 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
 
 		snum := types.BigDiv(mpow, types.NewInt(ssize))
 
-		// FORK START
-		if h.Height > build.ForkCCM {
-			if len(h.EPostProof.Candidates) == 0 {
-				return xerrors.Errorf("no candidates")
-			}
+		if len(h.EPostProof.Candidates) == 0 {
+			return xerrors.Errorf("no candidates")
 		}
-		// FORK END
 
+		wins := make(map[uint64]bool)
 		for _, t := range h.EPostProof.Candidates {
+			if wins[t.ChallengeIndex] {
+				return xerrors.Errorf("block had duplicate epost candidates")
+			}
+			wins[t.ChallengeIndex] = true
+
 			if !types.IsTicketWinner(t.Partial, ssize, snum.Uint64(), tpow) {
 				return xerrors.Errorf("miner created a block but was not a winner")
 			}
@@ -662,13 +672,10 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc
 			SectorChallengeIndex: t.ChallengeIndex,
 		})
 	}
-	// FORK START
-	if h.Height > build.ForkCCM {
-		if len(winners) == 0 {
-			return xerrors.Errorf("no candidates")
-		}
+
+	if len(winners) == 0 {
+		return xerrors.Errorf("no candidates")
 	}
-	// FORK END
 
 	sectorInfo, err := stmgr.GetSectorsForElectionPost(ctx, syncer.sm, baseTs, h.Miner)
 	if err != nil {
@@ -683,7 +690,7 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc
 	}
 	hvrf := sha256.Sum256(h.EPostProof.PostRand)
 
-	ok, err := sectorbuilder.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner)
+	ok, err := sectorbuilder.ProofVerifier.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner)
 	if err != nil {
 		return xerrors.Errorf("failed to verify election post: %w", err)
 	}

chain/sync_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p-core/peer"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	"github.com/stretchr/testify/require"

chain/types/bitfield.go

@@ -1,6 +1,7 @@
 package types
 
 import (
+	"errors"
 	"fmt"
 	"io"
 
@@ -9,6 +10,8 @@ import (
 	"golang.org/x/xerrors"
 )
 
+var ErrBitFieldTooMany = errors.New("to many items in RLE")
+
 type BitField struct {
 	rle rlepluslazy.RLE
 
@@ -16,14 +19,23 @@ type BitField struct {
 }
 
 func NewBitField() BitField {
-	rle, err := rlepluslazy.FromBuf([]byte{})
+	bf, err := NewBitFieldFromBytes([]byte{})
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("creating empty rle: %+v", err))
 	}
-	return BitField{
-		rle:  rle,
-		bits: make(map[uint64]struct{}),
+	return bf
+}
+
+func NewBitFieldFromBytes(rle []byte) (BitField, error) {
+	bf := BitField{}
+	rlep, err := rlepluslazy.FromBuf(rle)
+	if err != nil {
+		return BitField{}, xerrors.Errorf("could not decode rle+: %w", err)
 	}
+	bf.rle = rlep
+	bf.bits = make(map[uint64]struct{})
+	return bf, nil
+
 }
 
 func BitFieldFromSet(setBits []uint64) BitField {
@@ -106,7 +118,14 @@ func (bf BitField) Count() (uint64, error) {
 }
 
 // All returns all set bits
-func (bf BitField) All() ([]uint64, error) {
+func (bf BitField) All(max uint64) ([]uint64, error) {
+	c, err := bf.Count()
+	if err != nil {
+		return nil, xerrors.Errorf("count errror: %w", err)
+	}
+	if c > max {
+		return nil, xerrors.Errorf("expected %d, got %d: %w", max, c, ErrBitFieldTooMany)
+	}
 
 	runs, err := bf.sum()
 	if err != nil {
@@ -121,7 +140,14 @@ func (bf BitField) All() ([]uint64, error) {
 	return res, nil
 }
 
-func (bf BitField) AllMap() (map[uint64]bool, error) {
+func (bf BitField) AllMap(max uint64) (map[uint64]bool, error) {
+	c, err := bf.Count()
+	if err != nil {
+		return nil, xerrors.Errorf("count errror: %w", err)
+	}
+	if c > max {
+		return nil, xerrors.Errorf("expected %d, got %d: %w", max, c, ErrBitFieldTooMany)
+	}
 
 	runs, err := bf.sum()
 	if err != nil {

chain/types/blockheader.go

@@ -59,6 +59,8 @@ type BlockHeader struct {
 	Timestamp uint64
 
 	BlockSig *Signature
+
+	ForkSignaling uint64
 }
 
 func (b *BlockHeader) ToStorageBlock() (block.Block, error) {
@@ -118,6 +120,10 @@ func (blk *BlockHeader) CheckBlockSignature(ctx context.Context, worker address.
 	_, span := trace.StartSpan(ctx, "checkBlockSignature")
 	defer span.End()
 
+	if blk.BlockSig == nil {
+		return xerrors.New("block signature not present")
+	}
+
 	sigb, err := blk.SigningBytes()
 	if err != nil {
 		return xerrors.Errorf("failed to get block signing bytes: %w", err)
@@ -216,7 +222,7 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow Big
 	return lhs.Cmp(rhs) < 0
 }
 
-func ElectionPostChallengeCount(sectors uint64, faults int) uint64 {
+func ElectionPostChallengeCount(sectors uint64, faults uint64) uint64 {
 	return sectorbuilder.ElectionPostChallengeCount(sectors, faults)
 }
 

chain/types/cbor_gen.go

@@ -19,7 +19,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{140}); err != nil {
+	if _, err := w.Write([]byte{141}); err != nil {
 		return err
 	}
 
@@ -94,6 +94,11 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
 	if err := t.BlockSig.MarshalCBOR(w); err != nil {
 		return err
 	}
+
+	// t.ForkSignaling (uint64) (uint64)
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ForkSignaling))); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -108,7 +113,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type array")
 	}
 
-	if extra != 12 {
+	if extra != 13 {
 		return fmt.Errorf("cbor input had wrong number of fields")
 	}
 
@@ -272,6 +277,16 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
 		}
 
 	}
+	// t.ForkSignaling (uint64) (uint64)
+
+	maj, extra, err = cbg.CborReadHeader(br)
+	if err != nil {
+		return err
+	}
+	if maj != cbg.MajUnsignedInt {
+		return fmt.Errorf("wrong type for uint64 field")
+	}
+	t.ForkSignaling = uint64(extra)
 	return nil
 }
 

chain/types/signature.go

@@ -116,5 +116,8 @@ func (s *Signature) UnmarshalCBOR(br io.Reader) error {
 }
 
 func (s *Signature) Equals(o *Signature) bool {
+	if s == nil || o == nil {
+		return s == o
+	}
 	return s.Type == o.Type && bytes.Equal(s.Data, o.Data)
 }

chain/types/signature_cgo.go

@@ -9,9 +9,14 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-crypto"
 	"github.com/minio/blake2b-simd"
+	"golang.org/x/xerrors"
 )
 
 func (s *Signature) Verify(addr address.Address, msg []byte) error {
+	if s == nil {
+		return xerrors.Errorf("signature is nil")
+	}
+
 	if addr.Protocol() == address.ID {
 		return fmt.Errorf("must resolve ID addresses before using them to verify a signature")
 	}

chain/types/tipset.go

@@ -8,7 +8,7 @@ import (
 	"sort"
 
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 )

chain/types/vmcontext.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	"context"
+	"github.com/filecoin-project/go-sectorbuilder"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-amt-ipld"
@@ -46,7 +47,15 @@ type VMContext interface {
 }
 
 type VMSyscalls struct {
-	ValidatePoRep func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError)
+	ValidatePoRep      func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError)
+	VerifyFallbackPost func(ctx context.Context,
+		sectorSize uint64,
+		sectorInfo sectorbuilder.SortedPublicSectorInfo,
+		challengeSeed []byte,
+		proof []byte,
+		candidates []sectorbuilder.EPostCandidate,
+		proverID address.Address,
+		faults uint64) (bool, error)
 }
 
 type storageWrapper struct {

chain/validation/applier.go

@@ -2,6 +2,7 @@ package validation
 
 import (
 	"context"
+	"github.com/filecoin-project/go-sectorbuilder"
 
 	vchain "github.com/filecoin-project/chain-validation/pkg/chain"
 	vstate "github.com/filecoin-project/chain-validation/pkg/state"
@@ -32,7 +33,7 @@ func (a *Applier) ApplyMessage(eCtx *vchain.ExecutionContext, state vstate.Wrapp
 	if err != nil {
 		return vchain.MessageReceipt{}, err
 	}
-	lotusVM, err := vm.NewVM(base, eCtx.Epoch, randSrc, minerAddr, st.bs)
+	lotusVM, err := vm.NewVM(base, eCtx.Epoch, randSrc, minerAddr, st.bs, vm.Syscalls(sectorbuilder.ProofVerifier))
 	if err != nil {
 		return vchain.MessageReceipt{}, err
 	}

chain/vm/invoker.go

@@ -35,6 +35,7 @@ func newInvoker() *invoker {
 	inv.register(actors.StoragePowerCodeCid, actors.StoragePowerActor{}, actors.StoragePowerState{})
 	inv.register(actors.StorageMarketCodeCid, actors.StorageMarketActor{}, actors.StorageMarketState{})
 	inv.register(actors.StorageMinerCodeCid, actors.StorageMinerActor{}, actors.StorageMinerActorState{})
+	inv.register(actors.StorageMiner2CodeCid, actors.StorageMinerActor2{}, actors.StorageMinerActorState{})
 	inv.register(actors.MultisigCodeCid, actors.MultiSigActor{}, actors.MultiSigActorState{})
 	inv.register(actors.PaymentChannelCodeCid, actors.PaymentChannelActor{}, actors.PaymentChannelActorState{})
 

chain/vm/syscalls.go

@@ -1,14 +1,29 @@
 package vm
 
 import (
+	"context"
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-sectorbuilder"
 	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/aerrors"
 	"github.com/filecoin-project/lotus/chain/types"
+	"go.opencensus.io/trace"
 )
 
 // Actual type is defined in chain/types/vmcontext.go because the VMContext interface is there
 
-func DefaultSyscalls() *types.VMSyscalls {
+func Syscalls(verifier sectorbuilder.Verifier) *types.VMSyscalls {
 	return &types.VMSyscalls{
-		ValidatePoRep: actors.ValidatePoRep,
+		ValidatePoRep: func(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, actors.ActorError) {
+			_, span := trace.StartSpan(ctx, "ValidatePoRep")
+			defer span.End()
+			ok, err := verifier.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof)
+			if err != nil {
+				return false, aerrors.Absorb(err, 25, "verify seal failed")
+			}
+
+			return ok, nil
+		},
+		VerifyFallbackPost: verifier.VerifyFallbackPost,
 	}
 }

chain/vm/vm.go

@@ -10,7 +10,7 @@ import (
 	cid "github.com/ipfs/go-cid"
 	hamt "github.com/ipfs/go-hamt-ipld"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
@@ -259,7 +259,7 @@ func (bs *gasChargingBlocks) GetBlock(ctx context.Context, c cid.Cid) (block.Blo
 	}
 	blk, err := bs.under.GetBlock(ctx, c)
 	if err != nil {
-		return nil, err
+		return nil, aerrors.Escalate(err, "failed to get block from blockstore")
 	}
 	if err := bs.chargeGas(uint64(len(blk.RawData())) * gasGetPerByte); err != nil {
 		return nil, err
@@ -272,7 +272,10 @@ func (bs *gasChargingBlocks) AddBlock(blk block.Block) error {
 	if err := bs.chargeGas(gasPutObj + uint64(len(blk.RawData()))*gasPutPerByte); err != nil {
 		return err
 	}
-	return bs.under.AddBlock(blk)
+	if err := bs.under.AddBlock(blk); err != nil {
+		return aerrors.Escalate(err, "failed to write data to disk")
+	}
+	return nil
 }
 
 func (vm *VM) makeVMContext(ctx context.Context, sroot cid.Cid, msg *types.Message, origin address.Address, usedGas types.BigInt) *VMContext {
@@ -309,7 +312,7 @@ type VM struct {
 	Syscalls *types.VMSyscalls
 }
 
-func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore) (*VM, error) {
+func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore, syscalls *types.VMSyscalls) (*VM, error) {
 	buf := bufbstore.NewBufferedBstore(cbs)
 	cst := hamt.CSTFromBstore(buf)
 	state, err := state.LoadStateTree(cst, base)
@@ -325,8 +328,8 @@ func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs block
 		blockHeight: height,
 		blockMiner:  maddr,
 		inv:         newInvoker(),
-		rand:        r,
-		Syscalls:    DefaultSyscalls(),
+		rand:        r, // TODO: Probably should be a syscall
+		Syscalls:    syscalls,
 	}, nil
 }
 
@@ -464,7 +467,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, msg *types.Message) (*ApplyRet,
 		return nil, xerrors.Errorf("[from=%s,to=%s,n=%d,m=%d,h=%d] fatal error: %w", msg.From, msg.To, msg.Nonce, msg.Method, vm.blockHeight, actorErr)
 	}
 	if actorErr != nil {
-		log.Warnf("[from=%s,to=%s,n=%d,m=%d,h=%d] Send actor error: %+v", msg.From, msg.To, msg.Nonce, msg.Method, vm.blockHeight, actorErr)
+		log.Warnw("Send actor error", "from", msg.From, "to", msg.To, "nonce", msg.Nonce, "method", msg.Method, "height", vm.blockHeight, "error", fmt.Sprintf("%+v", actorErr))
 	}
 
 	var errcode uint8

chain/wallet/wallet.go

@@ -9,7 +9,7 @@ import (
 
 	bls "github.com/filecoin-project/filecoin-ffi"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/minio/blake2b-simd"
 	"golang.org/x/xerrors"
 

cli/cmd.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"syscall"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/mitchellh/go-homedir"
 	"github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr-net"

cli/params.go

@@ -5,6 +5,8 @@ import (
 	paramfetch "github.com/filecoin-project/go-paramfetch"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
+
+	"github.com/filecoin-project/lotus/build"
 )
 
 var fetchParamCmd = &cli.Command{
@@ -22,7 +24,7 @@ var fetchParamCmd = &cli.Command{
 			return err
 		}
 		sectorSize := uint64(sectorSizeInt)
-		err = paramfetch.GetParams(sectorSize)
+		err = paramfetch.GetParams(build.ParametersJson, sectorSize)
 		if err != nil {
 			return xerrors.Errorf("fetching proof parameters: %w", err)
 		}

cli/state.go

@@ -9,6 +9,7 @@ import (
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/miner"
 	"golang.org/x/xerrors"
 
 	"github.com/ipfs/go-cid"
@@ -37,6 +38,7 @@ var stateCmd = &cli.Command{
 		stateSectorSizeCmd,
 		stateReadStateCmd,
 		stateListMessagesCmd,
+		stateComputeStateCmd,
 	},
 }
 
@@ -580,3 +582,69 @@ var stateListMessagesCmd = &cli.Command{
 		return nil
 	},
 }
+
+var stateComputeStateCmd = &cli.Command{
+	Name:  "compute-state",
+	Usage: "Perform state computations",
+	Flags: []cli.Flag{
+		&cli.Uint64Flag{
+			Name:  "height",
+			Usage: "set the height to compute state at",
+		},
+		&cli.BoolFlag{
+			Name:  "apply-mpool-messages",
+			Usage: "apply messages from the mempool to the computed state",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		ts, err := loadTipSet(ctx, cctx, api)
+		if err != nil {
+			return err
+		}
+
+		h := cctx.Uint64("height")
+		if h == 0 {
+			if ts == nil {
+				head, err := api.ChainHead(ctx)
+				if err != nil {
+					return err
+				}
+				ts = head
+			}
+			h = ts.Height()
+		}
+
+		var msgs []*types.Message
+		if cctx.Bool("apply-mpool-messages") {
+			pmsgs, err := api.MpoolPending(ctx, ts)
+			if err != nil {
+				return err
+			}
+
+			pmsgs, err = miner.SelectMessages(ctx, api.StateGetActor, ts, pmsgs)
+			if err != nil {
+				return err
+			}
+
+			for _, sm := range pmsgs {
+				msgs = append(msgs, &sm.Message)
+			}
+		}
+
+		nstate, err := api.StateCompute(ctx, h, msgs, ts)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("computed state cid: ", nstate)
+		return nil
+	},
+}

cli/wallet.go

@@ -41,7 +41,7 @@ var walletNew = &cli.Command{
 
 		t := cctx.Args().First()
 		if t == "" {
-			t = "bls"
+			t = "secp256k1"
 		}
 
 		nk, err := api.WalletNew(ctx, t)

cmd/lotus-bench/main.go

@@ -17,7 +17,7 @@ import (
 	ffi "github.com/filecoin-project/filecoin-ffi"
 	paramfetch "github.com/filecoin-project/go-paramfetch"
 	"github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/mitchellh/go-homedir"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
@@ -155,7 +155,7 @@ func main() {
 				}
 			}
 
-			if err := paramfetch.GetParams(sectorSize); err != nil {
+			if err := paramfetch.GetParams(build.ParametersJson, sectorSize); err != nil {
 				return xerrors.Errorf("getting params: %w", err)
 			}
 			sb, err := sectorbuilder.New(cfg, mds)
@@ -188,7 +188,7 @@ func main() {
 
 				log.Info("Running replication...")
 				pieces := []sectorbuilder.PublicPieceInfo{pi}
-				pco, err := sb.SealPreCommit(i, ticket, pieces)
+				pco, err := sb.SealPreCommit(context.TODO(), i, ticket, pieces)
 				if err != nil {
 					return xerrors.Errorf("commit: %w", err)
 				}
@@ -206,14 +206,14 @@ func main() {
 				}
 
 				log.Info("Generating PoRep for sector")
-				proof, err := sb.SealCommit(i, ticket, seed, pieces, pco)
+				proof, err := sb.SealCommit(context.TODO(), i, ticket, seed, pieces, pco)
 				if err != nil {
 					return err
 				}
 
 				sealcommit := time.Now()
 				commD := pi.CommP
-				ok, err := sectorbuilder.VerifySeal(sectorSize, pco.CommR[:], commD[:], maddr, ticket.TicketBytes[:], seed.TicketBytes[:], i, proof)
+				ok, err := sectorbuilder.ProofVerifier.VerifySeal(sectorSize, pco.CommR[:], commD[:], maddr, ticket.TicketBytes[:], seed.TicketBytes[:], i, proof)
 				if err != nil {
 					return err
 				}
@@ -303,11 +303,11 @@ func main() {
 
 			epost2 := time.Now()
 
-			if !bytes.Equal(proof1, proof2) {
-				log.Warn("separate epost calls returned different proof values (this might be bad)")
+			if bytes.Equal(proof1, proof2) {
+				log.Warn("separate epost calls returned the same proof values (this might be bad)")
 			}
 
-			ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof1, candidates[:1], maddr)
+			ok, err := sectorbuilder.ProofVerifier.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof1, candidates[:1], maddr)
 			if err != nil {
 				return err
 			}
@@ -317,7 +317,7 @@ func main() {
 
 			verifypost1 := time.Now()
 
-			ok, err = sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof2, candidates[:1], maddr)
+			ok, err = sectorbuilder.ProofVerifier.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof2, candidates[:1], maddr)
 			if err != nil {
 				return err
 			}
@@ -351,7 +351,9 @@ func main() {
 					fmt.Printf("seal: preCommit: %s (%s)\n", bo.SealingResults[0].PreCommit, bps(bo.SectorSize, bo.SealingResults[0].PreCommit))
 					fmt.Printf("seal: commit: %s (%s)\n", bo.SealingResults[0].Commit, bps(bo.SectorSize, bo.SealingResults[0].Commit))
 					fmt.Printf("seal: verify: %s\n", bo.SealingResults[0].Verify)
-					fmt.Printf("unseal: %s  (%s)\n", bo.SealingResults[0].Unseal, bps(bo.SectorSize, bo.SealingResults[0].Unseal))
+					if !c.Bool("skip-unseal") {
+						fmt.Printf("unseal: %s  (%s)\n", bo.SealingResults[0].Unseal, bps(bo.SectorSize, bo.SealingResults[0].Unseal))
+					}
 				}
 				fmt.Printf("generate candidates: %s (%s)\n", bo.PostGenerateCandidates, bps(bo.SectorSize*uint64(len(bo.SealingResults)), bo.PostGenerateCandidates))
 				fmt.Printf("compute epost proof (cold): %s\n", bo.PostEProofCold)

cmd/lotus-chainwatch/blockssub.go

@@ -21,7 +21,7 @@ func subBlocks(ctx context.Context, api aapi.FullNode, st *storage) {
 			bh.Cid(): bh,
 		}, false)
 		if err != nil {
-			log.Error(err)
+			//log.Errorf("%+v", err)
 		}
 	}
 }

cmd/lotus-chainwatch/dot.go

@@ -26,7 +26,7 @@ var dotCmd = &cli.Command{
 		res, err := st.db.Query(`select block, parent, b.miner, b.height, p.height from block_parents
     inner join blocks b on block_parents.block = b.cid
     inner join blocks p on block_parents.parent = p.cid
-where b.height > ? and b.height < ?`, minH, maxH)
+where b.height > $1 and b.height < $2`, minH, maxH)
 
 		if err != nil {
 			return err
@@ -34,6 +34,8 @@ where b.height > ? and b.height < ?`, minH, maxH)
 
 		fmt.Println("digraph D {")
 
+		hl := st.hasList()
+
 		for res.Next() {
 			var block, parent, miner string
 			var height, ph uint64
@@ -46,7 +48,7 @@ where b.height > ? and b.height < ?`, minH, maxH)
 				return err
 			}
 
-			has := st.hasBlock(bc)
+			_, has := hl[bc]
 
 			col := crc32.Checksum([]byte(miner), crc32.MakeTable(crc32.Castagnoli))&0xc0c0c0c0 + 0x30303030
 

cmd/lotus-chainwatch/main.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 	"os"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
 
@@ -38,7 +38,7 @@ func main() {
 			&cli.StringFlag{
 				Name:    "db",
 				EnvVars: []string{"LOTUS_DB"},
-				Value:   "./chainwatch.db",
+				Value:   "",
 			},
 		},
 
@@ -82,8 +82,6 @@ var runCmd = &cli.Command{
 		defer st.close()
 
 		runSyncer(ctx, api, st)
-		go subMpool(ctx, api, st)
-		go subBlocks(ctx, api, st)
 
 		h, err := newHandler(api, st)
 		if err != nil {

cmd/lotus-chainwatch/mpool.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"time"
 
 	"github.com/ipfs/go-cid"
 
@@ -15,24 +16,45 @@ func subMpool(ctx context.Context, api aapi.FullNode, st *storage) {
 		return
 	}
 
-	for change := range sub {
-		if change.Type != aapi.MpoolAdd {
-			continue
+	for {
+		var updates []aapi.MpoolUpdate
+
+		select {
+		case update := <-sub:
+			updates = append(updates, update)
+		case <-ctx.Done():
+			return
 		}
 
-		log.Info("mpool message")
+	loop:
+		for {
+			time.Sleep(10 * time.Millisecond)
+			select {
+			case update := <-sub:
+				updates = append(updates, update)
+			default:
+				break loop
+			}
+		}
 
-		err := st.storeMessages(map[cid.Cid]*types.Message{
-			change.Message.Message.Cid(): &change.Message.Message,
-		})
+		msgs := map[cid.Cid]*types.Message{}
+		for _, v := range updates {
+			if v.Type != aapi.MpoolAdd {
+				continue
+			}
+
+			msgs[v.Message.Message.Cid()] = &v.Message.Message
+		}
+
+		log.Infof("Processing %d mpool updates", len(msgs))
+
+		err := st.storeMessages(msgs)
 		if err != nil {
 			log.Error(err)
-			continue
 		}
 
-		if err := st.storeMpoolInclusion(change.Message.Message.Cid()); err != nil {
+		if err := st.storeMpoolInclusions(updates); err != nil {
 			log.Error(err)
-			continue
 		}
 	}
 }

cmd/lotus-chainwatch/site/block.html

@@ -0,0 +1,61 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Lotus ChainWatch</title>
+    <link rel="stylesheet" type="text/css" href="main.css">
+</head>
+<body>
+{{$cid := param "cid"}}
+
+<div class="Index">
+    <div class="Index-header">
+        <div>
+            <span>Lotus ChainWatch - Wallets</span>
+        </div>
+    </div>
+    <div class="Index-nodes">
+        <div class="Index-node">
+            <div>Miner: {{index (strings "blocks" "miner" "cid=$1" $cid) 0}}</div>
+            <div>Parents:</div>
+            <div>
+                {{range strings "block_parents" "parent" "block=$1" $cid}}
+                    {{$parent := .}}
+                    <a href="block.html?cid={{$parent}}">{{. | substr 54 62}}</a>
+                {{end}}
+            </div>
+            <div>Messages:</div>
+            <table>
+                {{range strings "block_messages" "message" "block=$1" $cid}}
+                    {{$msg := .}}
+                    <tr>
+                        <td><a href="message.html?cid={{$msg}}">{{$msg | substr 54 62}}</a></td>
+                        <td>
+                            {{$from := qstr "select \"from\" from messages where cid=$1" $msg}}
+                            {{$nonce := qstr "select nonce from messages where cid=$1" $msg}}
+                            <a href="key.html?w={{$from}}">{{$from}}</a> (N:{{$nonce}})
+                        </td>
+                        <td>-&gt;</td>
+                        <td>
+                            {{$to := qstr "select \"to\" from messages where cid=$1" $msg}}
+                            <a href="key.html?w={{$to}}">{{$to}}</a>
+                        </td>
+                        <td>
+                            Method:<b>{{qstr "select method from messages where cid=$1" $msg}}</b>
+                        </td>
+                        {{$rec := qstrs `select r.exit, r.gas_used from messages
+                            inner join block_messages bm on messages.cid = bm.message
+                            inner join blocks b on bm.block = b.cid
+                            inner join block_parents bp on b.cid = bp.parent
+                            inner join blocks chd on bp.block = chd.cid
+                            inner join receipts r on messages.cid = r.msg and chd.parentStateRoot = r.state
+                            where messages.cid=$1 and b.cid=$2` 2 $msg $cid}}
+                        <td>exit:<b>{{index $rec 0}}</b></td>
+                        <td>gasUsed:<b>{{index $rec 1}}</b></td>
+                    </tr>
+                {{end}}
+            </table>
+        </div>
+    </div>
+</div>
+</body>
+</html>

cmd/lotus-chainwatch/site/blocks.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Lotus ChainWatch</title>
+    <link rel="stylesheet" type="text/css" href="main.css">
+</head>
+<body>
+{{$start := param "start" | parseInt}}
+
+<div class="Index">
+    <div class="Index-header">
+        <div>
+            <span>Lotus ChainWatch - Wallets</span>
+        </div>
+    </div>
+    <div class="Index-nodes">
+        <div class="Index-node">
+            <table>
+                {{range pageDown $start 50}}
+                    <tr>
+                        <td>
+                            {{$h := .}}
+                            {{$h}};
+                        </td>
+                        <td>
+                            <b>{{qstr `select count(distinct block_messages.message) from block_messages
+                                    inner join blocks b on block_messages.block = b.cid
+                                    where b.height = $1` $h}}</b> Msgs
+                        </td>
+                        <td>
+                            {{range strings "blocks" "cid" "height = $1" $h}}
+                                <a href="block.html?cid={{.}}">{{. | substr 54 62}}</a>
+                            {{end}}
+                        </td>
+                    </tr>
+                {{end}}
+            </table>
+            <a href="blocks.html?start={{sub $start 50}}">Next 50</a>
+        </div>
+    </div>
+</div>
+</body>
+</html>

cmd/lotus-chainwatch/site/index.html

@@ -26,6 +26,11 @@
             {{count "id_address_map" "id != address"}} <a href="keys.html">Keys</a>;
             E% FIL in wallets; F% FIL in miners; M% in market; %G Other actors; %H FIL it treasury
         </div>
+        <div class="Index-node">
+            {{$maxH := queryNum "select max(height) from blocks inner join blocks_synced bs on blocks.cid = bs.cid"}}
+
+            {{count "blocks"}} <a href="blocks.html?start={{$maxH}}">Blocks</a>; Current Height: {{$maxH}};
+        </div>
     </div>
 </div>
 </body>

cmd/lotus-chainwatch/site/key.html

@@ -15,18 +15,18 @@
     </div>
     <div class="Index-nodes">
         <div class="Index-node">
-            Balance: {{queryNum "select balance from actors inner join main.id_address_map m on m.address = ? where actors.id = m.id order by nonce desc limit 1" $wallet }}
+            Balance: {{queryNum "select balance from actors inner join id_address_map m on m.address = $1 where actors.id = m.id order by nonce desc limit 1" $wallet }}
         </div>
         <div class="Index-node">
             Messages:
             <table>
                 <tr><td>Dir</td><td>Peer</td><td>Nonce</td><td>Value</td><td>Block</td><td>Mpool Wait</td></tr>
-                {{ range messages "`from` = ? or `to` = ?" $wallet $wallet $wallet}}
+                {{ range messages "\"from\" = $1 or \"to\" = $1" $wallet}}
                     <tr>
                         {{ if eq .From.String $wallet }}
-                            <td>To</td><td>{{.To.String}}</td>
+                            <td>To</td><td><a href="key.html?w={{.To.String}}">{{.To.String}}</a></td>
                         {{else}}
-                            <td>From</td><td>{{.From.String}}</td>
+                            <td>From</td><td><a href="key.html?w={{.From.String}}">{{.From.String}}</a></td>
                         {{end}}
                         <td>{{.Nonce}}</td>
                         <td>{{.Value}}</td>

cmd/lotus-chainwatch/site/keys.html

@@ -14,7 +14,12 @@
     <div class="Index-nodes">
         <div class="Index-node">
            {{range strings "id_address_map" "address" "address != id"}}
-               <div><a href="key.html?w={{.}}">{{.}}</a></div>
+               {{$addr := .}}
+               <div>
+                   <a href="key.html?w={{$addr}}">{{$addr}}</a>
+                   <span><b>{{qstr "select count(distinct cid) from messages where \"from\"=?" $addr}}</b> outmsgs;</span>
+                   <span><b>{{qstr "select count(distinct cid) from messages where \"to\"=?" $addr}}</b> inmsgs</span>
+               </div>
            {{end}}
         </div>
     </div>

cmd/lotus-chainwatch/site/main.css

@@ -6,6 +6,10 @@ body {
     margin: 0;
 }
 
+b {
+    color: #aff;
+}
+
 .Index {
     width: 100vw;
     height: 100vh;

cmd/lotus-chainwatch/storage.go

@@ -2,11 +2,14 @@ package main
 
 import (
 	"database/sql"
+	"fmt"
+	"github.com/filecoin-project/lotus/api"
+	"golang.org/x/xerrors"
 	"sync"
 	"time"
 
 	"github.com/ipfs/go-cid"
-	_ "github.com/mattn/go-sqlite3"
+	_ "github.com/lib/pq"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -19,11 +22,13 @@ type storage struct {
 }
 
 func openStorage(dbSource string) (*storage, error) {
-	db, err := sql.Open("sqlite3", dbSource)
+	db, err := sql.Open("postgres", dbSource)
 	if err != nil {
 		return nil, err
 	}
 
+	db.SetMaxOpenConns(1350)
+
 	st := &storage{db: db}
 
 	return st, st.setup()
@@ -35,33 +40,77 @@ func (st *storage) setup() error {
 		return err
 	}
 	_, err = tx.Exec(`
+create table if not exists blocks_synced
+(
+	cid text not null
+		constraint blocks_synced_pk
+			primary key,
+	add_ts int not null
+);
+
+create unique index if not exists blocks_synced_cid_uindex
+	on blocks_synced (cid);
+
+create table if not exists block_parents
+(
+	block text not null,
+	parent text not null
+);
+
+create unique index if not exists block_parents_block_parent_uindex
+	on block_parents (block, parent);
+
+create table if not exists blocks
+(
+	cid text not null
+		constraint blocks_pk
+			primary key,
+	parentWeight numeric not null,
+	parentStateRoot text not null,
+	height bigint not null,
+	miner text not null,
+	timestamp bigint not null,
+	vrfproof bytea,
+	tickets bigint not null,
+	eprof bytea,
+	prand bytea,
+	ep0partial bytea,
+	ep0sector bigint not null,
+	ep0challangei bigint not null
+);
+
+create unique index if not exists block_cid_uindex
+	on blocks (cid);
+
+create table if not exists id_address_map
+(
+	id text not null,
+	address text not null,
+	constraint id_address_map_pk
+		primary key (id, address)
+);
+
+create unique index if not exists id_address_map_id_uindex
+	on id_address_map (id);
+
+create unique index if not exists id_address_map_address_uindex
+	on id_address_map (address);
+
 create table if not exists actors
   (
-	id text not null,
+	id text not null
+		constraint id_address_map_actors_id_fk
+			references id_address_map (id),
 	code text not null,
 	head text not null,
 	nonce int not null,
 	balance text not null,
 	stateroot text
-		constraint actors_blocks_stateroot_fk
-			references blocks (parentStateRoot),
-	constraint actors_pk
-		primary key (id, nonce, balance, stateroot)
   );
   
 create index if not exists actors_id_index
 	on actors (id);
 
-create table if not exists id_address_map
-(
-	id text not null
-		constraint id_address_map_actors_id_fk
-			references actors (id),
-	address text not null,
-	constraint id_address_map_pk
-		primary key (id, address)
-);
-
 create index if not exists id_address_map_address_index
 	on id_address_map (address);
 
@@ -73,84 +122,23 @@ create table if not exists messages
 	cid text not null
 		constraint messages_pk
 			primary key,
-	"from" text not null
-		constraint messages_id_address_map_from_fk
-			references id_address_map (address),
-	"to" text not null
-		constraint messages_id_address_map_to_fk
-			references id_address_map (address),
+	"from" text not null,
+	"to" text not null,
 	nonce int not null,
 	value text not null,
 	gasprice int not null,
 	gaslimit int not null,
 	method int,
-	params blob
+	params bytea
 );
 
 create unique index if not exists messages_cid_uindex
 	on messages (cid);
-
-create table if not exists blocks
-(
-	cid text not null
-		constraint blocks_pk
-			primary key,
-	parentWeight numeric not null,
-	parentStateRoot text not null,
-	height int not null,
-	miner text not null
-		constraint blocks_id_address_map_miner_fk
-			references id_address_map (address),
-	timestamp int not null,
-	vrfproof blob,
-	tickets int not null,
-	eprof blob,
-	prand blob,
-	ep0partial blob,
-	ep0sector int not null,
-	ep0challangei int not null
-);
-
-create unique index if not exists block_cid_uindex
-	on blocks (cid);
-
-create table if not exists blocks_synced
-(
-	cid text not null
-		constraint blocks_synced_pk
-			primary key
-		constraint blocks_synced_blocks_cid_fk
-			references blocks,
-	add_ts int not null
-);
-
-create unique index if not exists blocks_synced_cid_uindex
-	on blocks_synced (cid);
-
-create table if not exists block_parents
-(
-	block text not null
-		constraint block_parents_blocks_cid_fk
-			references blocks,
-	parent text not null
-		constraint block_parents_blocks_cid_fk_2
-			references blocks
-);
-
-create unique index if not exists block_parents_block_parent_uindex
-	on block_parents (block, parent);
-
-create unique index if not exists blocks_cid_uindex
-	on blocks (cid);
 	
 create table if not exists block_messages
 (
-	block text not null
-		constraint block_messages_blk_fk
-			references blocks (cid),
-	message text not null
-		constraint block_messages_msg_fk
-			references messages,
+	block text not null,
+	message text not null,
 	constraint block_messages_pk
 		primary key (block, message)
 );
@@ -170,16 +158,12 @@ create unique index if not exists mpool_messages_msg_uindex
 
 create table if not exists receipts
 (
-	msg text not null
-		constraint receipts_messages_cid_fk
-			references messages,
-	state text not null
-		constraint receipts_blocks_parentStateRoot_fk
-			references blocks (parentStateRoot),
+	msg text not null,
+	state text not null,
 	idx int not null,
 	exit int not null,
 	gas_used int not null,
-	return blob,
+	return bytea,
 	constraint receipts_pk
 		primary key (msg, state)
 );
@@ -187,34 +171,23 @@ create table if not exists receipts
 create index if not exists receipts_msg_state_index
 	on receipts (msg, state);
 
-
 create table if not exists miner_heads
 (
-	head text not null
-		constraint miner_heads_actors_head_fk
-			references actors (head),
-	addr text not null
-		constraint miner_heads_actors_id_fk
-			references actors (id),
-	stateroot text not null
-		constraint miner_heads_blocks_stateroot_fk
-			references blocks (parentStateRoot),
+	head text not null,
+	addr text not null,
+	stateroot text not null,
 	sectorset text not null,
-	setsize int not null,
+	setsize decimal not null,
 	provingset text not null,
-	provingsize int not null,
+	provingsize decimal not null,
 	owner text not null,
 	worker text not null,
 	peerid text not null,
-	sectorsize int not null,
-	power text not null,
-	active int,
-	ppe int not null,
-	slashed_at int not null,
-	constraint miner_heads_id_address_map_owner_fk
-		foreign key (owner) references id_address_map (address),
-	constraint miner_heads_id_address_map_worker_fk
-		foreign key (worker) references id_address_map (address),
+	sectorsize bigint not null,
+	power bigint not null,
+	active bool,
+	ppe bigint not null,
+	slashed_at bigint not null,
 	constraint miner_heads_pk
 		primary key (head, addr)
 );
@@ -226,14 +199,31 @@ create table if not exists miner_heads
 	return tx.Commit()
 }
 
-func (st *storage) hasBlock(bh cid.Cid) bool {
-	var exitsts bool
-	err := st.db.QueryRow(`select exists (select 1 FROM blocks_synced where cid=?)`, bh.String()).Scan(&exitsts)
+func (st *storage) hasList() map[cid.Cid]struct{} {
+	rws, err := st.db.Query(`select cid FROM blocks_synced`)
 	if err != nil {
 		log.Error(err)
-		return false
+		return map[cid.Cid]struct{}{}
 	}
-	return exitsts
+	out := map[cid.Cid]struct{}{}
+
+	for rws.Next() {
+		var c string
+		if err := rws.Scan(&c); err != nil {
+			log.Error(err)
+			continue
+		}
+
+		ci, err := cid.Parse(c)
+		if err != nil {
+			log.Error(err)
+			continue
+		}
+
+		out[ci] = struct{}{}
+	}
+
+	return out
 }
 
 func (st *storage) storeActors(actors map[address.Address]map[types.Actor]cid.Cid) error {
@@ -241,12 +231,20 @@ func (st *storage) storeActors(actors map[address.Address]map[types.Actor]cid.Ci
 	if err != nil {
 		return err
 	}
+	if _, err := tx.Exec(`
 
-	stmt, err := tx.Prepare(`insert into actors (id, code, head, nonce, balance, stateroot) values (?, ?, ?, ?, ?, ?) on conflict do nothing`)
+create temp table a (like actors excluding constraints) on commit drop;
+
+
+`); err != nil {
+		return xerrors.Errorf("prep temp: %w", err)
+	}
+
+	stmt, err := tx.Prepare(`copy a (id, code, head, nonce, balance, stateroot) from stdin `)
 	if err != nil {
 		return err
 	}
-	defer stmt.Close()
+
 	for addr, acts := range actors {
 		for act, st := range acts {
 			if _, err := stmt.Exec(addr.String(), act.Code.String(), act.Head.String(), act.Nonce, act.Balance.String(), st.String()); err != nil {
@@ -255,6 +253,14 @@ func (st *storage) storeActors(actors map[address.Address]map[types.Actor]cid.Ci
 		}
 	}
 
+	if err := stmt.Close(); err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`insert into actors select * from a on conflict do nothing `); err != nil {
+		return xerrors.Errorf("actor put: %w", err)
+	}
+
 	return tx.Commit()
 }
 
@@ -264,20 +270,28 @@ func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
 		return err
 	}
 
-	stmt, err := tx.Prepare(`insert into miner_heads (head, addr, stateroot, sectorset, setsize, provingset, provingsize, owner, worker, peerid, sectorsize, power, active, ppe, slashed_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do nothing`)
+	if _, err := tx.Exec(`
+
+create temp table mh (like miner_heads excluding constraints) on commit drop;
+
+
+`); err != nil {
+		return xerrors.Errorf("prep temp: %w", err)
+	}
+
+	stmt, err := tx.Prepare(`copy mh (head, addr, stateroot, sectorset, setsize, provingset, provingsize, owner, worker, peerid, sectorsize, power, active, ppe, slashed_at) from STDIN`)
 	if err != nil {
 		return err
 	}
-	defer stmt.Close()
 	for k, i := range miners {
 		if _, err := stmt.Exec(
 			k.act.Head.String(),
 			k.addr.String(),
 			k.stateroot.String(),
 			i.state.Sectors.String(),
-			i.ssize,
+			fmt.Sprint(i.ssize),
 			i.state.ProvingSet.String(),
-			i.psize,
+			fmt.Sprint(i.psize),
 			i.info.Owner.String(),
 			i.info.Worker.String(),
 			i.info.PeerID.String(),
@@ -290,6 +304,13 @@ func (st *storage) storeMiners(miners map[minerKey]*minerInfo) error {
 			return err
 		}
 	}
+	if err := stmt.Close(); err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`insert into miner_heads select * from mh on conflict do nothing `); err != nil {
+		return xerrors.Errorf("actor put: %w", err)
+	}
 
 	return tx.Commit()
 }
@@ -300,21 +321,77 @@ func (st *storage) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool) e
 
 	tx, err := st.db.Begin()
 	if err != nil {
-		return err
+		return xerrors.Errorf("begin: %w", err)
 	}
 
-	stmt, err := tx.Prepare(`insert into blocks (cid, parentWeight, parentStateRoot, height, miner, "timestamp", vrfproof, tickets, eprof, prand, ep0partial, ep0sector, ep0challangei) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do nothing`)
+	if _, err := tx.Exec(`
+
+create temp table tbp (like block_parents excluding constraints) on commit drop;
+create temp table bs (like blocks_synced excluding constraints) on commit drop;
+create temp table b (like blocks excluding constraints) on commit drop;
+
+
+`); err != nil {
+		return xerrors.Errorf("prep temp: %w", err)
+	}
+
+	stmt, err := tx.Prepare(`copy tbp (block, parent) from STDIN`)
 	if err != nil {
 		return err
 	}
-	defer stmt.Close()
+
+	for _, bh := range bhs {
+		for _, parent := range bh.Parents {
+			if _, err := stmt.Exec(bh.Cid().String(), parent.String()); err != nil {
+				log.Error(err)
+			}
+		}
+	}
+
+	if err := stmt.Close(); err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`insert into block_parents select * from tbp on conflict do nothing `); err != nil {
+		return xerrors.Errorf("parent put: %w", err)
+	}
+
+	if sync {
+		now := time.Now().Unix()
+
+		stmt, err := tx.Prepare(`copy bs (cid, add_ts) from stdin `)
+		if err != nil {
+			return err
+		}
+
+		for _, bh := range bhs {
+			if _, err := stmt.Exec(bh.Cid().String(), now); err != nil {
+				log.Error(err)
+			}
+		}
+
+		if err := stmt.Close(); err != nil {
+			return err
+		}
+
+		if _, err := tx.Exec(`insert into blocks_synced select * from bs on conflict do nothing `); err != nil {
+			return xerrors.Errorf("syncd put: %w", err)
+		}
+	}
+
+	stmt2, err := tx.Prepare(`copy b (cid, parentWeight, parentStateRoot, height, miner, "timestamp", vrfproof, tickets, eprof, prand, ep0partial, ep0sector, ep0challangei) from stdin `)
+	if err != nil {
+		return err
+	}
+
 	for _, bh := range bhs {
 		l := len(bh.EPostProof.Candidates)
 		if len(bh.EPostProof.Candidates) == 0 {
 			bh.EPostProof.Candidates = append(bh.EPostProof.Candidates, types.EPostTicket{})
 		}
 
-		if _, err := stmt.Exec(bh.Cid().String(),
+		if _, err := stmt2.Exec(
+			bh.Cid().String(),
 			bh.ParentWeight.String(),
 			bh.ParentStateRoot.String(),
 			bh.Height,
@@ -326,41 +403,24 @@ func (st *storage) storeHeaders(bhs map[cid.Cid]*types.BlockHeader, sync bool) e
 			bh.EPostProof.PostRand,
 			bh.EPostProof.Candidates[0].Partial,
 			bh.EPostProof.Candidates[0].SectorID,
-			bh.EPostProof.Candidates[0].ChallengeIndex,
-		); err != nil {
-			return err
+			bh.EPostProof.Candidates[0].ChallengeIndex); err != nil {
+			log.Error(err)
 		}
 	}
 
-	stmt2, err := tx.Prepare(`insert into block_parents (block, parent) values (?, ?) on conflict do nothing`)
+	if err := stmt2.Close(); err != nil {
+		return xerrors.Errorf("s2 close: %w", err)
+	}
+
+	if _, err := tx.Exec(`insert into blocks select * from b on conflict do nothing `); err != nil {
+		return xerrors.Errorf("blk put: %w", err)
+	}
+
+	err = tx.Commit()
 	if err != nil {
-		return err
+		return xerrors.Errorf("commit: %w", err)
 	}
-	defer stmt2.Close()
-	for _, bh := range bhs {
-		for _, parent := range bh.Parents {
-			if _, err := stmt2.Exec(bh.Cid().String(), parent.String()); err != nil {
-				return err
-			}
-		}
-	}
-
-	if sync {
-		stmt, err := tx.Prepare(`insert into blocks_synced (cid, add_ts) values (?, ?) on conflict do nothing`)
-		if err != nil {
-			return err
-		}
-		defer stmt.Close()
-		now := time.Now().Unix()
-
-		for _, bh := range bhs {
-			if _, err := stmt.Exec(bh.Cid().String(), now); err != nil {
-				return err
-			}
-		}
-	}
-
-	return tx.Commit()
+	return nil
 }
 
 func (st *storage) storeMessages(msgs map[cid.Cid]*types.Message) error {
@@ -369,11 +429,19 @@ func (st *storage) storeMessages(msgs map[cid.Cid]*types.Message) error {
 		return err
 	}
 
-	stmt, err := tx.Prepare(`insert into messages (cid, "from", "to", nonce, "value", gasprice, gaslimit, method, params) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do nothing`)
+	if _, err := tx.Exec(`
+
+create temp table msgs (like messages excluding constraints) on commit drop;
+
+
+`); err != nil {
+		return xerrors.Errorf("prep temp: %w", err)
+	}
+
+	stmt, err := tx.Prepare(`copy msgs (cid, "from", "to", nonce, "value", gasprice, gaslimit, method, params) from stdin `)
 	if err != nil {
 		return err
 	}
-	defer stmt.Close()
 
 	for c, m := range msgs {
 		if _, err := stmt.Exec(
@@ -390,6 +458,13 @@ func (st *storage) storeMessages(msgs map[cid.Cid]*types.Message) error {
 			return err
 		}
 	}
+	if err := stmt.Close(); err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`insert into messages select * from msgs on conflict do nothing `); err != nil {
+		return xerrors.Errorf("actor put: %w", err)
+	}
 
 	return tx.Commit()
 }
@@ -400,11 +475,19 @@ func (st *storage) storeReceipts(recs map[mrec]*types.MessageReceipt) error {
 		return err
 	}
 
-	stmt, err := tx.Prepare(`insert into receipts (msg, state, idx, exit, gas_used, return) VALUES (?, ?, ?, ?, ?, ?) on conflict do nothing`)
+	if _, err := tx.Exec(`
+
+create temp table recs (like receipts excluding constraints) on commit drop;
+
+
+`); err != nil {
+		return xerrors.Errorf("prep temp: %w", err)
+	}
+
+	stmt, err := tx.Prepare(`copy recs (msg, state, idx, exit, gas_used, return) from stdin `)
 	if err != nil {
 		return err
 	}
-	defer stmt.Close()
 
 	for c, m := range recs {
 		if _, err := stmt.Exec(
@@ -418,6 +501,13 @@ func (st *storage) storeReceipts(recs map[mrec]*types.MessageReceipt) error {
 			return err
 		}
 	}
+	if err := stmt.Close(); err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`insert into receipts select * from recs on conflict do nothing `); err != nil {
+		return xerrors.Errorf("actor put: %w", err)
+	}
 
 	return tx.Commit()
 }
@@ -428,11 +518,19 @@ func (st *storage) storeAddressMap(addrs map[address.Address]address.Address) er
 		return err
 	}
 
-	stmt, err := tx.Prepare(`insert into id_address_map (id, address) VALUES (?, ?) on conflict do nothing`)
+	if _, err := tx.Exec(`
+
+create temp table iam (like id_address_map excluding constraints) on commit drop;
+
+
+`); err != nil {
+		return xerrors.Errorf("prep temp: %w", err)
+	}
+
+	stmt, err := tx.Prepare(`copy iam (id, address) from STDIN `)
 	if err != nil {
 		return err
 	}
-	defer stmt.Close()
 
 	for a, i := range addrs {
 		if i == address.Undef {
@@ -445,6 +543,13 @@ func (st *storage) storeAddressMap(addrs map[address.Address]address.Address) er
 			return err
 		}
 	}
+	if err := stmt.Close(); err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`insert into id_address_map select * from iam on conflict do nothing `); err != nil {
+		return xerrors.Errorf("actor put: %w", err)
+	}
 
 	return tx.Commit()
 }
@@ -455,11 +560,19 @@ func (st *storage) storeMsgInclusions(incls map[cid.Cid][]cid.Cid) error {
 		return err
 	}
 
-	stmt, err := tx.Prepare(`insert into block_messages (block, message) VALUES (?, ?) on conflict do nothing`)
+	if _, err := tx.Exec(`
+
+create temp table mi (like block_messages excluding constraints) on commit drop;
+
+
+`); err != nil {
+		return xerrors.Errorf("prep temp: %w", err)
+	}
+
+	stmt, err := tx.Prepare(`copy mi (block, message) from STDIN `)
 	if err != nil {
 		return err
 	}
-	defer stmt.Close()
 
 	for b, msgs := range incls {
 		for _, msg := range msgs {
@@ -471,28 +584,58 @@ func (st *storage) storeMsgInclusions(incls map[cid.Cid][]cid.Cid) error {
 			}
 		}
 	}
+	if err := stmt.Close(); err != nil {
+		return err
+	}
+
+	if _, err := tx.Exec(`insert into block_messages select * from mi on conflict do nothing `); err != nil {
+		return xerrors.Errorf("actor put: %w", err)
+	}
 
 	return tx.Commit()
 }
 
-func (st *storage) storeMpoolInclusion(msg cid.Cid) error {
+func (st *storage) storeMpoolInclusions(msgs []api.MpoolUpdate) error {
 	tx, err := st.db.Begin()
 	if err != nil {
 		return err
 	}
 
-	stmt, err := tx.Prepare(`insert into mpool_messages (msg, add_ts) VALUES (?, ?) on conflict do nothing`)
+	if _, err := tx.Exec(`
+
+create temp table mi (like mpool_messages excluding constraints) on commit drop;
+
+
+`); err != nil {
+		return xerrors.Errorf("prep temp: %w", err)
+	}
+
+	stmt, err := tx.Prepare(`copy mi (msg, add_ts) from stdin `)
 	if err != nil {
 		return err
 	}
-	defer stmt.Close()
 
-	if _, err := stmt.Exec(
-		msg.String(),
-		time.Now().Unix(),
-	); err != nil {
+	for _, msg := range msgs {
+		if msg.Type != api.MpoolAdd {
+			continue
+		}
+
+		if _, err := stmt.Exec(
+			msg.Message.Message.Cid().String(),
+			time.Now().Unix(),
+		); err != nil {
+			return err
+		}
+	}
+
+	if err := stmt.Close(); err != nil {
 		return err
 	}
+
+	if _, err := tx.Exec(`insert into mpool_messages select * from mi on conflict do nothing `); err != nil {
+		return xerrors.Errorf("actor put: %w", err)
+	}
+
 	return tx.Commit()
 }
 

cmd/lotus-chainwatch/sync.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"container/list"
 	"context"
+	"math"
 	"sync"
 
 	"github.com/filecoin-project/go-address"
@@ -16,6 +17,8 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 )
 
+const maxBatch = 3000
+
 func runSyncer(ctx context.Context, api api.FullNode, st *storage) {
 	notifs, err := api.ChainNotify(ctx)
 	if err != nil {
@@ -32,6 +35,12 @@ func runSyncer(ctx context.Context, api api.FullNode, st *storage) {
 				case store.HCRevert:
 					log.Warnf("revert todo")
 				}
+
+				if change.Type == store.HCCurrent {
+					go subMpool(ctx, api, st)
+					go subBlocks(ctx, api, st)
+				}
+
 			}
 		}
 	}()
@@ -56,9 +65,13 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 	actors := map[address.Address]map[types.Actor]cid.Cid{}
 	var alk sync.Mutex
 
+	log.Infof("Getting synced block list")
+
+	hazlist := st.hasList()
+
 	log.Infof("Getting headers / actors")
 
-	toSync := map[cid.Cid]*types.BlockHeader{}
+	allToSync := map[cid.Cid]*types.BlockHeader{}
 	toVisit := list.New()
 
 	for _, header := range ts.Blocks() {
@@ -68,15 +81,16 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 	for toVisit.Len() > 0 {
 		bh := toVisit.Remove(toVisit.Back()).(*types.BlockHeader)
 
-		if _, seen := toSync[bh.Cid()]; seen || st.hasBlock(bh.Cid()) {
+		_, has := hazlist[bh.Cid()]
+		if _, seen := allToSync[bh.Cid()]; seen || has {
 			continue
 		}
 
-		toSync[bh.Cid()] = bh
+		allToSync[bh.Cid()] = bh
 		addresses[bh.Miner] = address.Undef
 
-		if len(toSync)%500 == 10 {
-			log.Infof("todo: (%d) %s", len(toSync), bh.Cid())
+		if len(allToSync)%500 == 10 {
+			log.Infof("todo: (%d) %s @%d", len(allToSync), bh.Cid(), bh.Height)
 		}
 
 		if len(bh.Parents) == 0 {
@@ -94,186 +108,218 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
 		}
 	}
 
-	log.Infof("Syncing %d blocks", len(toSync))
+	for len(allToSync) > 0 {
+		minH := uint64(math.MaxUint64)
 
-	log.Infof("Persisting actors")
-
-	paDone := 0
-	par(50, maparr(toSync), func(bh *types.BlockHeader) {
-		paDone++
-		if paDone%100 == 0 {
-			log.Infof("pa: %d %d%%", paDone, (paDone*100)/len(toSync))
+		for _, header := range allToSync {
+			if header.Height < minH {
+				minH = header.Height
+			}
 		}
 
-		if len(bh.Parents) == 0 { // genesis case
-			ts, err := types.NewTipSet([]*types.BlockHeader{bh})
-			aadrs, err := api.StateListActors(ctx, ts)
+		toSync := map[cid.Cid]*types.BlockHeader{}
+		for c, header := range allToSync {
+			if header.Height < minH+maxBatch {
+				toSync[c] = header
+			}
+		}
+		for c := range toSync {
+			delete(allToSync, c)
+		}
+
+		log.Infof("Syncing %d blocks", len(toSync))
+
+		paDone := 0
+		par(50, maparr(toSync), func(bh *types.BlockHeader) {
+			paDone++
+			if paDone%100 == 0 {
+				log.Infof("pa: %d %d%%", paDone, (paDone*100)/len(toSync))
+			}
+
+			if len(bh.Parents) == 0 { // genesis case
+				ts, err := types.NewTipSet([]*types.BlockHeader{bh})
+				aadrs, err := api.StateListActors(ctx, ts)
+				if err != nil {
+					log.Error(err)
+					return
+				}
+
+				par(50, aadrs, func(addr address.Address) {
+					act, err := api.StateGetActor(ctx, addr, ts)
+					if err != nil {
+						log.Error(err)
+						return
+					}
+					alk.Lock()
+					_, ok := actors[addr]
+					if !ok {
+						actors[addr] = map[types.Actor]cid.Cid{}
+					}
+					actors[addr][*act] = bh.ParentStateRoot
+					addresses[addr] = address.Undef
+					alk.Unlock()
+				})
+
+				return
+			}
+
+			pts, err := api.ChainGetTipSet(ctx, types.NewTipSetKey(bh.Parents...))
 			if err != nil {
 				log.Error(err)
 				return
 			}
 
-			par(50, aadrs, func(addr address.Address) {
-				act, err := api.StateGetActor(ctx, addr, ts)
+			changes, err := api.StateChangedActors(ctx, pts.ParentState(), bh.ParentStateRoot)
+			if err != nil {
+				log.Error(err)
+				return
+			}
+
+			for a, act := range changes {
+				addr, err := address.NewFromString(a)
 				if err != nil {
 					log.Error(err)
 					return
 				}
+
 				alk.Lock()
 				_, ok := actors[addr]
 				if !ok {
 					actors[addr] = map[types.Actor]cid.Cid{}
 				}
-				actors[addr][*act] = bh.ParentStateRoot
+				actors[addr][act] = bh.ParentStateRoot
 				addresses[addr] = address.Undef
 				alk.Unlock()
-			})
+			}
+		})
 
-			return
+		log.Infof("Getting messages")
+
+		msgs, incls := fetchMessages(ctx, api, toSync)
+
+		log.Infof("Resolving addresses")
+
+		for _, message := range msgs {
+			addresses[message.To] = address.Undef
+			addresses[message.From] = address.Undef
 		}
 
-		pts, err := api.ChainGetTipSet(ctx, types.NewTipSetKey(bh.Parents...))
-		if err != nil {
-			log.Error(err)
-			return
+		par(50, kmaparr(addresses), func(addr address.Address) {
+			raddr, err := api.StateLookupID(ctx, addr, nil)
+			if err != nil {
+				log.Warn(err)
+				return
+			}
+			alk.Lock()
+			addresses[addr] = raddr
+			alk.Unlock()
+		})
+
+		log.Infof("Getting miner info")
+
+		miners := map[minerKey]*minerInfo{}
+
+		for addr, m := range actors {
+			for actor, c := range m {
+				if !(actor.Code == actors2.StorageMinerCodeCid || actor.Code == actors2.StorageMiner2CodeCid) {
+					continue
+				}
+
+				miners[minerKey{
+					addr:      addr,
+					act:       actor,
+					stateroot: c,
+				}] = &minerInfo{}
+			}
 		}
 
-		changes, err := api.StateChangedActors(ctx, pts.ParentState(), bh.ParentStateRoot)
-		if err != nil {
-			log.Error(err)
-			return
-		}
+		par(50, kvmaparr(miners), func(it func() (minerKey, *minerInfo)) {
+			k, info := it()
 
-		for a, act := range changes {
-			addr, err := address.NewFromString(a)
+			sszs, err := api.StateMinerSectorCount(ctx, k.addr, nil)
+			if err != nil {
+				log.Error(err)
+				return
+			}
+			info.psize = sszs.Pset
+			info.ssize = sszs.Sset
+
+			astb, err := api.ChainReadObj(ctx, k.act.Head)
 			if err != nil {
 				log.Error(err)
 				return
 			}
 
-			alk.Lock()
-			_, ok := actors[addr]
-			if !ok {
-				actors[addr] = map[types.Actor]cid.Cid{}
-			}
-			actors[addr][act] = bh.ParentStateRoot
-			addresses[addr] = address.Undef
-			alk.Unlock()
-		}
-	})
-
-	if err := st.storeActors(actors); err != nil {
-		log.Error(err)
-		return
-	}
-
-	log.Infof("Persisting miners")
-
-	miners := map[minerKey]*minerInfo{}
-
-	for addr, m := range actors {
-		for actor, c := range m {
-			if actor.Code != actors2.StorageMinerCodeCid {
-				continue
+			if err := info.state.UnmarshalCBOR(bytes.NewReader(astb)); err != nil {
+				log.Error(err)
+				return
 			}
 
-			miners[minerKey{
-				addr:      addr,
-				act:       actor,
-				stateroot: c,
-			}] = &minerInfo{}
-		}
-	}
+			ib, err := api.ChainReadObj(ctx, info.state.Info)
+			if err != nil {
+				log.Error(err)
+				return
+			}
 
-	par(50, kvmaparr(miners), func(it func() (minerKey, *minerInfo)) {
-		k, info := it()
+			if err := info.info.UnmarshalCBOR(bytes.NewReader(ib)); err != nil {
+				log.Error(err)
+				return
+			}
+		})
 
-		sszs, err := api.StateMinerSectorCount(ctx, k.addr, nil)
-		if err != nil {
-			log.Error(err)
+		log.Info("Getting receipts")
+
+		receipts := fetchParentReceipts(ctx, api, toSync)
+
+		log.Info("Storing headers")
+
+		if err := st.storeHeaders(toSync, true); err != nil {
+			log.Errorf("%+v", err)
 			return
 		}
-		info.psize = sszs.Pset
-		info.ssize = sszs.Sset
 
-		astb, err := api.ChainReadObj(ctx, k.act.Head)
-		if err != nil {
+		log.Info("Storing address mapping")
+
+		if err := st.storeAddressMap(addresses); err != nil {
 			log.Error(err)
 			return
 		}
 
-		if err := info.state.UnmarshalCBOR(bytes.NewReader(astb)); err != nil {
+		log.Info("Storing actors")
+
+		if err := st.storeActors(actors); err != nil {
 			log.Error(err)
 			return
 		}
 
-		ib, err := api.ChainReadObj(ctx, info.state.Info)
-		if err != nil {
+		log.Info("Storing miners")
+
+		if err := st.storeMiners(miners); err != nil {
 			log.Error(err)
 			return
 		}
 
-		if err := info.info.UnmarshalCBOR(bytes.NewReader(ib)); err != nil {
+		log.Infof("Storing messages")
+
+		if err := st.storeMessages(msgs); err != nil {
 			log.Error(err)
 			return
 		}
-	})
 
-	if err := st.storeMiners(miners); err != nil {
-		log.Error(err)
-		return
-	}
+		log.Info("Storing message inclusions")
 
-	log.Infof("Persisting headers")
-	if err := st.storeHeaders(toSync, true); err != nil {
-		log.Error(err)
-		return
-	}
-
-	log.Infof("Getting messages")
-
-	msgs, incls := fetchMessages(ctx, api, toSync)
-
-	if err := st.storeMessages(msgs); err != nil {
-		log.Error(err)
-		return
-	}
-
-	if err := st.storeMsgInclusions(incls); err != nil {
-		log.Error(err)
-		return
-	}
-
-	log.Infof("Getting parent receipts")
-
-	receipts := fetchParentReceipts(ctx, api, toSync)
-
-	if err := st.storeReceipts(receipts); err != nil {
-		log.Error(err)
-		return
-	}
-
-	log.Infof("Resolving addresses")
-
-	for _, message := range msgs {
-		addresses[message.To] = address.Undef
-		addresses[message.From] = address.Undef
-	}
-
-	par(50, kmaparr(addresses), func(addr address.Address) {
-		raddr, err := api.StateLookupID(ctx, addr, nil)
-		if err != nil {
-			log.Warn(err)
+		if err := st.storeMsgInclusions(incls); err != nil {
+			log.Error(err)
 			return
 		}
-		alk.Lock()
-		addresses[addr] = raddr
-		alk.Unlock()
-	})
 
-	if err := st.storeAddressMap(addresses); err != nil {
-		log.Error(err)
-		return
+		log.Infof("Storing parent receipts")
+
+		if err := st.storeReceipts(receipts); err != nil {
+			log.Error(err)
+			return
+		}
+		log.Infof("Sync stage done")
 	}
 
 	log.Infof("Sync done")

cmd/lotus-chainwatch/templates.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strconv"
 
 	rice "github.com/GeertJohan/go.rice"
 	"github.com/ipfs/go-cid"
@@ -42,8 +43,15 @@ func newHandler(api api.FullNode, st *storage) (*handler, error) {
 		"queryNum": h.queryNum,
 		"sizeStr":  sizeStr,
 		"strings":  h.strings,
+		"qstr":     h.qstr,
+		"qstrs":    h.qstrs,
 		"messages": h.messages,
 
+		"pageDown": pageDown,
+		"parseInt": func(s string) (int, error) { i, e := strconv.ParseInt(s, 10, 64); return int(i), e },
+		"substr":   func(i, j int, s string) string { return s[i:j] },
+		"sub":      func(a, b int) int { return a - b }, // TODO: really not builtin?
+
 		"param": func(string) string { return "" }, // replaced in request handler
 	}
 
@@ -153,10 +161,9 @@ func (h *handler) netPower(slashFilt string) (types.BigInt, error) {
 	if slashFilt != "" {
 		slashFilt = " where " + slashFilt
 	}
-	return h.queryNum(`select sum(power) from (
-	select miner_heads.power, miner_heads.slashed_at, max(height) from miner_heads
-    	inner join blocks b on miner_heads.stateroot = b.parentStateRoot
-	group by miner_heads.addr)` + slashFilt)
+	return h.queryNum(`select sum(power) from (select distinct on (addr) power, slashed_at from miner_heads
+    inner join blocks b on miner_heads.stateroot = b.parentStateRoot
+order by addr, height desc) as p` + slashFilt)
 }
 
 func (h *handler) queryNum(q string, p ...interface{}) (types.BigInt, error) {
@@ -193,7 +200,7 @@ func (h *handler) strings(table string, col string, filter string, args ...inter
 	if len(filter) > 0 {
 		filter = " where " + filter
 	}
-	log.Info("strings qstr ", "select "+col+" from "+table+filter)
+	log.Info("strings qstr ", "select "+col+" from "+table+filter, args)
 	rws, err := h.st.db.Query("select "+col+" from "+table+filter, args...)
 	if err != nil {
 		return nil, err
@@ -209,11 +216,40 @@ func (h *handler) strings(table string, col string, filter string, args ...inter
 	return
 }
 
+func (h *handler) qstr(q string, p ...interface{}) (string, error) {
+	// explicitly not caring about sql injection too much, this doesn't take user input
+
+	r, err := h.qstrs(q, 1, p...)
+	if err != nil {
+		return "", err
+	}
+	return r[0], nil
+}
+
+func (h *handler) qstrs(q string, n int, p ...interface{}) ([]string, error) {
+	// explicitly not caring about sql injection too much, this doesn't take user input
+
+	c := make([]string, n)
+	ia := make([]interface{}, n)
+	for i := range c {
+		ia[i] = &c[i]
+	}
+	err := h.st.db.QueryRow(q, p...).Scan(ia...)
+	if err != nil {
+		log.Error("qnum ", q, p, err)
+		return nil, err
+	}
+
+	return c, nil
+}
+
 func (h *handler) messages(filter string, args ...interface{}) (out []types.Message, err error) {
 	if len(filter) > 0 {
 		filter = " where " + filter
 	}
 
+	log.Info("select * from messages " + filter)
+
 	rws, err := h.st.db.Query("select * from messages "+filter, args...)
 	if err != nil {
 		return nil, err
@@ -250,4 +286,13 @@ func (h *handler) messages(filter string, args ...interface{}) (out []types.Mess
 	return
 }
 
+func pageDown(base, n int) []int {
+	out := make([]int, n)
+	for i := range out {
+		out[i] = base - i
+	}
+
+	return out
+}
+
 var _ http.Handler = &handler{}

cmd/lotus-fountain/main.go

@@ -11,7 +11,7 @@ import (
 
 	rice "github.com/GeertJohan/go.rice"
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	peer "github.com/libp2p/go-libp2p-peer"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
@@ -275,7 +275,7 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) {
 	createStorageMinerMsg := &types.Message{
 		To:    actors.StoragePowerAddress,
 		From:  h.from,
-		Value: collateral,
+		Value: types.BigAdd(collateral, types.BigDiv(collateral, types.NewInt(100))),
 
 		Method: actors.SPAMethods.CreateStorageMiner,
 		Params: params,

cmd/lotus-fountain/site/miner.html

@@ -15,8 +15,7 @@
                 <span>Enter destination address:</span>
                 <input type='text' name='address' style="width: 300px">
                 <select name="sectorSize">
-                    <option selected value="1073741824">1GiB sectors</option>
-                    <option value="34359738368">32GiB sectors</option>
+                    <option selected value="34359738368">32GiB sectors</option>
                 </select>
                 <button type='submit'>Create Miner</button>
             </form>

cmd/lotus-health/main.go

@@ -0,0 +1,211 @@
+package main
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
+	lcli "github.com/filecoin-project/lotus/cli"
+	cid "github.com/ipfs/go-cid"
+	logging "github.com/ipfs/go-log"
+	"gopkg.in/urfave/cli.v2"
+)
+
+type CidWindow [][]cid.Cid
+
+var log = logging.Logger("lotus-health")
+
+func main() {
+	logging.SetLogLevel("*", "INFO")
+
+	log.Info("Starting health agent")
+
+	local := []*cli.Command{
+		watchHeadCmd,
+	}
+
+	app := &cli.App{
+		Name:     "lotus-health",
+		Usage:    "Tools for monitoring lotus daemon health",
+		Version:  build.UserVersion,
+		Commands: local,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "repo",
+				EnvVars: []string{"LOTUS_PATH"},
+				Value:   "~/.lotus", // TODO: Consider XDG_DATA_HOME
+			},
+		},
+	}
+
+	if err := app.Run(os.Args); err != nil {
+		log.Warn(err)
+		return
+	}
+}
+
+var watchHeadCmd = &cli.Command{
+	Name: "watch-head",
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Name:  "threshold",
+			Value: 3,
+			Usage: "number of times head remains unchanged before failing health check",
+		},
+		&cli.IntFlag{
+			Name:  "interval",
+			Value: build.BlockDelay,
+			Usage: "interval in seconds between chain head checks",
+		},
+		&cli.StringFlag{
+			Name:  "systemd-unit",
+			Value: "lotus-daemon.service",
+			Usage: "systemd unit name to restart on health check failure",
+		},
+	},
+	Action: func(c *cli.Context) error {
+		threshold := c.Int("threshold")
+		interval := time.Duration(c.Int("interval")) * time.Second
+		name := c.String("systemd-unit")
+
+		var headCheckWindow CidWindow
+
+		api, closer, err := lcli.GetFullNodeAPI(c)
+		if err != nil {
+			return err
+		}
+		defer closer()
+		ctx := lcli.ReqContext(c)
+
+		if err := WaitForSyncComplete(ctx, api); err != nil {
+			log.Fatal(err)
+		}
+
+		ch := make(chan CidWindow, 1)
+		aCh := make(chan interface{}, 1)
+
+		go func() {
+			for {
+				headCheckWindow, err = updateWindow(ctx, api, headCheckWindow, threshold, ch)
+				if err != nil {
+					log.Fatal(err)
+				}
+				time.Sleep(interval)
+			}
+		}()
+
+		go func() {
+			result, err := alertHandler(name, aCh)
+			if err != nil {
+				log.Fatal(err)
+			}
+			if result != "done" {
+				log.Fatal("systemd unit failed to restart:", result)
+			}
+			log.Info("restarting health agent")
+			// Exit health agent and let supervisor restart health agent
+			// Restarting lotus systemd unit kills api connection
+			os.Exit(130)
+		}()
+
+		for {
+			ok := checkWindow(ch, int(interval))
+			if !ok {
+				log.Warn("chain head has not updated. Restarting systemd service")
+				aCh <- nil
+				break
+			}
+			log.Info("chain head is healthy")
+		}
+		return nil
+	},
+}
+
+/*
+ * reads channel of slices of Cids
+ * compares slices of Cids when len is greater or equal to `t` - threshold
+ * if all slices are equal, head has not updated and returns false
+ */
+func checkWindow(ch chan CidWindow, t int) bool {
+	select {
+	case window := <-ch:
+		var dup int
+		windowLen := len(window)
+		if windowLen >= t {
+		cidWindow:
+			for i, cids := range window {
+				next := windowLen - 1 - i
+				// if array length is different, head is changing
+				if next >= 1 && len(window[next]) != len(window[next-1]) {
+					break cidWindow
+				}
+				// if cids are different, head is changing
+				for j := range cids {
+					if next >= 1 && window[next][j] != window[next-1][j] {
+						break cidWindow
+					}
+				}
+				if i < (t - 1) {
+					dup++
+				}
+			}
+
+			if dup == (t - 1) {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+/*
+ * reads channel of slices of slices of Cids
+ * compares Cids when len of window is greater or equal to `t` - threshold
+ * if all slices are the equal, head has not updated and returns false
+ */
+func updateWindow(ctx context.Context, a api.FullNode, w CidWindow, t int, ch chan CidWindow) (CidWindow, error) {
+	head, err := a.ChainHead(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	window := appendCIDsToWindow(w, head.Cids(), t)
+	ch <- window
+	return window, nil
+}
+
+/*
+ * appends slice of Cids to window slice
+ * keeps a fixed window slice size, dropping older slices
+ * returns new window
+ */
+func appendCIDsToWindow(w CidWindow, c []cid.Cid, t int) CidWindow {
+	offset := len(w) - t + 1
+	if offset >= 0 {
+		return append(w[offset:], c)
+	}
+	return append(w, c)
+}
+
+/*
+ * wait for node to sync
+ */
+func WaitForSyncComplete(ctx context.Context, napi api.FullNode) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(3 * time.Second):
+			head, err := napi.ChainHead(ctx)
+			if err != nil {
+				return err
+			}
+
+			if time.Now().Unix()-int64(head.MinTimestamp()) < build.BlockDelay {
+				return nil
+			}
+		}
+	}
+}

cmd/lotus-health/main_test.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"testing"
+
+	cid "github.com/ipfs/go-cid"
+	mh "github.com/multiformats/go-multihash"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAppendCIDsToWindow(t *testing.T) {
+	assert := assert.New(t)
+	var window CidWindow
+	threshold := 3
+	cid0 := makeCID("0")
+	cid1 := makeCID("1")
+	cid2 := makeCID("2")
+	cid3 := makeCID("3")
+	window = appendCIDsToWindow(window, []cid.Cid{cid0}, threshold)
+	window = appendCIDsToWindow(window, []cid.Cid{cid1}, threshold)
+	window = appendCIDsToWindow(window, []cid.Cid{cid2}, threshold)
+	window = appendCIDsToWindow(window, []cid.Cid{cid3}, threshold)
+	assert.Len(window, 3)
+	assert.Equal(window[0][0], cid1)
+	assert.Equal(window[1][0], cid2)
+	assert.Equal(window[2][0], cid3)
+}
+
+func TestCheckWindow(t *testing.T) {
+	assert := assert.New(t)
+	ch := make(chan CidWindow, 1)
+	och := make(chan bool, 1)
+	threshold := 3
+	go func() {
+		och <- checkWindow(ch, threshold)
+	}()
+	var healthyHeadCheckWindow CidWindow
+	healthyHeadCheckWindow = appendCIDsToWindow(healthyHeadCheckWindow, []cid.Cid{
+		makeCID("abcd"),
+	}, threshold)
+	healthyHeadCheckWindow = appendCIDsToWindow(healthyHeadCheckWindow, []cid.Cid{
+		makeCID("bbcd"),
+		makeCID("bbfe"),
+	}, threshold)
+	healthyHeadCheckWindow = appendCIDsToWindow(healthyHeadCheckWindow, []cid.Cid{
+		makeCID("bbcd"),
+		makeCID("bbfe"),
+	}, threshold)
+	ch <- healthyHeadCheckWindow
+	select {
+	case ok := <-och:
+		assert.True(ok)
+	}
+	go func() {
+		och <- checkWindow(ch, threshold)
+	}()
+	var unhealthyHeadCheckWindow CidWindow
+	unhealthyHeadCheckWindow = appendCIDsToWindow(unhealthyHeadCheckWindow, []cid.Cid{
+		makeCID("abcd"),
+		makeCID("fbcd"),
+	}, threshold)
+	unhealthyHeadCheckWindow = appendCIDsToWindow(unhealthyHeadCheckWindow, []cid.Cid{
+		makeCID("abcd"),
+		makeCID("fbcd"),
+	}, threshold)
+	unhealthyHeadCheckWindow = appendCIDsToWindow(unhealthyHeadCheckWindow, []cid.Cid{
+		makeCID("abcd"),
+		makeCID("fbcd"),
+	}, threshold)
+	ch <- unhealthyHeadCheckWindow
+	select {
+	case ok := <-och:
+		assert.False(ok)
+	}
+
+}
+
+func makeCID(s string) cid.Cid {
+	h1, err := mh.Sum([]byte(s), mh.SHA2_256, -1)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return cid.NewCidV1(0x55, h1)
+}

cmd/lotus-health/systemd.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"github.com/coreos/go-systemd/dbus"
+)
+
+func alertHandler(n string, ch chan interface{}) (string, error) {
+	select {
+	case <-ch:
+		statusCh := make(chan string, 1)
+		c, err := dbus.New()
+		if err != nil {
+			return "", err
+		}
+		_, err = c.TryRestartUnit(n, "fail", statusCh)
+		if err != nil {
+			return "", err
+		}
+		select {
+		case result := <-statusCh:
+			return result, nil
+		}
+	}
+}

cmd/lotus-seal-worker/main.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/mitchellh/go-homedir"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
 

cmd/lotus-seal-worker/sub.go

@@ -5,10 +5,11 @@ import (
 	"net/http"
 
 	paramfetch "github.com/filecoin-project/go-paramfetch"
+	"github.com/filecoin-project/go-sectorbuilder"
 	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/go-sectorbuilder"
 	lapi "github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
 )
 
 type worker struct {
@@ -40,7 +41,7 @@ func acceptJobs(ctx context.Context, api lapi.StorageMiner, endpoint string, aut
 		return err
 	}
 
-	if err := paramfetch.GetParams(ssize); err != nil {
+	if err := paramfetch.GetParams(build.ParametersJson, ssize); err != nil {
 		return xerrors.Errorf("get params: %w", err)
 	}
 
@@ -102,7 +103,7 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask)
 
 	switch task.Type {
 	case sectorbuilder.WorkerPreCommit:
-		rspco, err := w.sb.SealPreCommit(task.SectorID, task.SealTicket, task.Pieces)
+		rspco, err := w.sb.SealPreCommit(ctx, task.SectorID, task.SealTicket, task.Pieces)
 		if err != nil {
 			return errRes(xerrors.Errorf("precomitting: %w", err))
 		}
@@ -115,8 +116,12 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask)
 		if err := w.push("cache", task.SectorID); err != nil {
 			return errRes(xerrors.Errorf("pushing precommited data: %w", err))
 		}
+
+		if err := w.remove("staging", task.SectorID); err != nil {
+			return errRes(xerrors.Errorf("cleaning up staged sector: %w", err))
+		}
 	case sectorbuilder.WorkerCommit:
-		proof, err := w.sb.SealCommit(task.SectorID, task.SealTicket, task.SealSeed, task.Pieces, task.Rspco)
+		proof, err := w.sb.SealCommit(ctx, task.SectorID, task.SealTicket, task.SealSeed, task.Pieces, task.Rspco)
 		if err != nil {
 			return errRes(xerrors.Errorf("comitting: %w", err))
 		}
@@ -126,6 +131,10 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask)
 		if err := w.push("cache", task.SectorID); err != nil {
 			return errRes(xerrors.Errorf("pushing precommited data: %w", err))
 		}
+
+		if err := w.remove("sealed", task.SectorID); err != nil {
+			return errRes(xerrors.Errorf("cleaning up sealed sector: %w", err))
+		}
 	}
 
 	return res

cmd/lotus-seal-worker/transfer.go

@@ -125,6 +125,11 @@ func (w *worker) push(typ string, sectorID uint64) error {
 	}
 
 	// TODO: keep files around for later stages of sealing
+	return w.remove(typ, sectorID)
+}
+
+func (w *worker) remove(typ string, sectorID uint64) error {
+	filename := filepath.Join(w.repo, typ, w.sb.SectorName(sectorID))
 	return os.RemoveAll(filename)
 }
 

cmd/lotus-seed/main.go

@@ -8,8 +8,10 @@ import (
 	"encoding/json"
 
 	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
-	badger "github.com/ipfs/go-ds-badger"
-	logging "github.com/ipfs/go-log"
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore/namespace"
+	badger "github.com/ipfs/go-ds-badger2"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/mitchellh/go-homedir"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
@@ -196,7 +198,7 @@ var aggregateSectorDirsCmd = &cli.Command{
 			SectorSize:    ssize,
 			Dir:           destdir,
 			WorkerThreads: 2,
-		}, agmds)
+		}, namespace.Wrap(agmds, datastore.NewKey("/sectorbuilder")))
 		if err != nil {
 			return err
 		}
@@ -257,7 +259,7 @@ var aggregateSectorDirsCmd = &cli.Command{
 				SectorSize:    genm.SectorSize,
 				Dir:           dir,
 				WorkerThreads: 2,
-			}, mds)
+			}, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder")))
 			if err != nil {
 				return err
 			}

cmd/lotus-seed/seed/seed.go

@@ -11,8 +11,10 @@ import (
 	"path/filepath"
 
 	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
-	badger "github.com/ipfs/go-ds-badger"
-	logging "github.com/ipfs/go-log"
+	"github.com/ipfs/go-datastore"
+	"github.com/ipfs/go-datastore/namespace"
+	badger "github.com/ipfs/go-ds-badger2"
+	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
@@ -43,7 +45,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb
 		return nil, err
 	}
 
-	sb, err := sectorbuilder.New(cfg, mds)
+	sb, err := sectorbuilder.New(cfg, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder")))
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +71,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb
 
 		fmt.Printf("sector-id: %d, piece info: %v", sid, pi)
 
-		pco, err := sb.SealPreCommit(sid, ticket, []sectorbuilder.PublicPieceInfo{pi})
+		pco, err := sb.SealPreCommit(context.TODO(), sid, ticket, []sectorbuilder.PublicPieceInfo{pi})
 		if err != nil {
 			return nil, xerrors.Errorf("commit: %w", err)
 		}

cmd/lotus-shed/main.go

@@ -3,7 +3,7 @@ package main
 import (
 	"os"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"gopkg.in/urfave/cli.v2"
 
 	"github.com/filecoin-project/lotus/build"