Merge pull request #997 from filecoin-project/feat/testnet2

Testnet2

Makefile

@@ -76,6 +76,7 @@ BINS+=lotus-seal-worker
 lotus-shed: $(BUILD_DEPS)
 	rm -f lotus-shed
 	go build $(GOFLAGS) -o lotus-shed ./cmd/lotus-shed
+	go run github.com/GeertJohan/go.rice/rice append --exec lotus-shed -i ./build
 .PHONY: lotus-seal-worker
 BINS+=lotus-seal-worker
 
@@ -144,6 +145,7 @@ BINS+=bench
 stats:
 	rm -f stats
 	go build -o stats ./tools/stats
+	go run github.com/GeertJohan/go.rice/rice append --exec stats -i ./build
 .PHONY: stats
 BINS+=stats
 

api/api_storage.go

@@ -13,7 +13,8 @@ type SectorState = uint64
 const (
 	UndefinedSectorState SectorState = iota
 
-	Empty   // TODO: Is this useful
+	// happy path
+	Empty
 	Packing // sector not in sealStore, and not on chain
 
 	Unsealed      // sealing / queued
@@ -22,13 +23,32 @@ const (
 	Committing
 	CommitWait // waiting for message to land on chain
 	Proving
+	_ // reserved
+	_
+	_
+	_
+
+	// recovery handling
+	// Reseal
+	_
+	_
+	_
+	_
+	_
+	_
+	_
+
+	// error modes
+	FailedUnrecoverable
 
 	SealFailed
 	PreCommitFailed
 	SealCommitFailed
 	CommitFailed
-
+	_
-	FailedUnrecoverable
+	_
+	_
+	_
 
 	Faulty        // sector is corrupted or gone for some reason
 	FaultReported // sector has been declared as a fault on chain

api/cbor_gen.go

@@ -23,6 +23,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Channel (address.Address) (struct)
+	if len("Channel") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Channel\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Channel")))); err != nil {
 		return err
 	}
@@ -35,6 +39,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.ChannelMessage (cid.Cid) (struct)
+	if len("ChannelMessage") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"ChannelMessage\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("ChannelMessage")))); err != nil {
 		return err
 	}
@@ -53,6 +61,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Vouchers ([]*types.SignedVoucher) (slice)
+	if len("Vouchers") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Vouchers\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Vouchers")))); err != nil {
 		return err
 	}
@@ -60,6 +72,10 @@ func (t *PaymentInfo) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
+	if len(t.Vouchers) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.Vouchers was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Vouchers)))); err != nil {
 		return err
 	}
@@ -200,6 +216,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.SectorID (uint64) (uint64)
+	if len("SectorID") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"SectorID\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("SectorID")))); err != nil {
 		return err
 	}
@@ -212,6 +232,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Offset (uint64) (uint64)
+	if len("Offset") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Offset\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Offset")))); err != nil {
 		return err
 	}
@@ -224,6 +248,10 @@ func (t *SealedRef) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Size (uint64) (uint64)
+	if len("Size") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Size\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Size")))); err != nil {
 		return err
 	}
@@ -336,6 +364,10 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 	}
 
 	// t.Refs ([]api.SealedRef) (slice)
+	if len("Refs") > cbg.MaxLength {
+		return xerrors.Errorf("Value in field \"Refs\" was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len("Refs")))); err != nil {
 		return err
 	}
@@ -343,6 +375,10 @@ func (t *SealedRefs) MarshalCBOR(w io.Writer) error {
 		return err
 	}
 
+	if len(t.Refs) > cbg.MaxLength {
+		return xerrors.Errorf("Slice value in field t.Refs was too long")
+	}
+
 	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajArray, uint64(len(t.Refs)))); err != nil {
 		return err
 	}

api/test/deals.go

@@ -11,7 +11,7 @@ import (
 	"testing"
 	"time"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
@@ -24,7 +24,7 @@ func init() {
 	build.InsecurePoStValidation = true
 }
 
-func TestDealFlow(t *testing.T, b APIBuilder) {
+func TestDealFlow(t *testing.T, b APIBuilder, blocktime time.Duration) {
 	os.Setenv("BELLMAN_NO_GPU", "1")
 
 	ctx := context.Background()
@@ -64,7 +64,7 @@ func TestDealFlow(t *testing.T, b APIBuilder) {
 	go func() {
 		defer close(done)
 		for mine {
-			time.Sleep(time.Second)
+			time.Sleep(blocktime)
 			fmt.Println("mining a block now")
 			if err := sn[0].MineOne(ctx); err != nil {
 				t.Error(err)

build/bootstrap/bootstrappers.pi

@@ -1,6 +1,6 @@
-/dns4/lotus-bootstrap-0.dfw.fil-test.net/tcp/1347/p2p/12D3KooWHwGBSiLR5ts7KW9MgH4BMzC2iXe18kwAQ8Ee3LUd1jeR
+/dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWLZs8BWtEzRTYET4yR4jzDtPamaA1YsyPQJq6cf2RfxBD
-/dns4/lotus-bootstrap-1.dfw.fil-test.net/tcp/1347/p2p/12D3KooWCLFaawdhLGcSpiqg43DtZ9QzPQ6HcB8Vvyu2Cnta8UWc
+/dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWGvrgjWw4Yqo4AFWqYp4g37FpUvUCQBkNWudZVSwR9tY1
-/dns4/lotus-bootstrap-0.fra.fil-test.net/tcp/1347/p2p/12D3KooWMmaL7eaUCF6tVAghVmgozxz4uztbuFUQv6dyFpHRarHR
+/dns4/lotus-bootstrap-0.fra.fil-test.net/tcp/1347/p2p/12D3KooWSfNcrD1cs5Cj5eSHbK6nHCqJLffAuPqvRMBRgvUdqQhX
-/dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWLLpNYoKdf9NgcWudBhXLdTcXncqAsTzozw1scMMu6nS5
+/dns4/lotus-bootstrap-1.fra.fil-test.net/tcp/1347/p2p/12D3KooWNkXyVPspUnrHUiSC3VJPMcXvHuNdy3BTCLTPPnDgwwTT
-/dns4/lotus-bootstrap-0.sin.fil-test.net/tcp/1347/p2p/12D3KooWCNL9vXaXwNs3Bu8uRAJK4pxpCyPeM7jZLSDpJma1wrV8
+/dns4/lotus-bootstrap-0.dfw.fil-test.net/tcp/1347/p2p/12D3KooWSgJWJZK8LTRtCWzPa5FQheCFJjHpficVYgEQWeimcqCu
-/dns4/lotus-bootstrap-1.sin.fil-test.net/tcp/1347/p2p/12D3KooWNGGxFda1eC5U2YKAgs4ypoFHn3Z3xHCsjmFdrCcytoxm
+/dns4/lotus-bootstrap-1.dfw.fil-test.net/tcp/1347/p2p/12D3KooWFPaC4dyGpbNXCpVHjZucdJnDwmv4ng9tponPx5GrzJkT

build/forks.go

@@ -1,4 +1,3 @@
 package build
 
-const ForkCCM = 1750
+// No forks yet \o/
-const ForkNoPowerEPSUpdates = 16450

build/genesis.go

@@ -2,7 +2,7 @@ package build
 
 import (
 	rice "github.com/GeertJohan/go.rice"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 )
 
 // moved from now-defunct build/paramfetch.go
@@ -11,12 +11,12 @@ var log = logging.Logger("build")
 func MaybeGenesis() []byte {
 	builtinGen, err := rice.FindBox("genesis")
 	if err != nil {
-		log.Warn("loading built-in genesis: %s", err)
+		log.Warnf("loading built-in genesis: %s", err)
 		return nil
 	}
 	genBytes, err := builtinGen.Bytes("devnet.car")
 	if err != nil {
-		log.Warn("loading built-in genesis: %s", err)
+		log.Warnf("loading built-in genesis: %s", err)
 	}
 
 	return genBytes

build/params_shared.go

@@ -56,6 +56,9 @@ const SealRandomnessLookback = Finality
 // Epochs
 const SealRandomnessLookbackLimit = SealRandomnessLookback + 2000
 
+// Maximum lookback that randomness can be sourced from for a seal proof submission
+const MaxSealLookback = SealRandomnessLookbackLimit + 2000
+
 // /////
 // Mining
 
@@ -97,3 +100,8 @@ const BadBlockCacheSize = 1 << 15
 // assuming 4000 messages per round, this lets us not lose any messages across a
 // 10 block reorg.
 const BlsSignatureCacheSize = 40000
+
+// ///////
+// Limits
+
+const BlockMessageLimit = 512

build/params_testnet.go

@@ -3,7 +3,6 @@
 package build
 
 var SectorSizes = []uint64{
-	1 << 30,
 	32 << 30,
 }
 

build/version.go

@@ -5,7 +5,7 @@ import "fmt"
 var CurrentCommit string
 
 // BuildVersion is the local build version, set by build system
-const BuildVersion = "0.1.6"
+const BuildVersion = "0.2.0"
 
 var UserVersion = BuildVersion + CurrentCommit
 

chain/actors/actor_init.go

@@ -15,7 +15,7 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-hamt-ipld"
 	cbor "github.com/ipfs/go-ipld-cbor"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	mh "github.com/multiformats/go-multihash"
 )
 

chain/actors/actor_miner.go

@@ -19,12 +19,10 @@ import (
 	cbor "github.com/ipfs/go-ipld-cbor"
 	"github.com/libp2p/go-libp2p-core/peer"
 	cbg "github.com/whyrusleeping/cbor-gen"
-	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
 )
 
 const MaxSectors = 1 << 48
-const RLEMax = 100e3
 
 type StorageMinerActor struct{}
 
@@ -141,10 +139,7 @@ func (sma StorageMinerActor) Exports() []interface{} {
 	return []interface{}{
 		1: sma.StorageMinerConstructor,
 		2: sma.PreCommitSector,
-		3: withUpdates(
+		3: sma.ProveCommitSector,
-			update{start: 0, method: sma.ProveCommitSectorV0},
-			update{start: build.ForkNoPowerEPSUpdates, method: sma.ProveCommitSectorV1},
-		),
 		4: sma.SubmitFallbackPoSt,
 		//5: sma.SlashStorageFault,
 		//6: sma.GetCurrentProvingSet,
@@ -159,10 +154,7 @@ func (sma StorageMinerActor) Exports() []interface{} {
 		//15: sma.ChangeWorker,
 		16: sma.IsSlashed,
 		17: sma.CheckMiner,
-		18: withUpdates(
+		18: sma.DeclareFaults,
-			update{start: 0, method: sma.DeclareFaultsV0},
-			update{start: build.ForkNoPowerEPSUpdates, method: sma.DeclareFaultsV1},
-		),
 		19: sma.SlashConsensusFault,
 		20: sma.SubmitElectionPoSt,
 	}
@@ -298,110 +290,7 @@ type SectorProveCommitInfo struct {
 	DealIDs  []uint64
 }
 
-func (sma StorageMinerActor) ProveCommitSectorV0(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) {
+func (sma StorageMinerActor) ProveCommitSector(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) {
-	ctx := vmctx.Context()
-	oldstate, self, err := loadState(vmctx)
-	if err != nil {
-		return nil, err
-	}
-
-	mi, err := loadMinerInfo(vmctx, self)
-	if err != nil {
-		return nil, err
-	}
-
-	us, ok := self.PreCommittedSectors[uintToStringKey(params.SectorID)]
-	if !ok {
-		return nil, aerrors.New(1, "no pre-commitment found for sector")
-	}
-
-	if us.ReceivedEpoch+build.InteractivePoRepDelay >= vmctx.BlockHeight() {
-		return nil, aerrors.New(2, "too early for proof submission")
-	}
-
-	delete(self.PreCommittedSectors, uintToStringKey(params.SectorID))
-
-	// TODO: ensure normalization to ID address
-	maddr := vmctx.Message().To
-
-	ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback)
-	if err != nil {
-		return nil, aerrors.Wrap(err, "failed to get ticket randomness")
-	}
-
-	seed, err := vmctx.GetRandomness(us.ReceivedEpoch + build.InteractivePoRepDelay)
-	if err != nil {
-		return nil, aerrors.Wrap(err, "failed to get randomness for prove sector commitment")
-	}
-
-	enc, err := SerializeParams(&ComputeDataCommitmentParams{
-		DealIDs:    params.DealIDs,
-		SectorSize: mi.SectorSize,
-	})
-	if err != nil {
-		return nil, aerrors.Wrap(err, "failed to serialize ComputeDataCommitmentParams")
-	}
-
-	commD, err := vmctx.Send(StorageMarketAddress, SMAMethods.ComputeDataCommitment, types.NewInt(0), enc)
-	if err != nil {
-		return nil, aerrors.Wrapf(err, "failed to compute data commitment (sector %d, deals: %v)", params.SectorID, params.DealIDs)
-	}
-
-	if ok, err := vmctx.Sys().ValidatePoRep(ctx, maddr, mi.SectorSize, commD, us.Info.CommR, ticket, params.Proof, seed, params.SectorID); err != nil {
-		return nil, err
-	} else if !ok {
-		return nil, aerrors.Newf(2, "porep proof was invalid (t:%x; s:%x(%d); p:%s)", ticket, seed, us.ReceivedEpoch+build.InteractivePoRepDelay, truncateHexPrint(params.Proof))
-	}
-
-	// Note: There must exist a unique index in the miner's sector set for each
-	// sector ID. The `faults`, `recovered`, and `done` parameters of the
-	// SubmitPoSt method express indices into this sector set.
-	nssroot, err := AddToSectorSet(ctx, types.WrapStorage(vmctx.Storage()), self.Sectors, params.SectorID, us.Info.CommR, commD)
-	if err != nil {
-		return nil, err
-	}
-	self.Sectors = nssroot
-
-	// if miner is not mining, start their proving period now
-	// Note: As written here, every miners first PoSt will only be over one sector.
-	// We could set up a 'grace period' for starting mining that would allow miners
-	// to submit several sectors for their first proving period. Alternatively, we
-	// could simply make the 'PreCommitSector' call take multiple sectors at a time.
-	//
-	// Note: Proving period is a function of sector size; small sectors take less
-	// time to prove than large sectors do. Sector size is selected when pledging.
-	pss, lerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
-	if lerr != nil {
-		return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
-	}
-
-	if pss.Count == 0 {
-		self.ProvingSet = self.Sectors
-		// TODO: probably want to wait until the miner is above a certain
-		//  threshold before starting this
-		self.ElectionPeriodStart = vmctx.BlockHeight()
-	}
-
-	nstate, err := vmctx.Storage().Put(self)
-	if err != nil {
-		return nil, err
-	}
-	if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
-		return nil, err
-	}
-
-	activateParams, err := SerializeParams(&ActivateStorageDealsParams{
-		Deals: params.DealIDs,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	_, err = vmctx.Send(StorageMarketAddress, SMAMethods.ActivateStorageDeals, types.NewInt(0), activateParams)
-	return nil, aerrors.Wrapf(err, "calling ActivateStorageDeals failed")
-}
-
-func (sma StorageMinerActor) ProveCommitSectorV1(act *types.Actor, vmctx types.VMContext, params *SectorProveCommitInfo) ([]byte, ActorError) {
 	ctx := vmctx.Context()
 	oldstate, self, err := loadState(vmctx)
 	if err != nil {
@@ -431,6 +320,14 @@ func (sma StorageMinerActor) ProveCommitSectorV1(act *types.Actor, vmctx types.V
 	// TODO: ensure normalization to ID address
 	maddr := vmctx.Message().To
 
+	if vmctx.BlockHeight()-us.Info.SealEpoch > build.MaxSealLookback {
+		return nil, aerrors.Newf(5, "source randomness for sector SealEpoch too far in past (epoch %d)", us.Info.SealEpoch)
+	}
+
+	if vmctx.BlockHeight()-us.ReceivedEpoch > build.MaxSealLookback {
+		return nil, aerrors.Newf(6, "source randomness for sector ReceivedEpoch too far in past (epoch %d)", us.ReceivedEpoch)
+	}
+
 	ticket, err := vmctx.GetRandomness(us.Info.SealEpoch - build.SealRandomnessLookback)
 	if err != nil {
 		return nil, aerrors.Wrap(err, "failed to get ticket randomness")
@@ -577,25 +474,21 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
 		return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
 	}
 
-	{
+	ss, lerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
-		c, nerr := self.FaultSet.Count()
+	if lerr != nil {
-		if nerr != nil {
+		return nil, aerrors.HandleExternalError(lerr, "could not load proving set node")
-			return nil, aerrors.Absorb(nerr, 6, "invalid bitfield")
-		}
-
-		if c > RLEMax {
-			return nil, aerrors.Newf(7, "too many items in bitfield: %d", c)
-		}
 	}
 
-	faults, nerr := self.FaultSet.AllMap()
+	faults, nerr := self.FaultSet.AllMap(2 * ss.Count)
 	if nerr != nil {
 		return nil, aerrors.Absorb(err, 5, "RLE+ invalid")
 	}
 
+	activeFaults := uint64(0)
 	var sectorInfos []ffi.PublicSectorInfo
 	if err := pss.ForEach(func(id uint64, v *cbg.Deferred) error {
 		if faults[id] {
+			activeFaults++
 			return nil
 		}
 
@@ -632,8 +525,8 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
 		})
 	}
 
-	if ok, lerr := sectorbuilder.VerifyFallbackPost(vmctx.Context(), mi.SectorSize,
+	if ok, lerr := vmctx.Sys().VerifyFallbackPost(vmctx.Context(), mi.SectorSize,
-		sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, 0); !ok || lerr != nil { // TODO: FORK - set faults to len(faults)
+		sectorbuilder.NewSortedPublicSectorInfo(sectorInfos), seed[:], params.Proof, candidates, proverID, activeFaults); !ok || lerr != nil {
 		if lerr != nil {
 			// TODO: study PoST errors
 			return nil, aerrors.Absorb(lerr, 4, "PoST error")
@@ -644,7 +537,7 @@ func (sma StorageMinerActor) SubmitFallbackPoSt(act *types.Actor, vmctx types.VM
 	}
 
 	// Post submission is successful!
-	if err := onSuccessfulPoSt(self, vmctx); err != nil {
+	if err := onSuccessfulPoSt(self, vmctx, activeFaults); err != nil {
 		return nil, err
 	}
 
@@ -747,17 +640,6 @@ func RemoveFromSectorSet(ctx context.Context, s types.Storage, ss cid.Cid, ids [
 	return ncid, nil
 }
 
-func ValidatePoRep(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, ActorError) {
-	_, span := trace.StartSpan(ctx, "ValidatePoRep")
-	defer span.End()
-	ok, err := sectorbuilder.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof)
-	if err != nil {
-		return false, aerrors.Absorb(err, 25, "verify seal failed")
-	}
-
-	return ok, nil
-}
-
 func CollateralForPower(power types.BigInt) types.BigInt {
 	return types.BigMul(power, types.NewInt(10))
 	/* TODO: this
@@ -935,33 +817,7 @@ type DeclareFaultsParams struct {
 	Faults types.BitField
 }
 
-func (sma StorageMinerActor) DeclareFaultsV0(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) {
+func (sma StorageMinerActor) DeclareFaults(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) {
-	oldstate, self, aerr := loadState(vmctx)
-	if aerr != nil {
-		return nil, aerr
-	}
-
-	nfaults, err := types.MergeBitFields(params.Faults, self.FaultSet)
-	if err != nil {
-		return nil, aerrors.Absorb(err, 1, "failed to merge bitfields")
-	}
-
-	self.FaultSet = nfaults
-
-	self.LastFaultSubmission = vmctx.BlockHeight()
-
-	nstate, aerr := vmctx.Storage().Put(self)
-	if err != nil { // TODO: FORK: should be aerr
-		return nil, aerr
-	}
-	if err := vmctx.Storage().Commit(oldstate, nstate); err != nil {
-		return nil, err
-	}
-
-	return nil, nil
-}
-
-func (sma StorageMinerActor) DeclareFaultsV1(act *types.Actor, vmctx types.VMContext, params *DeclareFaultsParams) ([]byte, ActorError) {
 	oldstate, self, aerr := loadState(vmctx)
 	if aerr != nil {
 		return nil, aerr
@@ -981,6 +837,20 @@ func (sma StorageMinerActor) DeclareFaultsV1(act *types.Actor, vmctx types.VMCon
 		return nil, aerrors.Absorb(err, 1, "failed to merge bitfields")
 	}
 
+	ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
+	if nerr != nil {
+		return nil, aerrors.HandleExternalError(nerr, "failed to load sector set")
+	}
+
+	cf, nerr := nfaults.Count()
+	if nerr != nil {
+		return nil, aerrors.Absorb(nerr, 2, "could not decode RLE+")
+	}
+
+	if cf > 2*ss.Count {
+		return nil, aerrors.Newf(3, "too many declared faults: %d > %d", cf, 2*ss.Count)
+	}
+
 	self.FaultSet = nfaults
 
 	self.LastFaultSubmission = vmctx.BlockHeight()
@@ -1056,7 +926,41 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM
 		return nil, aerrors.New(1, "slashed miners can't perform election PoSt")
 	}
 
-	if err := onSuccessfulPoSt(self, vmctx); err != nil {
+	pss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
+	if nerr != nil {
+		return nil, aerrors.HandleExternalError(nerr, "failed to load proving set")
+	}
+
+	ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
+	if nerr != nil {
+		return nil, aerrors.HandleExternalError(nerr, "failed to load proving set")
+	}
+
+	faults, nerr := self.FaultSet.AllMap(2 * ss.Count)
+	if nerr != nil {
+		return nil, aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)")
+	}
+
+	activeFaults := uint64(0)
+	for f := range faults {
+		if f > amt.MaxIndex {
+			continue
+		}
+
+		var comms [][]byte
+		err := pss.Get(f, &comms)
+		if err != nil {
+			var notfound *amt.ErrNotFound
+			if !xerrors.As(err, &notfound) {
+				return nil, aerrors.HandleExternalError(err, "failed to find sector in sector set")
+			}
+			continue
+		}
+
+		activeFaults++
+	}
+
+	if err := onSuccessfulPoSt(self, vmctx, activeFaults); err != nil { // TODO
 		return nil, err
 	}
 
@@ -1071,82 +975,7 @@ func (sma StorageMinerActor) SubmitElectionPoSt(act *types.Actor, vmctx types.VM
 	return nil, nil
 }
 
-func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError {
+func onSuccessfulPoSt(self *StorageMinerActorState, vmctx types.VMContext, activeFaults uint64) aerrors.ActorError {
-	if vmctx.BlockHeight() >= build.ForkNoPowerEPSUpdates {
-		return onSuccessfulPoStV1(self, vmctx)
-	}
-
-	return onSuccessfulPoStV0(self, vmctx)
-}
-
-func onSuccessfulPoStV0(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError {
-	var mi MinerInfo
-	if err := vmctx.Storage().Get(self.Info, &mi); err != nil {
-		return err
-	}
-
-	pss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.ProvingSet)
-	if nerr != nil {
-		return aerrors.HandleExternalError(nerr, "failed to load proving set")
-	}
-
-	faults, nerr := self.FaultSet.All()
-	if nerr != nil {
-		return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)")
-	}
-
-	self.FaultSet = types.NewBitField()
-
-	oldPower := self.Power
-	newPower := types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), types.NewInt(mi.SectorSize))
-
-	// If below the minimum size requirement, miners have zero power
-	if newPower.LessThan(types.NewInt(build.MinimumMinerPower)) {
-		newPower = types.NewInt(0)
-	}
-
-	self.Power = newPower
-
-	delta := types.BigSub(self.Power, oldPower)
-	if self.SlashedAt != 0 {
-		self.SlashedAt = 0
-		delta = self.Power
-	}
-
-	prevSlashingDeadline := self.ElectionPeriodStart + build.SlashablePowerDelay
-	if !self.Active && newPower.GreaterThan(types.NewInt(0)) {
-		self.Active = true
-		prevSlashingDeadline = 0
-	}
-
-	if !(oldPower.IsZero() && newPower.IsZero()) {
-		enc, err := SerializeParams(&UpdateStorageParams{
-			Delta:                 delta,
-			NextSlashDeadline:     vmctx.BlockHeight() + build.SlashablePowerDelay,
-			PreviousSlashDeadline: prevSlashingDeadline,
-		})
-		if err != nil {
-			return err
-		}
-
-		_, err = vmctx.Send(StoragePowerAddress, SPAMethods.UpdateStorage, types.NewInt(0), enc)
-		if err != nil {
-			return aerrors.Wrap(err, "updating storage failed")
-		}
-	}
-
-	ncid, err := RemoveFromSectorSet(vmctx.Context(), vmctx.Storage(), self.Sectors, faults)
-	if err != nil {
-		return err
-	}
-
-	self.Sectors = ncid
-	self.ProvingSet = ncid
-	self.ElectionPeriodStart = vmctx.BlockHeight()
-	return nil
-}
-
-func onSuccessfulPoStV1(self *StorageMinerActorState, vmctx types.VMContext) aerrors.ActorError {
 	// TODO: some sector upkeep stuff that is very haphazard and unclear in the spec
 
 	var mi MinerInfo
@@ -1159,17 +988,12 @@ func onSuccessfulPoStV1(self *StorageMinerActorState, vmctx types.VMContext) aer
 		return aerrors.HandleExternalError(nerr, "failed to load proving set")
 	}
 
-	{
+	ss, nerr := amt.LoadAMT(types.WrapStorage(vmctx.Storage()), self.Sectors)
-		c, nerr := self.FaultSet.Count()
+	if nerr != nil {
-		if nerr != nil {
+		return aerrors.HandleExternalError(nerr, "failed to load sector set")
-			return aerrors.Absorb(nerr, 2, "invalid bitfield")
-		}
-		if c > RLEMax {
-			return aerrors.Newf(3, "too many items in bitfield: %d", c)
-		}
 	}
 
-	faults, nerr := self.FaultSet.All()
+	faults, nerr := self.FaultSet.All(2 * ss.Count)
 	if nerr != nil {
 		return aerrors.Absorb(nerr, 1, "invalid bitfield (fatal?)")
 	}
@@ -1177,7 +1001,7 @@ func onSuccessfulPoStV1(self *StorageMinerActorState, vmctx types.VMContext) aer
 	self.FaultSet = types.NewBitField()
 
 	oldPower := self.Power
-	newPower := types.BigMul(types.NewInt(pss.Count-uint64(len(faults))), types.NewInt(mi.SectorSize))
+	newPower := types.BigMul(types.NewInt(pss.Count-activeFaults), types.NewInt(mi.SectorSize))
 
 	// If below the minimum size requirement, miners have zero power
 	if newPower.LessThan(types.NewInt(build.MinimumMinerPower)) {

chain/actors/actor_miner_test.go

@@ -3,18 +3,22 @@ package actors_test
 import (
 	"bytes"
 	"context"
+	"math"
 	"math/rand"
 	"testing"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-sectorbuilder"
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/aerrors"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/lib/rlepluslazy"
 	hamt "github.com/ipfs/go-hamt-ipld"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	"github.com/stretchr/testify/assert"
 	cbg "github.com/whyrusleeping/cbor-gen"
 )
 
@@ -50,23 +54,37 @@ func TestMinerCommitSectors(t *testing.T) {
 	addSectorToMiner(h, t, minerAddr, worker, client, 1)
 
 	assertSectorIDs(h, t, minerAddr, []uint64{1})
+
 }
 
+type badRuns struct {
+	done bool
+}
+
+func (br *badRuns) HasNext() bool {
+	return !br.done
+}
+
+func (br *badRuns) NextRun() (rlepluslazy.Run, error) {
+	br.done = true
+	return rlepluslazy.Run{true, math.MaxInt64}, nil
+}
+
+var _ rlepluslazy.RunIterator = (*badRuns)(nil)
+
 func TestMinerSubmitBadFault(t *testing.T) {
+	oldSS, oldMin := build.SectorSizes, build.MinimumMinerPower
+	build.SectorSizes, build.MinimumMinerPower = []uint64{1024}, 1024
+	defer func() {
+		build.SectorSizes, build.MinimumMinerPower = oldSS, oldMin
+	}()
+
 	var worker, client address.Address
 	var minerAddr address.Address
 	opts := []HarnessOpt{
 		HarnessAddr(&worker, 1000000),
 		HarnessAddr(&client, 1000000),
-		HarnessActor(&minerAddr, &worker, actors.StorageMinerCodeCid,
+		HarnessAddMiner(&minerAddr, &worker),
-			func() cbg.CBORMarshaler {
-				return &actors.StorageMinerConstructorParams{
-					Owner:      worker,
-					Worker:     worker,
-					SectorSize: 1024,
-					PeerID:     "fakepeerid",
-				}
-			}),
 	}
 
 	h := NewHarness(t, opts...)
@@ -92,18 +110,52 @@ func TestMinerSubmitBadFault(t *testing.T) {
 
 	ret, _ = h.Invoke(t, actors.NetworkAddress, minerAddr, actors.MAMethods.SubmitElectionPoSt, nil)
 	ApplyOK(t, ret)
-
 	assertSectorIDs(h, t, minerAddr, []uint64{1})
 
+	st, err := getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr)
+	assert.NoError(t, err)
+	expectedPower := st.Power
+	if types.BigCmp(expectedPower, types.NewInt(1024)) != 0 {
+		t.Errorf("Expected power of 1024, got %s", expectedPower)
+	}
+
 	badnum := uint64(0)
 	badnum--
 	bf = types.NewBitField()
 	bf.Set(badnum)
+	bf.Set(badnum - 1)
 	ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf})
 	ApplyOK(t, ret)
 
 	ret, _ = h.Invoke(t, actors.NetworkAddress, minerAddr, actors.MAMethods.SubmitElectionPoSt, nil)
+
 	ApplyOK(t, ret)
+	assertSectorIDs(h, t, minerAddr, []uint64{1})
+
+	st, err = getMinerState(context.TODO(), h.vm.StateTree(), h.bs, minerAddr)
+	assert.NoError(t, err)
+	currentPower := st.Power
+	if types.BigCmp(expectedPower, currentPower) != 0 {
+		t.Errorf("power changed and shouldn't have: %s != %s", expectedPower, currentPower)
+	}
+
+	bf.Set(badnum - 2)
+	ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf})
+	if ret.ExitCode != 3 {
+		t.Errorf("expected exit code 3, got %d: %+v", ret.ExitCode, ret.ActorErr)
+	}
+	assertSectorIDs(h, t, minerAddr, []uint64{1})
+
+	rle, err := rlepluslazy.EncodeRuns(&badRuns{}, []byte{})
+	assert.NoError(t, err)
+
+	bf, err = types.NewBitFieldFromBytes(rle)
+	assert.NoError(t, err)
+	ret, _ = h.Invoke(t, worker, minerAddr, actors.MAMethods.DeclareFaults, &actors.DeclareFaultsParams{bf})
+	if ret.ExitCode != 3 {
+		t.Errorf("expected exit code 3, got %d: %+v", ret.ExitCode, ret.ActorErr)
+	}
+	assertSectorIDs(h, t, minerAddr, []uint64{1})
 
 	bf = types.NewBitField()
 	bf.Set(1)
@@ -175,7 +227,7 @@ func assertSectorIDs(h *Harness, t *testing.T, maddr address.Address, ids []uint
 	}
 }
 
-func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) {
+func getMinerState(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) (*actors.StorageMinerActorState, error) {
 	mact, err := st.GetActor(maddr)
 	if err != nil {
 		return nil, err
@@ -187,6 +239,14 @@ func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Bl
 	if err := cst.Get(ctx, mact.Head, &mstate); err != nil {
 		return nil, err
 	}
+	return &mstate, nil
+}
+
+func getMinerSectorSet(ctx context.Context, st types.StateTree, bs blockstore.Blockstore, maddr address.Address) ([]*api.ChainSectorInfo, error) {
+	mstate, err := getMinerState(ctx, st, bs, maddr)
+	if err != nil {
+		return nil, err
+	}
 
 	return stmgr.LoadSectorsFromSet(ctx, bs, mstate.Sectors)
 }
@@ -202,7 +262,6 @@ func (h *Harness) makeFakeDeal(t *testing.T, miner, worker, client address.Addre
 	prop := actors.StorageDealProposal{
 		PieceRef:  commP[:],
 		PieceSize: size,
-		//PieceSerialization SerializationMode // Needs to be here as it tells how data in the sector maps to PieceRef cid
 
 		Client:   client,
 		Provider: miner,

chain/actors/actor_paych.go

@@ -116,7 +116,7 @@ func (pca PaymentChannelActor) UpdateChannelState(act *types.Actor, vmctx types.
 
 	vb, nerr := sv.SigningBytes()
 	if nerr != nil {
-		return nil, aerrors.Escalate(nerr, "failed to serialize signedvoucher")
+		return nil, aerrors.Absorb(nerr, 1, "failed to serialize signedvoucher")
 	}
 
 	if err := vmctx.VerifySignature(sv.Signature, self.From, vb); err != nil {

chain/actors/actor_storagemarket.go

@@ -122,15 +122,21 @@ func (sdp *StorageDealProposal) Cid() (cid.Cid, error) {
 	return nd.Cid(), nil
 }
 
-func (sdp *StorageDealProposal) Verify() error {
+func (sdp *StorageDealProposal) Verify(worker address.Address) error {
-	unsigned := *sdp
+	if sdp.Client != worker || worker == address.Undef {
-	unsigned.ProposerSignature = nil
+		unsigned := *sdp
-	var buf bytes.Buffer
+		unsigned.ProposerSignature = nil
-	if err := unsigned.MarshalCBOR(&buf); err != nil {
+		var buf bytes.Buffer
-		return err
+		if err := unsigned.MarshalCBOR(&buf); err != nil {
+			return err
+		}
+
+		if err := sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes()); err != nil {
+			return err
+		}
 	}
 
-	return sdp.ProposerSignature.Verify(sdp.Client, buf.Bytes())
+	return nil
 }
 
 type OnChainDeal struct {
@@ -396,7 +402,7 @@ func (st *StorageMarketState) validateDeal(vmctx types.VMContext, deal StorageDe
 		return aerrors.New(2, "Deals must be submitted by the miner worker")
 	}
 
-	if err := deal.Verify(); err != nil {
+	if err := deal.Verify(providerWorker); err != nil {
 		return aerrors.Absorb(err, 3, "verifying proposer signature")
 	}
 

chain/actors/actor_storagepower.go

@@ -67,7 +67,7 @@ type CreateStorageMinerParams struct {
 
 func (spa StoragePowerActor) CreateStorageMiner(act *types.Actor, vmctx types.VMContext, params *CreateStorageMinerParams) ([]byte, ActorError) {
 	if !build.SupportedSectorSize(params.SectorSize) {
-		return nil, aerrors.New(1, "Unsupported sector size")
+		return nil, aerrors.Newf(1, "Unsupported sector size: %d", params.SectorSize)
 	}
 
 	var self StoragePowerState

chain/actors/actors_test.go

@@ -50,10 +50,10 @@ func setupVMTestEnv(t *testing.T) (*vm.VM, []address.Address, bstore.Blockstore)
 		t.Fatal(err)
 	}
 
-	cs := store.NewChainStore(bs, nil)
+	cs := store.NewChainStore(bs, nil, nil)
 
 	// TODO: should probabaly mock out the randomness bit, nil works for now
-	vm, err := vm.NewVM(stateroot, 1, nil, maddr, cs.Blockstore())
+	vm, err := vm.NewVM(stateroot, 1, nil, maddr, cs.Blockstore(), cs.VMSys())
 	if err != nil {
 		t.Fatal(err)
 	}

chain/actors/aerrors/wrap.go

@@ -171,7 +171,8 @@ func HandleExternalError(err error, msg string) ActorError {
 	}
 
 	return &actorError{
-		fatal: true,
+		fatal:   false,
+		retCode: 219,
 
 		msg:   msg,
 		frame: xerrors.Caller(1),

chain/actors/harness2_test.go

@@ -3,6 +3,7 @@ package actors_test
 import (
 	"bytes"
 	"context"
+	"github.com/filecoin-project/go-sectorbuilder"
 	"math/rand"
 	"testing"
 
@@ -115,6 +116,32 @@ func HarnessActor(actor *address.Address, creator *address.Address, code cid.Cid
 
 }
 
+func HarnessAddMiner(addr *address.Address, creator *address.Address) HarnessOpt {
+	return func(t testing.TB, h *Harness) error {
+		if h.Stage != HarnessPostInit {
+			return nil
+		}
+		if !addr.Empty() {
+			return xerrors.New("actor address should be empty")
+		}
+		ret, _ := h.InvokeWithValue(t, *creator, actors.StoragePowerAddress,
+			actors.SPAMethods.CreateStorageMiner, types.NewInt(3000), &actors.StorageMinerConstructorParams{
+				Owner:      *creator,
+				Worker:     *creator,
+				SectorSize: 1024,
+				PeerID:     "fakepeerid",
+			})
+
+		if ret.ExitCode != 0 {
+			return xerrors.Errorf("creating actor: %w", ret.ActorErr)
+		}
+		var err error
+		*addr, err = address.NewFromBytes(ret.Return)
+		return err
+
+	}
+}
+
 func HarnessCtx(ctx context.Context) HarnessOpt {
 	return func(t testing.TB, h *Harness) error {
 		h.ctx = ctx
@@ -168,8 +195,8 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness {
 		t.Fatal(err)
 	}
 
-	h.cs = store.NewChainStore(h.bs, nil)
+	h.cs = store.NewChainStore(h.bs, nil, vm.Syscalls(sectorbuilder.ProofVerifier))
-	h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore())
+	h.vm, err = vm.NewVM(stateroot, 1, h.Rand, h.HI.Miner, h.cs.Blockstore(), h.cs.VMSys())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -177,7 +204,7 @@ func NewHarness(t *testing.T, options ...HarnessOpt) *Harness {
 	for _, opt := range options {
 		err := opt(t, h)
 		if err != nil {
-			t.Fatalf("Applying options: %v", err)
+			t.Fatalf("Applying options: %+v", err)
 		}
 	}
 

chain/blocksync/blocksync.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	inet "github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 )

chain/blocksync/cbor_gen.go

@@ -5,7 +5,7 @@ import (
 	"io"
 
 	"github.com/filecoin-project/lotus/chain/types"
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
 )

chain/deals/cbor_gen.go

@@ -1,883 +0,0 @@
-package deals
-
-import (
-	"fmt"
-	"io"
-
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/libp2p/go-libp2p-core/peer"
-	cbg "github.com/whyrusleeping/cbor-gen"
-	xerrors "golang.org/x/xerrors"
-)
-
-// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
-
-var _ = xerrors.Errorf
-
-func (t *AskRequest) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{129}); err != nil {
-		return err
-	}
-
-	// t.Miner (address.Address) (struct)
-	if err := t.Miner.MarshalCBOR(w); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *AskRequest) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 1 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Miner (address.Address) (struct)
-
-	{
-
-		if err := t.Miner.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	return nil
-}
-
-func (t *AskResponse) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{129}); err != nil {
-		return err
-	}
-
-	// t.Ask (types.SignedStorageAsk) (struct)
-	if err := t.Ask.MarshalCBOR(w); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *AskResponse) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 1 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Ask (types.SignedStorageAsk) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-			t.Ask = new(types.SignedStorageAsk)
-			if err := t.Ask.UnmarshalCBOR(br); err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
-func (t *Proposal) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{130}); err != nil {
-		return err
-	}
-
-	// t.DealProposal (actors.StorageDealProposal) (struct)
-	if err := t.DealProposal.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.Piece (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.Piece); err != nil {
-		return xerrors.Errorf("failed to write cid field t.Piece: %w", err)
-	}
-
-	return nil
-}
-
-func (t *Proposal) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 2 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.DealProposal (actors.StorageDealProposal) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-			t.DealProposal = new(actors.StorageDealProposal)
-			if err := t.DealProposal.UnmarshalCBOR(br); err != nil {
-				return err
-			}
-		}
-
-	}
-	// t.Piece (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.Piece: %w", err)
-		}
-
-		t.Piece = c
-
-	}
-	return nil
-}
-
-func (t *Response) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{132}); err != nil {
-		return err
-	}
-
-	// t.State (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
-		return err
-	}
-
-	// t.Message (string) (string)
-	if len(t.Message) > cbg.MaxLength {
-		return xerrors.Errorf("Value in field t.Message was too long")
-	}
-
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Message)))); err != nil {
-		return err
-	}
-	if _, err := w.Write([]byte(t.Message)); err != nil {
-		return err
-	}
-
-	// t.Proposal (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.Proposal); err != nil {
-		return xerrors.Errorf("failed to write cid field t.Proposal: %w", err)
-	}
-
-	// t.StorageDealSubmission (types.SignedMessage) (struct)
-	if err := t.StorageDealSubmission.MarshalCBOR(w); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *Response) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 4 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.State (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.State = uint64(extra)
-	// t.Message (string) (string)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		t.Message = string(sval)
-	}
-	// t.Proposal (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.Proposal: %w", err)
-		}
-
-		t.Proposal = c
-
-	}
-	// t.StorageDealSubmission (types.SignedMessage) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-			t.StorageDealSubmission = new(types.SignedMessage)
-			if err := t.StorageDealSubmission.UnmarshalCBOR(br); err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
-func (t *SignedResponse) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{130}); err != nil {
-		return err
-	}
-
-	// t.Response (deals.Response) (struct)
-	if err := t.Response.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.Signature (types.Signature) (struct)
-	if err := t.Signature.MarshalCBOR(w); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *SignedResponse) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 2 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Response (deals.Response) (struct)
-
-	{
-
-		if err := t.Response.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.Signature (types.Signature) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-			t.Signature = new(types.Signature)
-			if err := t.Signature.UnmarshalCBOR(br); err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
-func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{136}); err != nil {
-		return err
-	}
-
-	// t.Data (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.Data); err != nil {
-		return xerrors.Errorf("failed to write cid field t.Data: %w", err)
-	}
-
-	// t.PricePerEpoch (types.BigInt) (struct)
-	if err := t.PricePerEpoch.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.ProposalExpiration (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ProposalExpiration))); err != nil {
-		return err
-	}
-
-	// t.Duration (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.Duration))); err != nil {
-		return err
-	}
-
-	// t.ProviderAddress (address.Address) (struct)
-	if err := t.ProviderAddress.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.Client (address.Address) (struct)
-	if err := t.Client.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.MinerWorker (address.Address) (struct)
-	if err := t.MinerWorker.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.MinerID (peer.ID) (string)
-	if len(t.MinerID) > cbg.MaxLength {
-		return xerrors.Errorf("Value in field t.MinerID was too long")
-	}
-
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.MinerID)))); err != nil {
-		return err
-	}
-	if _, err := w.Write([]byte(t.MinerID)); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 8 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Data (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.Data: %w", err)
-		}
-
-		t.Data = c
-
-	}
-	// t.PricePerEpoch (types.BigInt) (struct)
-
-	{
-
-		if err := t.PricePerEpoch.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.ProposalExpiration (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.ProposalExpiration = uint64(extra)
-	// t.Duration (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.Duration = uint64(extra)
-	// t.ProviderAddress (address.Address) (struct)
-
-	{
-
-		if err := t.ProviderAddress.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.Client (address.Address) (struct)
-
-	{
-
-		if err := t.Client.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.MinerWorker (address.Address) (struct)
-
-	{
-
-		if err := t.MinerWorker.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.MinerID (peer.ID) (string)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		t.MinerID = peer.ID(sval)
-	}
-	return nil
-}
-
-func (t *ClientDeal) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{135}); err != nil {
-		return err
-	}
-
-	// t.ProposalCid (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.ProposalCid); err != nil {
-		return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err)
-	}
-
-	// t.Proposal (actors.StorageDealProposal) (struct)
-	if err := t.Proposal.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.State (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
-		return err
-	}
-
-	// t.Miner (peer.ID) (string)
-	if len(t.Miner) > cbg.MaxLength {
-		return xerrors.Errorf("Value in field t.Miner was too long")
-	}
-
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Miner)))); err != nil {
-		return err
-	}
-	if _, err := w.Write([]byte(t.Miner)); err != nil {
-		return err
-	}
-
-	// t.MinerWorker (address.Address) (struct)
-	if err := t.MinerWorker.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.DealID (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
-		return err
-	}
-
-	// t.PublishMessage (types.SignedMessage) (struct)
-	if err := t.PublishMessage.MarshalCBOR(w); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 7 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.ProposalCid (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err)
-		}
-
-		t.ProposalCid = c
-
-	}
-	// t.Proposal (actors.StorageDealProposal) (struct)
-
-	{
-
-		if err := t.Proposal.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.State (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.State = uint64(extra)
-	// t.Miner (peer.ID) (string)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		t.Miner = peer.ID(sval)
-	}
-	// t.MinerWorker (address.Address) (struct)
-
-	{
-
-		if err := t.MinerWorker.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.DealID (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.DealID = uint64(extra)
-	// t.PublishMessage (types.SignedMessage) (struct)
-
-	{
-
-		pb, err := br.PeekByte()
-		if err != nil {
-			return err
-		}
-		if pb == cbg.CborNull[0] {
-			var nbuf [1]byte
-			if _, err := br.Read(nbuf[:]); err != nil {
-				return err
-			}
-		} else {
-			t.PublishMessage = new(types.SignedMessage)
-			if err := t.PublishMessage.UnmarshalCBOR(br); err != nil {
-				return err
-			}
-		}
-
-	}
-	return nil
-}
-
-func (t *MinerDeal) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{135}); err != nil {
-		return err
-	}
-
-	// t.Client (peer.ID) (string)
-	if len(t.Client) > cbg.MaxLength {
-		return xerrors.Errorf("Value in field t.Client was too long")
-	}
-
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Client)))); err != nil {
-		return err
-	}
-	if _, err := w.Write([]byte(t.Client)); err != nil {
-		return err
-	}
-
-	// t.Proposal (actors.StorageDealProposal) (struct)
-	if err := t.Proposal.MarshalCBOR(w); err != nil {
-		return err
-	}
-
-	// t.ProposalCid (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.ProposalCid); err != nil {
-		return xerrors.Errorf("failed to write cid field t.ProposalCid: %w", err)
-	}
-
-	// t.State (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.State))); err != nil {
-		return err
-	}
-
-	// t.Ref (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.Ref); err != nil {
-		return xerrors.Errorf("failed to write cid field t.Ref: %w", err)
-	}
-
-	// t.DealID (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
-		return err
-	}
-
-	// t.SectorID (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.SectorID))); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 7 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Client (peer.ID) (string)
-
-	{
-		sval, err := cbg.ReadString(br)
-		if err != nil {
-			return err
-		}
-
-		t.Client = peer.ID(sval)
-	}
-	// t.Proposal (actors.StorageDealProposal) (struct)
-
-	{
-
-		if err := t.Proposal.UnmarshalCBOR(br); err != nil {
-			return err
-		}
-
-	}
-	// t.ProposalCid (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.ProposalCid: %w", err)
-		}
-
-		t.ProposalCid = c
-
-	}
-	// t.State (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.State = uint64(extra)
-	// t.Ref (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.Ref: %w", err)
-		}
-
-		t.Ref = c
-
-	}
-	// t.DealID (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.DealID = uint64(extra)
-	// t.SectorID (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.SectorID = uint64(extra)
-	return nil
-}
-
-func (t *StorageDataTransferVoucher) MarshalCBOR(w io.Writer) error {
-	if t == nil {
-		_, err := w.Write(cbg.CborNull)
-		return err
-	}
-	if _, err := w.Write([]byte{130}); err != nil {
-		return err
-	}
-
-	// t.Proposal (cid.Cid) (struct)
-
-	if err := cbg.WriteCid(w, t.Proposal); err != nil {
-		return xerrors.Errorf("failed to write cid field t.Proposal: %w", err)
-	}
-
-	// t.DealID (uint64) (uint64)
-	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (t *StorageDataTransferVoucher) UnmarshalCBOR(r io.Reader) error {
-	br := cbg.GetPeeker(r)
-
-	maj, extra, err := cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajArray {
-		return fmt.Errorf("cbor input should be of type array")
-	}
-
-	if extra != 2 {
-		return fmt.Errorf("cbor input had wrong number of fields")
-	}
-
-	// t.Proposal (cid.Cid) (struct)
-
-	{
-
-		c, err := cbg.ReadCid(br)
-		if err != nil {
-			return xerrors.Errorf("failed to read cid field t.Proposal: %w", err)
-		}
-
-		t.Proposal = c
-
-	}
-	// t.DealID (uint64) (uint64)
-
-	maj, extra, err = cbg.CborReadHeader(br)
-	if err != nil {
-		return err
-	}
-	if maj != cbg.MajUnsignedInt {
-		return fmt.Errorf("wrong type for uint64 field")
-	}
-	t.DealID = uint64(extra)
-	return nil
-}

chain/deals/client.go

@@ -1,311 +0,0 @@
-package deals
-
-import (
-	"context"
-
-	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
-	"github.com/libp2p/go-libp2p-core/host"
-	inet "github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/go-statestore"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/events"
-	"github.com/filecoin-project/lotus/chain/market"
-	"github.com/filecoin-project/lotus/chain/stmgr"
-	"github.com/filecoin-project/lotus/chain/store"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/chain/wallet"
-	"github.com/filecoin-project/lotus/node/impl/full"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/filecoin-project/lotus/retrieval/discovery"
-)
-
-var log = logging.Logger("deals")
-
-type ClientDeal struct {
-	ProposalCid cid.Cid
-	Proposal    actors.StorageDealProposal
-	State       api.DealState
-	Miner       peer.ID
-	MinerWorker address.Address
-	DealID      uint64
-
-	PublishMessage *types.SignedMessage
-
-	s inet.Stream
-}
-
-type Client struct {
-	sm    *stmgr.StateManager
-	chain *store.ChainStore
-	h     host.Host
-	w     *wallet.Wallet
-	// dataTransfer
-	// TODO: once the data transfer module is complete, the
-	// client will listen to events on the data transfer module
-	// Because we are using only a fake DAGService
-	// implementation, there's no validation or events on the client side
-	dataTransfer dtypes.ClientDataTransfer
-	dag          dtypes.ClientDAG
-	discovery    *discovery.Local
-	events       *events.Events
-	fm           *market.FundMgr
-
-	deals *statestore.StateStore
-	conns map[cid.Cid]inet.Stream
-
-	incoming chan *ClientDeal
-	updated  chan clientDealUpdate
-
-	stop    chan struct{}
-	stopped chan struct{}
-}
-
-type clientDealUpdate struct {
-	newState api.DealState
-	id       cid.Cid
-	err      error
-	mut      func(*ClientDeal)
-}
-
-type clientApi struct {
-	full.ChainAPI
-	full.StateAPI
-}
-
-func NewClient(sm *stmgr.StateManager, chain *store.ChainStore, h host.Host, w *wallet.Wallet, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, fm *market.FundMgr, deals dtypes.ClientDealStore, chainapi full.ChainAPI, stateapi full.StateAPI) *Client {
-	c := &Client{
-		sm:           sm,
-		chain:        chain,
-		h:            h,
-		w:            w,
-		dataTransfer: dataTransfer,
-		dag:          dag,
-		discovery:    discovery,
-		fm:           fm,
-		events:       events.NewEvents(context.TODO(), &clientApi{chainapi, stateapi}),
-
-		deals: deals,
-		conns: map[cid.Cid]inet.Stream{},
-
-		incoming: make(chan *ClientDeal, 16),
-		updated:  make(chan clientDealUpdate, 16),
-
-		stop:    make(chan struct{}),
-		stopped: make(chan struct{}),
-	}
-
-	return c
-}
-
-func (c *Client) Run(ctx context.Context) {
-	go func() {
-		defer close(c.stopped)
-
-		for {
-			select {
-			case deal := <-c.incoming:
-				c.onIncoming(deal)
-			case update := <-c.updated:
-				c.onUpdated(ctx, update)
-			case <-c.stop:
-				return
-			}
-		}
-	}()
-}
-
-func (c *Client) onIncoming(deal *ClientDeal) {
-	log.Info("incoming deal")
-
-	if _, ok := c.conns[deal.ProposalCid]; ok {
-		log.Errorf("tracking deal connection: already tracking connection for deal %s", deal.ProposalCid)
-		return
-	}
-	c.conns[deal.ProposalCid] = deal.s
-
-	if err := c.deals.Begin(deal.ProposalCid, deal); err != nil {
-		// We may have re-sent the proposal
-		log.Errorf("deal tracking failed: %s", err)
-		c.failDeal(deal.ProposalCid, err)
-		return
-	}
-
-	go func() {
-		c.updated <- clientDealUpdate{
-			newState: api.DealUnknown,
-			id:       deal.ProposalCid,
-			err:      nil,
-		}
-	}()
-}
-
-func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) {
-	log.Infof("Client deal %s updated state to %s", update.id, api.DealStates[update.newState])
-	var deal ClientDeal
-	err := c.deals.Mutate(update.id, func(d *ClientDeal) error {
-		d.State = update.newState
-		if update.mut != nil {
-			update.mut(d)
-		}
-		deal = *d
-		return nil
-	})
-	if update.err != nil {
-		log.Errorf("deal %s failed: %s", update.id, update.err)
-		c.failDeal(update.id, update.err)
-		return
-	}
-	if err != nil {
-		c.failDeal(update.id, err)
-		return
-	}
-
-	switch update.newState {
-	case api.DealUnknown: // new
-		c.handle(ctx, deal, c.new, api.DealAccepted)
-	case api.DealAccepted:
-		c.handle(ctx, deal, c.accepted, api.DealStaged)
-	case api.DealStaged:
-		c.handle(ctx, deal, c.staged, api.DealSealing)
-	case api.DealSealing:
-		c.handle(ctx, deal, c.sealing, api.DealNoUpdate)
-		// TODO: DealComplete -> watch for faults, expiration, etc.
-	}
-}
-
-type ClientDealProposal struct {
-	Data cid.Cid
-
-	PricePerEpoch      types.BigInt
-	ProposalExpiration uint64
-	Duration           uint64
-
-	ProviderAddress address.Address
-	Client          address.Address
-	MinerWorker     address.Address
-	MinerID         peer.ID
-}
-
-func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, error) {
-	if err := c.fm.EnsureAvailable(ctx, p.Client, types.BigMul(p.PricePerEpoch, types.NewInt(p.Duration))); err != nil {
-		return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err)
-	}
-
-	commP, pieceSize, err := c.commP(ctx, p.Data)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("computing commP failed: %w", err)
-	}
-
-	dealProposal := &actors.StorageDealProposal{
-		PieceRef:             commP,
-		PieceSize:            uint64(pieceSize),
-		Client:               p.Client,
-		Provider:             p.ProviderAddress,
-		ProposalExpiration:   p.ProposalExpiration,
-		Duration:             p.Duration,
-		StoragePricePerEpoch: p.PricePerEpoch,
-		StorageCollateral:    types.NewInt(uint64(pieceSize)), // TODO: real calc
-	}
-
-	if err := api.SignWith(ctx, c.w.Sign, p.Client, dealProposal); err != nil {
-		return cid.Undef, xerrors.Errorf("signing deal proposal failed: %w", err)
-	}
-
-	proposalNd, err := cborutil.AsIpld(dealProposal)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("getting proposal node failed: %w", err)
-	}
-
-	s, err := c.h.NewStream(ctx, p.MinerID, DealProtocolID)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("connecting to storage provider failed: %w", err)
-	}
-
-	proposal := &Proposal{
-		DealProposal: dealProposal,
-		Piece:        p.Data,
-	}
-
-	if err := cborutil.WriteCborRPC(s, proposal); err != nil {
-		s.Reset()
-		return cid.Undef, xerrors.Errorf("sending proposal to storage provider failed: %w", err)
-	}
-
-	deal := &ClientDeal{
-		ProposalCid: proposalNd.Cid(),
-		Proposal:    *dealProposal,
-		State:       api.DealUnknown,
-		Miner:       p.MinerID,
-		MinerWorker: p.MinerWorker,
-
-		s: s,
-	}
-
-	c.incoming <- deal
-
-	return deal.ProposalCid, c.discovery.AddPeer(p.Data, discovery.RetrievalPeer{
-		Address: dealProposal.Provider,
-		ID:      deal.Miner,
-	})
-}
-
-func (c *Client) QueryAsk(ctx context.Context, p peer.ID, a address.Address) (*types.SignedStorageAsk, error) {
-	s, err := c.h.NewStream(ctx, p, AskProtocolID)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to open stream to miner: %w", err)
-	}
-
-	req := &AskRequest{
-		Miner: a,
-	}
-	if err := cborutil.WriteCborRPC(s, req); err != nil {
-		return nil, xerrors.Errorf("failed to send ask request: %w", err)
-	}
-
-	var out AskResponse
-	if err := cborutil.ReadCborRPC(s, &out); err != nil {
-		return nil, xerrors.Errorf("failed to read ask response: %w", err)
-	}
-
-	if out.Ask == nil {
-		return nil, xerrors.Errorf("got no ask back")
-	}
-
-	if out.Ask.Ask.Miner != a {
-		return nil, xerrors.Errorf("got back ask for wrong miner")
-	}
-
-	if err := c.checkAskSignature(out.Ask); err != nil {
-		return nil, xerrors.Errorf("ask was not properly signed")
-	}
-
-	return out.Ask, nil
-}
-
-func (c *Client) List() ([]ClientDeal, error) {
-	var out []ClientDeal
-	if err := c.deals.List(&out); err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *Client) GetDeal(d cid.Cid) (*ClientDeal, error) {
-	var out ClientDeal
-	if err := c.deals.Get(d, &out); err != nil {
-		return nil, err
-	}
-	return &out, nil
-}
-
-func (c *Client) Stop() {
-	close(c.stop)
-	<-c.stopped
-}

chain/deals/client_states.go

@@ -1,233 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"context"
-
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/stmgr"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-type clientHandlerFunc func(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error)
-
-func (c *Client) handle(ctx context.Context, deal ClientDeal, cb clientHandlerFunc, next api.DealState) {
-	go func() {
-		mut, err := cb(ctx, deal)
-		if err != nil {
-			next = api.DealError
-		}
-
-		if err == nil && next == api.DealNoUpdate {
-			return
-		}
-
-		select {
-		case c.updated <- clientDealUpdate{
-			newState: next,
-			id:       deal.ProposalCid,
-			err:      err,
-			mut:      mut,
-		}:
-		case <-c.stop:
-		}
-	}()
-}
-
-func (c *Client) new(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
-	resp, err := c.readStorageDealResp(deal)
-	if err != nil {
-		return nil, err
-	}
-
-	// TODO: verify StorageDealSubmission
-
-	if err := c.disconnect(deal); err != nil {
-		return nil, err
-	}
-
-	/* data transfer happens */
-	if resp.State != api.DealAccepted {
-		return nil, xerrors.Errorf("deal wasn't accepted (State=%d)", resp.State)
-	}
-
-	return func(info *ClientDeal) {
-		info.PublishMessage = resp.StorageDealSubmission
-	}, nil
-}
-
-func (c *Client) accepted(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
-	log.Infow("DEAL ACCEPTED!")
-
-	pubmsg := deal.PublishMessage.Message
-	pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider)
-	if err != nil {
-		return nil, xerrors.Errorf("getting miner worker failed: %w", err)
-	}
-
-	if pubmsg.From != pw {
-		return nil, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider)
-	}
-
-	if pubmsg.To != actors.StorageMarketAddress {
-		return nil, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To)
-	}
-
-	if pubmsg.Method != actors.SMAMethods.PublishStorageDeals {
-		return nil, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method)
-	}
-
-	var params actors.PublishStorageDealsParams
-	if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil {
-		return nil, err
-	}
-
-	dealIdx := -1
-	for i, storageDeal := range params.Deals {
-		// TODO: make it less hacky
-		sd := storageDeal
-		eq, err := cborutil.Equals(&deal.Proposal, &sd)
-		if err != nil {
-			return nil, err
-		}
-		if eq {
-			dealIdx = i
-			break
-		}
-	}
-
-	if dealIdx == -1 {
-		return nil, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage.Cid())
-	}
-
-	// TODO: timeout
-	_, ret, err := c.sm.WaitForMessage(ctx, deal.PublishMessage.Cid())
-	if err != nil {
-		return nil, xerrors.Errorf("waiting for deal publish message: %w", err)
-	}
-	if ret.ExitCode != 0 {
-		return nil, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode)
-	}
-
-	var res actors.PublishStorageDealResponse
-	if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil {
-		return nil, err
-	}
-
-	return func(info *ClientDeal) {
-		info.DealID = res.DealIDs[dealIdx]
-	}, nil
-}
-
-func (c *Client) staged(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
-	// TODO: Maybe wait for pre-commit
-
-	return nil, nil
-}
-
-func (c *Client) sealing(ctx context.Context, deal ClientDeal) (func(*ClientDeal), error) {
-	checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) {
-		sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts)
-		if err != nil {
-			// TODO: This may be fine for some errors
-			return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err)
-		}
-
-		if sd.ActivationEpoch > 0 {
-			select {
-			case c.updated <- clientDealUpdate{
-				newState: api.DealComplete,
-				id:       deal.ProposalCid,
-			}:
-			case <-c.stop:
-			}
-
-			return true, false, nil
-		}
-
-		return false, true, nil
-	}
-
-	called := func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) {
-		defer func() {
-			if err != nil {
-				select {
-				case c.updated <- clientDealUpdate{
-					newState: api.DealComplete,
-					id:       deal.ProposalCid,
-					err:      xerrors.Errorf("handling applied event: %w", err),
-				}:
-				case <-c.stop:
-				}
-			}
-		}()
-
-		if msg == nil {
-			log.Error("timed out waiting for deal activation... what now?")
-			return false, nil
-		}
-
-		sd, err := stmgr.GetStorageDeal(ctx, c.sm, deal.DealID, ts)
-		if err != nil {
-			return false, xerrors.Errorf("failed to look up deal on chain: %w", err)
-		}
-
-		if sd.ActivationEpoch == 0 {
-			return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", deal.DealID, ts.ParentState(), ts.Height())
-		}
-
-		log.Infof("Storage deal %d activated at epoch %d", deal.DealID, sd.ActivationEpoch)
-
-		select {
-		case c.updated <- clientDealUpdate{
-			newState: api.DealComplete,
-			id:       deal.ProposalCid,
-		}:
-		case <-c.stop:
-		}
-
-		return false, nil
-	}
-
-	revert := func(ctx context.Context, ts *types.TipSet) error {
-		log.Warn("deal activation reverted; TODO: actually handle this!")
-		// TODO: Just go back to DealSealing?
-		return nil
-	}
-
-	matchEvent := func(msg *types.Message) (bool, error) {
-		if msg.To != deal.Proposal.Provider {
-			return false, nil
-		}
-
-		if msg.Method != actors.MAMethods.ProveCommitSector {
-			return false, nil
-		}
-
-		var params actors.SectorProveCommitInfo
-		if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil {
-			return false, err
-		}
-
-		var found bool
-		for _, dealID := range params.DealIDs {
-			if dealID == deal.DealID {
-				found = true
-				break
-			}
-		}
-
-		return found, nil
-	}
-
-	if err := c.events.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, matchEvent); err != nil {
-		return nil, xerrors.Errorf("failed to set up called handler")
-	}
-
-	return nil, nil
-}

chain/deals/client_utils.go

@@ -1,171 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"context"
-	"runtime"
-
-	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
-	"github.com/ipfs/go-cid"
-	files "github.com/ipfs/go-ipfs-files"
-	unixfile "github.com/ipfs/go-unixfs/file"
-	"github.com/ipld/go-ipld-prime"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"golang.org/x/xerrors"
-
-	cborutil "github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/go-statestore"
-
-	"github.com/filecoin-project/lotus/datatransfer"
-	"github.com/filecoin-project/lotus/lib/padreader"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-)
-
-func (c *Client) failDeal(id cid.Cid, cerr error) {
-	if cerr == nil {
-		_, f, l, _ := runtime.Caller(1)
-		cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l)
-	}
-
-	s, ok := c.conns[id]
-	if ok {
-		_ = s.Reset()
-		delete(c.conns, id)
-	}
-
-	// TODO: store in some sort of audit log
-	log.Errorf("deal %s failed: %+v", id, cerr)
-}
-
-func (c *Client) commP(ctx context.Context, data cid.Cid) ([]byte, uint64, error) {
-	root, err := c.dag.Get(ctx, data)
-	if err != nil {
-		log.Errorf("failed to get file root for deal: %s", err)
-		return nil, 0, err
-	}
-
-	n, err := unixfile.NewUnixfsFile(ctx, c.dag, root)
-	if err != nil {
-		log.Errorf("cannot open unixfs file: %s", err)
-		return nil, 0, err
-	}
-
-	uf, ok := n.(files.File)
-	if !ok {
-		// TODO: we probably got directory, how should we handle this in unixfs mode?
-		return nil, 0, xerrors.New("unsupported unixfs type")
-	}
-
-	s, err := uf.Size()
-	if err != nil {
-		return nil, 0, err
-	}
-
-	pr, psize := padreader.New(uf, uint64(s))
-
-	commp, err := sectorbuilder.GeneratePieceCommitment(pr, psize)
-	if err != nil {
-		return nil, 0, xerrors.Errorf("generating CommP: %w", err)
-	}
-
-	return commp[:], psize, nil
-}
-
-func (c *Client) readStorageDealResp(deal ClientDeal) (*Response, error) {
-	s, ok := c.conns[deal.ProposalCid]
-	if !ok {
-		// TODO: Try to re-establish the connection using query protocol
-		return nil, xerrors.Errorf("no connection to miner")
-	}
-
-	var resp SignedResponse
-	if err := cborutil.ReadCborRPC(s, &resp); err != nil {
-		log.Errorw("failed to read Response message", "error", err)
-		return nil, err
-	}
-
-	if err := resp.Verify(deal.MinerWorker); err != nil {
-		return nil, xerrors.Errorf("verifying response signature failed", err)
-	}
-
-	if resp.Response.Proposal != deal.ProposalCid {
-		return nil, xerrors.Errorf("miner responded to a wrong proposal: %s != %s", resp.Response.Proposal, deal.ProposalCid)
-	}
-
-	return &resp.Response, nil
-}
-
-func (c *Client) disconnect(deal ClientDeal) error {
-	s, ok := c.conns[deal.ProposalCid]
-	if !ok {
-		return nil
-	}
-
-	err := s.Close()
-	delete(c.conns, deal.ProposalCid)
-	return err
-}
-
-var _ datatransfer.RequestValidator = &ClientRequestValidator{}
-
-// ClientRequestValidator validates data transfer requests for the client
-// in a storage market
-type ClientRequestValidator struct {
-	deals *statestore.StateStore
-}
-
-// NewClientRequestValidator returns a new client request validator for the
-// given datastore
-func NewClientRequestValidator(deals dtypes.ClientDealStore) *ClientRequestValidator {
-	crv := &ClientRequestValidator{
-		deals: deals,
-	}
-	return crv
-}
-
-// ValidatePush validates a push request received from the peer that will send data
-// Will always error because clients should not accept push requests from a provider
-// in a storage deal (i.e. send data to client).
-func (c *ClientRequestValidator) ValidatePush(
-	sender peer.ID,
-	voucher datatransfer.Voucher,
-	baseCid cid.Cid,
-	Selector ipld.Node) error {
-	return ErrNoPushAccepted
-}
-
-// ValidatePull validates a pull request received from the peer that will receive data
-// Will succeed only if:
-// - voucher has correct type
-// - voucher references an active deal
-// - referenced deal matches the receiver (miner)
-// - referenced deal matches the given base CID
-// - referenced deal is in an acceptable state
-func (c *ClientRequestValidator) ValidatePull(
-	receiver peer.ID,
-	voucher datatransfer.Voucher,
-	baseCid cid.Cid,
-	Selector ipld.Node) error {
-	dealVoucher, ok := voucher.(*StorageDataTransferVoucher)
-	if !ok {
-		return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType)
-	}
-
-	var deal ClientDeal
-	err := c.deals.Get(dealVoucher.Proposal, &deal)
-	if err != nil {
-		return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal)
-	}
-	if deal.Miner != receiver {
-		return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer)
-	}
-	if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) {
-		return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece)
-	}
-	for _, state := range DataTransferStates {
-		if deal.State == state {
-			return nil
-		}
-	}
-	return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState)
-}

chain/deals/provider.go

@@ -1,294 +0,0 @@
-package deals
-
-import (
-	"context"
-	"errors"
-	"sync"
-
-	cid "github.com/ipfs/go-cid"
-	datastore "github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
-	inet "github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/go-statestore"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/datatransfer"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/filecoin-project/lotus/storage"
-	"github.com/filecoin-project/lotus/storage/sectorblocks"
-)
-
-var ProviderDsPrefix = "/deals/provider"
-
-type MinerDeal struct {
-	Client      peer.ID
-	Proposal    actors.StorageDealProposal
-	ProposalCid cid.Cid
-	State       api.DealState
-
-	Ref cid.Cid
-
-	DealID   uint64
-	SectorID uint64 // Set when State >= DealStaged
-
-	s inet.Stream
-}
-
-type Provider struct {
-	pricePerByteBlock types.BigInt // how much we want for storing one byte for one block
-	minPieceSize      uint64
-
-	ask   *types.SignedStorageAsk
-	askLk sync.Mutex
-
-	secb   *sectorblocks.SectorBlocks
-	sminer *storage.Miner
-	full   api.FullNode
-
-	// TODO: This will go away once storage market module + CAR
-	// is implemented
-	dag dtypes.StagingDAG
-
-	// dataTransfer is the manager of data transfers used by this storage provider
-	dataTransfer dtypes.ProviderDataTransfer
-
-	deals *statestore.StateStore
-	ds    dtypes.MetadataDS
-
-	conns map[cid.Cid]inet.Stream
-
-	actor address.Address
-
-	incoming chan MinerDeal
-	updated  chan minerDealUpdate
-	stop     chan struct{}
-	stopped  chan struct{}
-}
-
-type minerDealUpdate struct {
-	newState api.DealState
-	id       cid.Cid
-	err      error
-	mut      func(*MinerDeal)
-}
-
-var (
-	// ErrDataTransferFailed means a data transfer for a deal failed
-	ErrDataTransferFailed = errors.New("deal data transfer failed")
-)
-
-func NewProvider(ds dtypes.MetadataDS, sminer *storage.Miner, secb *sectorblocks.SectorBlocks, dag dtypes.StagingDAG, dataTransfer dtypes.ProviderDataTransfer, fullNode api.FullNode) (*Provider, error) {
-	addr, err := ds.Get(datastore.NewKey("miner-address"))
-	if err != nil {
-		return nil, err
-	}
-	minerAddress, err := address.NewFromBytes(addr)
-	if err != nil {
-		return nil, err
-	}
-
-	h := &Provider{
-		sminer:       sminer,
-		dag:          dag,
-		dataTransfer: dataTransfer,
-		full:         fullNode,
-		secb:         secb,
-
-		pricePerByteBlock: types.NewInt(3), // TODO: allow setting
-		minPieceSize:      256,             // TODO: allow setting (BUT KEEP MIN 256! (because of how we fill sectors up))
-
-		conns: map[cid.Cid]inet.Stream{},
-
-		incoming: make(chan MinerDeal),
-		updated:  make(chan minerDealUpdate),
-		stop:     make(chan struct{}),
-		stopped:  make(chan struct{}),
-
-		actor: minerAddress,
-
-		deals: statestore.New(namespace.Wrap(ds, datastore.NewKey(ProviderDsPrefix))),
-		ds:    ds,
-	}
-
-	if err := h.tryLoadAsk(); err != nil {
-		return nil, err
-	}
-
-	if h.ask == nil {
-		// TODO: we should be fine with this state, and just say it means 'not actively accepting deals'
-		// for now... lets just set a price
-		if err := h.SetPrice(types.NewInt(500_000_000), 1000000); err != nil {
-			return nil, xerrors.Errorf("failed setting a default price: %w", err)
-		}
-	}
-
-	// register a data transfer event handler -- this will move deals from
-	// accepted to staged
-	h.dataTransfer.SubscribeToEvents(h.onDataTransferEvent)
-
-	return h, nil
-}
-
-func (p *Provider) Run(ctx context.Context) {
-	// TODO: restore state
-
-	go func() {
-		defer log.Warn("quitting deal provider loop")
-		defer close(p.stopped)
-
-		for {
-			select {
-			case deal := <-p.incoming: // DealAccepted
-				p.onIncoming(deal)
-			case update := <-p.updated: // DealStaged
-				p.onUpdated(ctx, update)
-			case <-p.stop:
-				return
-			}
-		}
-	}()
-}
-
-func (p *Provider) onIncoming(deal MinerDeal) {
-	log.Info("incoming deal")
-
-	p.conns[deal.ProposalCid] = deal.s
-
-	if err := p.deals.Begin(deal.ProposalCid, &deal); err != nil {
-		// This can happen when client re-sends proposal
-		p.failDeal(deal.ProposalCid, err)
-		log.Errorf("deal tracking failed: %s", err)
-		return
-	}
-
-	go func() {
-		p.updated <- minerDealUpdate{
-			newState: api.DealAccepted,
-			id:       deal.ProposalCid,
-			err:      nil,
-		}
-	}()
-}
-
-func (p *Provider) onUpdated(ctx context.Context, update minerDealUpdate) {
-	log.Infof("Deal %s updated state to %s", update.id, api.DealStates[update.newState])
-	if update.err != nil {
-		log.Errorf("deal %s (newSt: %d) failed: %+v", update.id, update.newState, update.err)
-		p.failDeal(update.id, update.err)
-		return
-	}
-	var deal MinerDeal
-	err := p.deals.Mutate(update.id, func(d *MinerDeal) error {
-		d.State = update.newState
-		if update.mut != nil {
-			update.mut(d)
-		}
-		deal = *d
-		return nil
-	})
-	if err != nil {
-		p.failDeal(update.id, err)
-		return
-	}
-
-	switch update.newState {
-	case api.DealAccepted:
-		p.handle(ctx, deal, p.accept, api.DealNoUpdate)
-	case api.DealStaged:
-		p.handle(ctx, deal, p.staged, api.DealSealing)
-	case api.DealSealing:
-		p.handle(ctx, deal, p.sealing, api.DealComplete)
-	case api.DealComplete:
-		p.handle(ctx, deal, p.complete, api.DealNoUpdate)
-	}
-}
-
-// onDataTransferEvent is the function called when an event occurs in a data
-// transfer -- it reads the voucher to verify this even occurred in a storage
-// market deal, then, based on the data transfer event that occurred, it generates
-// and update message for the deal -- either moving to staged for a completion
-// event or moving to error if a data transfer error occurs
-func (p *Provider) onDataTransferEvent(event datatransfer.Event, channelState datatransfer.ChannelState) {
-	voucher, ok := channelState.Voucher().(*StorageDataTransferVoucher)
-	// if this event is for a transfer not related to storage, ignore
-	if !ok {
-		return
-	}
-
-	// data transfer events for opening and progress do not affect deal state
-	var next api.DealState
-	var err error
-	var mut func(*MinerDeal)
-	switch event {
-	case datatransfer.Complete:
-		next = api.DealStaged
-		mut = func(deal *MinerDeal) {
-			deal.DealID = voucher.DealID
-		}
-	case datatransfer.Error:
-		next = api.DealFailed
-		err = ErrDataTransferFailed
-	default:
-		// the only events we care about are complete and error
-		return
-	}
-
-	select {
-	case p.updated <- minerDealUpdate{
-		newState: next,
-		id:       voucher.Proposal,
-		err:      err,
-		mut:      mut,
-	}:
-	case <-p.stop:
-	}
-}
-
-func (p *Provider) newDeal(s inet.Stream, proposal Proposal) (MinerDeal, error) {
-	proposalNd, err := cborutil.AsIpld(proposal.DealProposal)
-	if err != nil {
-		return MinerDeal{}, err
-	}
-
-	return MinerDeal{
-		Client:      s.Conn().RemotePeer(),
-		Proposal:    *proposal.DealProposal,
-		ProposalCid: proposalNd.Cid(),
-		State:       api.DealUnknown,
-
-		Ref: proposal.Piece,
-
-		s: s,
-	}, nil
-}
-
-func (p *Provider) HandleStream(s inet.Stream) {
-	log.Info("Handling storage deal proposal!")
-
-	proposal, err := p.readProposal(s)
-	if err != nil {
-		log.Error(err)
-		s.Close()
-		return
-	}
-
-	deal, err := p.newDeal(s, proposal)
-	if err != nil {
-		log.Errorf("%+v", err)
-		s.Close()
-		return
-	}
-
-	p.incoming <- deal
-}
-
-func (p *Provider) Stop() {
-	close(p.stop)
-	<-p.stopped
-}

chain/deals/provider_asks.go

@@ -1,160 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"context"
-	"time"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/lotus/chain/stmgr"
-	"github.com/filecoin-project/lotus/chain/types"
-	datastore "github.com/ipfs/go-datastore"
-	inet "github.com/libp2p/go-libp2p-core/network"
-	"golang.org/x/xerrors"
-)
-
-func (p *Provider) SetPrice(price types.BigInt, ttlsecs int64) error {
-	p.askLk.Lock()
-	defer p.askLk.Unlock()
-
-	var seqno uint64
-	if p.ask != nil {
-		seqno = p.ask.Ask.SeqNo + 1
-	}
-
-	now := time.Now().Unix()
-	ask := &types.StorageAsk{
-		Price:        price,
-		Timestamp:    uint64(now),
-		Expiry:       uint64(now + ttlsecs),
-		Miner:        p.actor,
-		SeqNo:        seqno,
-		MinPieceSize: p.minPieceSize,
-	}
-
-	ssa, err := p.signAsk(ask)
-	if err != nil {
-		return err
-	}
-
-	return p.saveAsk(ssa)
-}
-
-func (p *Provider) getAsk(m address.Address) *types.SignedStorageAsk {
-	p.askLk.Lock()
-	defer p.askLk.Unlock()
-	if m != p.actor {
-		return nil
-	}
-
-	return p.ask
-}
-
-func (p *Provider) HandleAskStream(s inet.Stream) {
-	defer s.Close()
-	var ar AskRequest
-	if err := cborutil.ReadCborRPC(s, &ar); err != nil {
-		log.Errorf("failed to read AskRequest from incoming stream: %s", err)
-		return
-	}
-
-	resp := p.processAskRequest(&ar)
-
-	if err := cborutil.WriteCborRPC(s, resp); err != nil {
-		log.Errorf("failed to write ask response: %s", err)
-		return
-	}
-}
-
-func (p *Provider) processAskRequest(ar *AskRequest) *AskResponse {
-	return &AskResponse{
-		Ask: p.getAsk(ar.Miner),
-	}
-}
-
-var bestAskKey = datastore.NewKey("latest-ask")
-
-func (p *Provider) tryLoadAsk() error {
-	p.askLk.Lock()
-	defer p.askLk.Unlock()
-
-	err := p.loadAsk()
-	if err != nil {
-		if xerrors.Is(err, datastore.ErrNotFound) {
-			log.Warn("no previous ask found, miner will not accept deals until a price is set")
-			return nil
-		}
-		return err
-	}
-
-	return nil
-}
-
-func (p *Provider) loadAsk() error {
-	askb, err := p.ds.Get(datastore.NewKey("latest-ask"))
-	if err != nil {
-		return xerrors.Errorf("failed to load most recent ask from disk: %w", err)
-	}
-
-	var ssa types.SignedStorageAsk
-	if err := cborutil.ReadCborRPC(bytes.NewReader(askb), &ssa); err != nil {
-		return err
-	}
-
-	p.ask = &ssa
-	return nil
-}
-
-func (p *Provider) signAsk(a *types.StorageAsk) (*types.SignedStorageAsk, error) {
-	b, err := cborutil.Dump(a)
-	if err != nil {
-		return nil, err
-	}
-
-	worker, err := p.getWorker(p.actor)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to get worker to sign ask: %w", err)
-	}
-
-	sig, err := p.full.WalletSign(context.TODO(), worker, b)
-	if err != nil {
-		return nil, err
-	}
-
-	return &types.SignedStorageAsk{
-		Ask:       a,
-		Signature: sig,
-	}, nil
-}
-
-func (p *Provider) saveAsk(a *types.SignedStorageAsk) error {
-	b, err := cborutil.Dump(a)
-	if err != nil {
-		return err
-	}
-
-	if err := p.ds.Put(bestAskKey, b); err != nil {
-		return err
-	}
-
-	p.ask = a
-	return nil
-}
-
-func (c *Client) checkAskSignature(ask *types.SignedStorageAsk) error {
-	tss := c.sm.ChainStore().GetHeaviestTipSet().ParentState()
-
-	w, err := stmgr.GetMinerWorkerRaw(context.TODO(), c.sm, tss, ask.Ask.Miner)
-	if err != nil {
-		return xerrors.Errorf("failed to get worker for miner in ask", err)
-	}
-
-	sigb, err := cborutil.Dump(ask.Ask)
-	if err != nil {
-		return xerrors.Errorf("failed to re-serialize ask")
-	}
-
-	return ask.Signature.Verify(w, sigb)
-
-}

chain/deals/provider_states.go

@@ -1,215 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"context"
-
-	ipldfree "github.com/ipld/go-ipld-prime/impl/free"
-	"github.com/ipld/go-ipld-prime/traversal/selector"
-	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
-
-	unixfile "github.com/ipfs/go-unixfs/file"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/filecoin-project/lotus/lib/padreader"
-	"github.com/filecoin-project/lotus/storage/sectorblocks"
-)
-
-type providerHandlerFunc func(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error)
-
-func (p *Provider) handle(ctx context.Context, deal MinerDeal, cb providerHandlerFunc, next api.DealState) {
-	go func() {
-		mut, err := cb(ctx, deal)
-
-		if err == nil && next == api.DealNoUpdate {
-			return
-		}
-
-		select {
-		case p.updated <- minerDealUpdate{
-			newState: next,
-			id:       deal.ProposalCid,
-			err:      err,
-			mut:      mut,
-		}:
-		case <-p.stop:
-		}
-	}()
-}
-
-// ACCEPTED
-func (p *Provider) accept(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
-
-	head, err := p.full.ChainHead(ctx)
-	if err != nil {
-		return nil, err
-	}
-	if head.Height() >= deal.Proposal.ProposalExpiration {
-		return nil, xerrors.Errorf("deal proposal already expired")
-	}
-
-	// TODO: check StorageCollateral
-
-	minPrice := types.BigDiv(types.BigMul(p.ask.Ask.Price, types.NewInt(deal.Proposal.PieceSize)), types.NewInt(1<<30))
-	if deal.Proposal.StoragePricePerEpoch.LessThan(minPrice) {
-		return nil, xerrors.Errorf("storage price per epoch less than asking price: %s < %s", deal.Proposal.StoragePricePerEpoch, minPrice)
-	}
-
-	if deal.Proposal.PieceSize < p.ask.Ask.MinPieceSize {
-		return nil, xerrors.Errorf("piece size less than minimum required size: %d < %d", deal.Proposal.PieceSize, p.ask.Ask.MinPieceSize)
-	}
-
-	// check market funds
-	clientMarketBalance, err := p.full.StateMarketBalance(ctx, deal.Proposal.Client, nil)
-	if err != nil {
-		return nil, xerrors.Errorf("getting client market balance failed: %w", err)
-	}
-
-	// This doesn't guarantee that the client won't withdraw / lock those funds
-	// but it's a decent first filter
-	if clientMarketBalance.Available.LessThan(deal.Proposal.TotalStoragePrice()) {
-		return nil, xerrors.New("clientMarketBalance.Available too small")
-	}
-
-	waddr, err := p.full.StateMinerWorker(ctx, deal.Proposal.Provider, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	// TODO: check StorageCollateral (may be too large (or too small))
-	if err := p.full.MarketEnsureAvailable(ctx, waddr, deal.Proposal.StorageCollateral); err != nil {
-		return nil, err
-	}
-
-	log.Info("publishing deal")
-
-	params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{
-		Deals: []actors.StorageDealProposal{deal.Proposal},
-	})
-	if err != nil {
-		return nil, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err)
-	}
-
-	// TODO: We may want this to happen after fetching data
-	smsg, err := p.full.MpoolPushMessage(ctx, &types.Message{
-		To:       actors.StorageMarketAddress,
-		From:     waddr,
-		Value:    types.NewInt(0),
-		GasPrice: types.NewInt(0),
-		GasLimit: types.NewInt(1000000),
-		Method:   actors.SMAMethods.PublishStorageDeals,
-		Params:   params,
-	})
-	if err != nil {
-		return nil, err
-	}
-	r, err := p.full.StateWaitMsg(ctx, smsg.Cid())
-	if err != nil {
-		return nil, err
-	}
-	if r.Receipt.ExitCode != 0 {
-		return nil, xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode)
-	}
-	var resp actors.PublishStorageDealResponse
-	if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil {
-		return nil, err
-	}
-	if len(resp.DealIDs) != 1 {
-		return nil, xerrors.Errorf("got unexpected number of DealIDs from SMA")
-	}
-
-	log.Infof("fetching data for a deal %d", resp.DealIDs[0])
-	err = p.sendSignedResponse(&Response{
-		State: api.DealAccepted,
-
-		Proposal:              deal.ProposalCid,
-		StorageDealSubmission: smsg,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	if err := p.disconnect(deal); err != nil {
-		log.Warnf("closing client connection: %+v", err)
-	}
-
-	ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder())
-
-	// this is the selector for "get the whole DAG"
-	// TODO: support storage deals with custom payload selectors
-	allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(),
-		ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
-
-	// initiate a pull data transfer. This will complete asynchronously and the
-	// completion of the data transfer will trigger a change in deal state
-	// (see onDataTransferEvent)
-	_, err = p.dataTransfer.OpenPullDataChannel(ctx,
-		deal.Client,
-		&StorageDataTransferVoucher{Proposal: deal.ProposalCid, DealID: resp.DealIDs[0]},
-		deal.Ref,
-		allSelector,
-	)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to open pull data channel: %w", err)
-	}
-
-	return nil, nil
-}
-
-// STAGED
-
-func (p *Provider) staged(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
-	root, err := p.dag.Get(ctx, deal.Ref)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to get file root for deal: %s", err)
-	}
-
-	// TODO: abstract this away into ReadSizeCloser + implement different modes
-	n, err := unixfile.NewUnixfsFile(ctx, p.dag, root)
-	if err != nil {
-		return nil, xerrors.Errorf("cannot open unixfs file: %s", err)
-	}
-
-	uf, ok := n.(sectorblocks.UnixfsReader)
-	if !ok {
-		// we probably got directory, unsupported for now
-		return nil, xerrors.Errorf("unsupported unixfs file type")
-	}
-
-	// TODO: uf.Size() is user input, not trusted
-	// This won't be useful / here after we migrate to putting CARs into sectors
-	size, err := uf.Size()
-	if err != nil {
-		return nil, xerrors.Errorf("getting unixfs file size: %w", err)
-	}
-	if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize {
-		return nil, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size")
-	}
-
-	sectorID, err := p.secb.AddUnixfsPiece(ctx, uf, deal.DealID)
-	if err != nil {
-		return nil, xerrors.Errorf("AddPiece failed: %s", err)
-	}
-	log.Warnf("New Sector: %d (deal %d)", sectorID, deal.DealID)
-
-	return func(deal *MinerDeal) {
-		deal.SectorID = sectorID
-	}, nil
-}
-
-// SEALING
-
-func (p *Provider) sealing(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
-	// TODO: consider waiting for seal to happen
-
-	return nil, nil
-}
-
-func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) {
-	// TODO: observe sector lifecycle, status, expiration..
-
-	return nil, nil
-}

chain/deals/provider_utils.go

@@ -1,198 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"context"
-	"runtime"
-
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/datatransfer"
-	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/ipld/go-ipld-prime"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/go-statestore"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/types"
-
-	"github.com/ipfs/go-cid"
-	inet "github.com/libp2p/go-libp2p-core/network"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"golang.org/x/xerrors"
-)
-
-func (p *Provider) failDeal(id cid.Cid, cerr error) {
-	if err := p.deals.End(id); err != nil {
-		log.Warnf("deals.End: %s", err)
-	}
-
-	if cerr == nil {
-		_, f, l, _ := runtime.Caller(1)
-		cerr = xerrors.Errorf("unknown error (fail called at %s:%d)", f, l)
-	}
-
-	log.Warnf("deal %s failed: %s", id, cerr)
-
-	err := p.sendSignedResponse(&Response{
-		State:    api.DealFailed,
-		Message:  cerr.Error(),
-		Proposal: id,
-	})
-
-	s, ok := p.conns[id]
-	if ok {
-		_ = s.Reset()
-		delete(p.conns, id)
-	}
-
-	if err != nil {
-		log.Warnf("notifying client about deal failure: %s", err)
-	}
-}
-
-func (p *Provider) readProposal(s inet.Stream) (proposal Proposal, err error) {
-	if err := cborutil.ReadCborRPC(s, &proposal); err != nil {
-		log.Errorw("failed to read proposal message", "error", err)
-		return proposal, err
-	}
-
-	if err := proposal.DealProposal.Verify(); err != nil {
-		return proposal, xerrors.Errorf("verifying StorageDealProposal: %w", err)
-	}
-
-	if proposal.DealProposal.Provider != p.actor {
-		log.Errorf("proposal with wrong ProviderAddress: %s", proposal.DealProposal.Provider)
-		return proposal, err
-	}
-
-	return
-}
-
-func (p *Provider) sendSignedResponse(resp *Response) error {
-	s, ok := p.conns[resp.Proposal]
-	if !ok {
-		return xerrors.New("couldn't send response: not connected")
-	}
-
-	msg, err := cborutil.Dump(resp)
-	if err != nil {
-		return xerrors.Errorf("serializing response: %w", err)
-	}
-
-	worker, err := p.getWorker(p.actor)
-	if err != nil {
-		return err
-	}
-
-	sig, err := p.full.WalletSign(context.TODO(), worker, msg)
-	if err != nil {
-		return xerrors.Errorf("failed to sign response message: %w", err)
-	}
-
-	signedResponse := &SignedResponse{
-		Response:  *resp,
-		Signature: sig,
-	}
-
-	err = cborutil.WriteCborRPC(s, signedResponse)
-	if err != nil {
-		// Assume client disconnected
-		s.Close()
-		delete(p.conns, resp.Proposal)
-	}
-	return err
-}
-
-func (p *Provider) disconnect(deal MinerDeal) error {
-	s, ok := p.conns[deal.ProposalCid]
-	if !ok {
-		return nil
-	}
-
-	err := s.Close()
-	delete(p.conns, deal.ProposalCid)
-	return err
-}
-
-func (p *Provider) getWorker(miner address.Address) (address.Address, error) {
-	getworker := &types.Message{
-		To:     miner,
-		From:   miner,
-		Method: actors.MAMethods.GetWorkerAddr,
-	}
-	r, err := p.full.StateCall(context.TODO(), getworker, nil)
-	if err != nil {
-		return address.Undef, xerrors.Errorf("getting worker address: %w", err)
-	}
-
-	if r.ExitCode != 0 {
-		return address.Undef, xerrors.Errorf("getWorker call failed: %d", r.ExitCode)
-	}
-
-	return address.NewFromBytes(r.Return)
-}
-
-var _ datatransfer.RequestValidator = &ProviderRequestValidator{}
-
-// ProviderRequestValidator validates data transfer requests for the provider
-// in a storage market
-type ProviderRequestValidator struct {
-	deals *statestore.StateStore
-}
-
-// NewProviderRequestValidator returns a new client request validator for the
-// given datastore
-func NewProviderRequestValidator(deals dtypes.ProviderDealStore) *ProviderRequestValidator {
-	return &ProviderRequestValidator{
-		deals: deals,
-	}
-}
-
-// ValidatePush validates a push request received from the peer that will send data
-// Will succeed only if:
-// - voucher has correct type
-// - voucher references an active deal
-// - referenced deal matches the client
-// - referenced deal matches the given base CID
-// - referenced deal is in an acceptable state
-func (m *ProviderRequestValidator) ValidatePush(
-	sender peer.ID,
-	voucher datatransfer.Voucher,
-	baseCid cid.Cid,
-	Selector ipld.Node) error {
-	dealVoucher, ok := voucher.(*StorageDataTransferVoucher)
-	if !ok {
-		return xerrors.Errorf("voucher type %s: %w", voucher.Identifier(), ErrWrongVoucherType)
-	}
-
-	var deal MinerDeal
-	err := m.deals.Get(dealVoucher.Proposal, &deal)
-	if err != nil {
-		return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal)
-	}
-	if deal.Client != sender {
-		return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Client.String(), sender.String(), ErrWrongPeer)
-	}
-
-	if !bytes.Equal(deal.Proposal.PieceRef, baseCid.Bytes()) {
-		return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece)
-	}
-	for _, state := range DataTransferStates {
-		if deal.State == state {
-			return nil
-		}
-	}
-	return xerrors.Errorf("Deal State %s: %w", deal.State, ErrInacceptableDealState)
-}
-
-// ValidatePull validates a pull request received from the peer that will receive data.
-// Will always error because providers should not accept pull requests from a client
-// in a storage deal (i.e. send data to client).
-func (m *ProviderRequestValidator) ValidatePull(
-	receiver peer.ID,
-	voucher datatransfer.Voucher,
-	baseCid cid.Cid,
-	Selector ipld.Node) error {
-	return ErrNoPullAccepted
-}

chain/deals/request_validation_test.go

@@ -1,291 +0,0 @@
-package deals_test
-
-import (
-	"fmt"
-	"math/rand"
-	"testing"
-
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-datastore"
-	"github.com/ipfs/go-datastore/namespace"
-	dss "github.com/ipfs/go-datastore/sync"
-	blocksutil "github.com/ipfs/go-ipfs-blocksutil"
-	"github.com/libp2p/go-libp2p-core/peer"
-	xerrors "golang.org/x/xerrors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/go-statestore"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/deals"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-var blockGenerator = blocksutil.NewBlockGenerator()
-
-type wrongDTType struct {
-}
-
-func (wrongDTType) ToBytes() ([]byte, error) {
-	return []byte{}, nil
-}
-
-func (wrongDTType) FromBytes([]byte) error {
-	return fmt.Errorf("not implemented")
-}
-
-func (wrongDTType) Identifier() string {
-	return "WrongDTTYPE"
-}
-
-func uniqueStorageDealProposal() (actors.StorageDealProposal, error) {
-	clientAddr, err := address.NewIDAddress(uint64(rand.Int()))
-	if err != nil {
-		return actors.StorageDealProposal{}, err
-	}
-	providerAddr, err := address.NewIDAddress(uint64(rand.Int()))
-	if err != nil {
-		return actors.StorageDealProposal{}, err
-	}
-	return actors.StorageDealProposal{
-		PieceRef: blockGenerator.Next().Cid().Bytes(),
-		Client:   clientAddr,
-		Provider: providerAddr,
-		ProposerSignature: &types.Signature{
-			Data: []byte("foo bar cat dog"),
-			Type: types.KTBLS,
-		},
-	}, nil
-}
-
-func newClientDeal(minerID peer.ID, state api.DealState) (deals.ClientDeal, error) {
-	newProposal, err := uniqueStorageDealProposal()
-	if err != nil {
-		return deals.ClientDeal{}, err
-	}
-	proposalNd, err := cborutil.AsIpld(&newProposal)
-	if err != nil {
-		return deals.ClientDeal{}, err
-	}
-	minerAddr, err := address.NewIDAddress(uint64(rand.Int()))
-	if err != nil {
-		return deals.ClientDeal{}, err
-	}
-
-	return deals.ClientDeal{
-		Proposal:    newProposal,
-		ProposalCid: proposalNd.Cid(),
-		Miner:       minerID,
-		MinerWorker: minerAddr,
-		State:       state,
-	}, nil
-}
-
-func newMinerDeal(clientID peer.ID, state api.DealState) (deals.MinerDeal, error) {
-	newProposal, err := uniqueStorageDealProposal()
-	if err != nil {
-		return deals.MinerDeal{}, err
-	}
-	proposalNd, err := cborutil.AsIpld(&newProposal)
-	if err != nil {
-		return deals.MinerDeal{}, err
-	}
-	ref, err := cid.Cast(newProposal.PieceRef)
-	if err != nil {
-		return deals.MinerDeal{}, err
-	}
-
-	return deals.MinerDeal{
-		Proposal:    newProposal,
-		ProposalCid: proposalNd.Cid(),
-		Client:      clientID,
-		State:       state,
-		Ref:         ref,
-	}, nil
-}
-
-func TestClientRequestValidation(t *testing.T) {
-	ds := dss.MutexWrap(datastore.NewMapDatastore())
-	state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))
-
-	crv := deals.NewClientRequestValidator(state)
-	minerID := peer.ID("fakepeerid")
-	block := blockGenerator.Next()
-	t.Run("ValidatePush fails", func(t *testing.T) {
-		if !xerrors.Is(crv.ValidatePush(minerID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPushAccepted) {
-			t.Fatal("Push should fail for the client request validator for storage deals")
-		}
-	})
-	t.Run("ValidatePull fails deal not found", func(t *testing.T) {
-		proposal, err := uniqueStorageDealProposal()
-		if err != nil {
-			t.Fatal("error creating proposal")
-		}
-		proposalNd, err := cborutil.AsIpld(&proposal)
-		if err != nil {
-			t.Fatal("error serializing proposal")
-		}
-		pieceRef, err := cid.Cast(proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) {
-			t.Fatal("Pull should fail if there is no deal stored")
-		}
-	})
-	t.Run("ValidatePull fails wrong client", func(t *testing.T) {
-		otherMiner := peer.ID("otherminer")
-		clientDeal, err := newClientDeal(otherMiner, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) {
-			t.Fatal("Pull should fail if miner address is incorrect")
-		}
-	})
-	t.Run("ValidatePull fails wrong piece ref", func(t *testing.T) {
-		clientDeal, err := newClientDeal(minerID, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
-			t.Fatal("Pull should fail if piece ref is incorrect")
-		}
-	})
-	t.Run("ValidatePull fails wrong deal state", func(t *testing.T) {
-		clientDeal, err := newClientDeal(minerID, api.DealComplete)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) {
-			t.Fatal("Pull should fail if deal is in a state that cannot be data transferred")
-		}
-	})
-	t.Run("ValidatePull succeeds", func(t *testing.T) {
-		clientDeal, err := newClientDeal(minerID, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(clientDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid, 1}, pieceRef, nil) != nil {
-			t.Fatal("Pull should should succeed when all parameters are correct")
-		}
-	})
-}
-
-func TestProviderRequestValidation(t *testing.T) {
-	ds := dss.MutexWrap(datastore.NewMapDatastore())
-	state := statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))
-
-	mrv := deals.NewProviderRequestValidator(state)
-	clientID := peer.ID("fakepeerid")
-	block := blockGenerator.Next()
-	t.Run("ValidatePull fails", func(t *testing.T) {
-		if !xerrors.Is(mrv.ValidatePull(clientID, wrongDTType{}, block.Cid(), nil), deals.ErrNoPullAccepted) {
-			t.Fatal("Pull should fail for the provider request validator for storage deals")
-		}
-	})
-
-	t.Run("ValidatePush fails deal not found", func(t *testing.T) {
-		proposal, err := uniqueStorageDealProposal()
-		if err != nil {
-			t.Fatal("error creating proposal")
-		}
-		proposalNd, err := cborutil.AsIpld(&proposal)
-		if err != nil {
-			t.Fatal("error serializing proposal")
-		}
-		pieceRef, err := cid.Cast(proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{proposalNd.Cid(), 1}, pieceRef, nil), deals.ErrNoDeal) {
-			t.Fatal("Push should fail if there is no deal stored")
-		}
-	})
-	t.Run("ValidatePush fails wrong miner", func(t *testing.T) {
-		otherClient := peer.ID("otherclient")
-		minerDeal, err := newMinerDeal(otherClient, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrWrongPeer) {
-			t.Fatal("Push should fail if miner address is incorrect")
-		}
-	})
-	t.Run("ValidatePush fails wrong piece ref", func(t *testing.T) {
-		minerDeal, err := newMinerDeal(clientID, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, blockGenerator.Next().Cid(), nil), deals.ErrWrongPiece) {
-			t.Fatal("Push should fail if piece ref is incorrect")
-		}
-	})
-	t.Run("ValidatePush fails wrong deal state", func(t *testing.T) {
-		minerDeal, err := newMinerDeal(clientID, api.DealComplete)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil), deals.ErrInacceptableDealState) {
-			t.Fatal("Push should fail if deal is in a state that cannot be data transferred")
-		}
-	})
-	t.Run("ValidatePush succeeds", func(t *testing.T) {
-		minerDeal, err := newMinerDeal(clientID, api.DealAccepted)
-		if err != nil {
-			t.Fatal("error creating client deal")
-		}
-		if err := state.Begin(minerDeal.ProposalCid, &minerDeal); err != nil {
-			t.Fatal("deal tracking failed")
-		}
-		pieceRef, err := cid.Cast(minerDeal.Proposal.PieceRef)
-		if err != nil {
-			t.Fatal("unable to construct piece cid")
-		}
-		if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid, 1}, pieceRef, nil) != nil {
-			t.Fatal("Push should should succeed when all parameters are correct")
-		}
-	})
-}

chain/deals/types.go

@@ -1,114 +0,0 @@
-package deals
-
-import (
-	"bytes"
-	"errors"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
-	"github.com/filecoin-project/lotus/api"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/types"
-	"github.com/ipfs/go-cid"
-)
-
-var (
-	// ErrWrongVoucherType means the voucher was not the correct type can validate against
-	ErrWrongVoucherType = errors.New("cannot validate voucher type.")
-
-	// ErrNoPushAccepted just means clients do not accept pushes for storage deals
-	ErrNoPushAccepted = errors.New("client should not receive data for a storage deal.")
-
-	// ErrNoPullAccepted just means providers do not accept pulls for storage deals
-	ErrNoPullAccepted = errors.New("provider should not send data for a storage deal.")
-
-	// ErrNoDeal means no active deal was found for this vouchers proposal cid
-	ErrNoDeal = errors.New("no deal found for this proposal.")
-
-	// ErrWrongPeer means that the other peer for this data transfer request does not match
-	// the other peer for the deal
-	ErrWrongPeer = errors.New("data Transfer peer id and Deal peer id do not match.")
-
-	// ErrWrongPiece means that the pieceref for this data transfer request does not match
-	// the one specified in the deal
-	ErrWrongPiece = errors.New("base CID for deal does not match CID for piece.")
-
-	// ErrInacceptableDealState means the deal for this transfer is not in a deal state
-	// where transfer can be performed
-	ErrInacceptableDealState = errors.New("deal is not a in a state where deals are accepted.")
-
-	// DataTransferStates are the states in which it would make sense to actually start a data transfer
-	DataTransferStates = []api.DealState{api.DealAccepted, api.DealUnknown}
-)
-
-const DealProtocolID = "/fil/storage/mk/1.0.1"
-const AskProtocolID = "/fil/storage/ask/1.0.1"
-
-type Proposal struct {
-	DealProposal *actors.StorageDealProposal
-
-	Piece cid.Cid // Used for retrieving from the client
-}
-
-type Response struct {
-	State api.DealState
-
-	// DealProposalRejected
-	Message  string
-	Proposal cid.Cid
-
-	// DealAccepted
-	StorageDealSubmission *types.SignedMessage
-}
-
-// TODO: Do we actually need this to be signed?
-type SignedResponse struct {
-	Response Response
-
-	Signature *types.Signature
-}
-
-func (r *SignedResponse) Verify(addr address.Address) error {
-	b, err := cborutil.Dump(&r.Response)
-	if err != nil {
-		return err
-	}
-
-	return r.Signature.Verify(addr, b)
-}
-
-type AskRequest struct {
-	Miner address.Address
-}
-
-type AskResponse struct {
-	Ask *types.SignedStorageAsk
-}
-
-// StorageDataTransferVoucher is the voucher type for data transfers
-// used by the storage market
-type StorageDataTransferVoucher struct {
-	Proposal cid.Cid
-	DealID   uint64
-}
-
-// ToBytes converts the StorageDataTransferVoucher to raw bytes
-func (dv *StorageDataTransferVoucher) ToBytes() ([]byte, error) {
-	var buf bytes.Buffer
-	err := dv.MarshalCBOR(&buf)
-	if err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-// FromBytes converts the StorageDataTransferVoucher to raw bytes
-func (dv *StorageDataTransferVoucher) FromBytes(raw []byte) error {
-	r := bytes.NewReader(raw)
-	return dv.UnmarshalCBOR(r)
-}
-
-// Identifier is the unique string identifier for a StorageDataTransferVoucher
-func (dv *StorageDataTransferVoucher) Identifier() string {
-	return "StorageDataTransferVoucher"
-}

chain/events/events.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"

chain/gen/gen.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/binary"
 	"fmt"
+	"github.com/filecoin-project/lotus/chain/vm"
 	"io/ioutil"
 	"sync/atomic"
 
@@ -34,7 +35,7 @@ import (
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 )
 
 var log = logging.Logger("gen")
@@ -170,7 +171,9 @@ func NewGenerator() (*ChainGen, error) {
 		MinerAddrs: []address.Address{maddr1, maddr2},
 	}
 
-	genb, err := MakeGenesisBlock(bs, map[address.Address]types.BigInt{
+	sys := vm.Syscalls(sectorbuilder.ProofVerifier)
+
+	genb, err := MakeGenesisBlock(bs, sys, map[address.Address]types.BigInt{
 		mk1:    types.FromFil(40000),
 		mk2:    types.FromFil(40000),
 		banker: types.FromFil(50000),
@@ -179,7 +182,7 @@ func NewGenerator() (*ChainGen, error) {
 		return nil, xerrors.Errorf("make genesis block failed: %w", err)
 	}
 
-	cs := store.NewChainStore(bs, ds)
+	cs := store.NewChainStore(bs, ds, sys)
 
 	genfb := &types.FullBlock{Header: genb.Genesis}
 	gents := store.NewFullTipSet([]*types.FullBlock{genfb})

chain/gen/utils.go

@@ -275,7 +275,7 @@ func mustEnc(i cbg.CBORMarshaler) []byte {
 }
 
 func SetupStorageMiners(ctx context.Context, cs *store.ChainStore, sroot cid.Cid, gmcfg *GenMinerCfg) (cid.Cid, []actors.StorageDealProposal, error) {
-	vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore())
+	vm, err := vm.NewVM(sroot, 0, nil, actors.NetworkAddress, cs.Blockstore(), cs.VMSys())
 	if err != nil {
 		return cid.Undef, nil, xerrors.Errorf("failed to create NewVM: %w", err)
 	}
@@ -555,7 +555,7 @@ func doExecValue(ctx context.Context, vm *vm.VM, to, from address.Address, value
 	return ret.Return, nil
 }
 
-func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.BigInt, gmcfg *GenMinerCfg, ts uint64) (*GenesisBootstrap, error) {
+func MakeGenesisBlock(bs bstore.Blockstore, sys *types.VMSyscalls, balances map[address.Address]types.BigInt, gmcfg *GenMinerCfg, ts uint64) (*GenesisBootstrap, error) {
 	ctx := context.Background()
 
 	state, err := MakeInitialStateTree(bs, balances)
@@ -569,7 +569,7 @@ func MakeGenesisBlock(bs bstore.Blockstore, balances map[address.Address]types.B
 	}
 
 	// temp chainstore
-	cs := store.NewChainStore(bs, datastore.NewMapDatastore())
+	cs := store.NewChainStore(bs, datastore.NewMapDatastore(), sys)
 	stateroot, deals, err := SetupStorageMiners(ctx, cs, stateroot, gmcfg)
 	if err != nil {
 		return nil, xerrors.Errorf("setup storage miners failed: %w", err)

chain/messagepool/messagepool.go

@@ -13,7 +13,7 @@ import (
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
 	"github.com/ipfs/go-datastore/query"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	lps "github.com/whyrusleeping/pubsub"
 	"go.uber.org/multierr"

chain/metrics/consensus.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 	"go.uber.org/fx"
 

chain/state/statetree.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/ipfs/go-cid"
 	hamt "github.com/ipfs/go-hamt-ipld"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"

chain/stmgr/call.go

@@ -18,7 +18,7 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate
 	ctx, span := trace.StartSpan(ctx, "statemanager.CallRaw")
 	defer span.End()
 
-	vmi, err := vm.NewVM(bstate, bheight, r, actors.NetworkAddress, sm.cs.Blockstore())
+	vmi, err := vm.NewVM(bstate, bheight, r, actors.NetworkAddress, sm.cs.Blockstore(), sm.cs.VMSys())
 	if err != nil {
 		return nil, xerrors.Errorf("failed to set up vm: %w", err)
 	}

chain/stmgr/fork_no_p_eps.go

@@ -1,127 +0,0 @@
-package stmgr
-
-import (
-	"context"
-
-	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-amt-ipld"
-	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-hamt-ipld"
-	cbor "github.com/ipfs/go-ipld-cbor"
-	"github.com/whyrusleeping/cbor-gen"
-	"golang.org/x/xerrors"
-
-	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/state"
-	"github.com/filecoin-project/lotus/chain/types"
-)
-
-func (sm *StateManager) forkNoPowerEPS(ctx context.Context, pstate cid.Cid) (cid.Cid, error) {
-	cst := hamt.CSTFromBstore(sm.cs.Blockstore())
-	st, err := state.LoadStateTree(cst, pstate)
-	if err != nil {
-		return cid.Undef, xerrors.Errorf("loading parent state tree: %w", err)
-	}
-
-	if err := st.MutateActor(actors.StoragePowerAddress, func(spa *types.Actor) error {
-		var head actors.StoragePowerState
-		if err := cst.Get(ctx, spa.Head, &head); err != nil {
-			return xerrors.Errorf("reading StoragePower state: %w", err)
-		}
-
-		buckets, err := amt.LoadAMT(amt.WrapBlockstore(sm.cs.Blockstore()), head.ProvingBuckets)
-		if err != nil {
-			return xerrors.Errorf("opening proving buckets AMT: %w", err)
-		}
-
-		fixedBuckets := map[uint64]map[address.Address]struct{}{}
-
-		if err := buckets.ForEach(func(bucketId uint64, ent *typegen.Deferred) error {
-			var bcid cid.Cid
-			if err := cbor.DecodeInto(ent.Raw, &bcid); err != nil {
-				return xerrors.Errorf("decoding bucket cid: %w", err)
-			}
-
-			bucket, err := hamt.LoadNode(ctx, cst, bcid)
-			if err != nil {
-				return xerrors.Errorf("loading bucket hamt: %w", err)
-			}
-
-			return bucket.ForEach(ctx, func(abytes string, _ interface{}) error {
-				addr, err := address.NewFromBytes([]byte(abytes))
-				if err != nil {
-					return xerrors.Errorf("parsing address in proving bucket: %w", err)
-				}
-
-				// now find the correct bucket
-				miner, err := st.GetActor(addr)
-				if err != nil {
-					return xerrors.Errorf("getting miner %s: %w", addr, err)
-				}
-
-				var minerHead actors.StorageMinerActorState
-				if err := cst.Get(ctx, miner.Head, &minerHead); err != nil {
-					return xerrors.Errorf("reading miner %s state: %w", addr, err)
-				}
-
-				correctBucket := minerHead.ElectionPeriodStart % build.SlashablePowerDelay
-				if correctBucket != bucketId {
-					log.Warnf("miner %s was in wrong proving bucket %d, putting in %d (eps: %d)", addr, bucketId, correctBucket, minerHead.ElectionPeriodStart)
-				}
-
-				if _, ok := fixedBuckets[correctBucket]; !ok {
-					fixedBuckets[correctBucket] = map[address.Address]struct{}{}
-				}
-				fixedBuckets[correctBucket][addr] = struct{}{}
-
-				return nil
-			})
-		}); err != nil {
-			return err
-		}
-
-		// /////
-		// Write fixed buckets
-
-		fixed := amt.NewAMT(amt.WrapBlockstore(sm.cs.Blockstore()))
-
-		for bucketId, addrss := range fixedBuckets {
-			bucket := hamt.NewNode(cst)
-			for addr := range addrss {
-				if err := bucket.Set(ctx, string(addr.Bytes()), actors.CborNull); err != nil {
-					return xerrors.Errorf("setting address in bucket: %w", err)
-				}
-			}
-
-			if err := bucket.Flush(ctx); err != nil {
-				return xerrors.Errorf("flushing bucket amt: %w", err)
-			}
-
-			bcid, err := cst.Put(context.TODO(), bucket)
-			if err != nil {
-				return xerrors.Errorf("put bucket: %w", err)
-			}
-
-			if err := fixed.Set(bucketId, bcid); err != nil {
-				return xerrors.Errorf("set bucket: %w", err)
-			}
-		}
-
-		head.ProvingBuckets, err = fixed.Flush()
-		if err != nil {
-			return xerrors.Errorf("flushing bucket amt: %w", err)
-		}
-
-		spa.Head, err = cst.Put(ctx, &head)
-		if err != nil {
-			return xerrors.Errorf("putting actor head: %w", err)
-		}
-
-		return nil
-	}); err != nil {
-		return cid.Undef, err
-	}
-
-	return st.Flush()
-}

chain/stmgr/forks.go

@@ -2,22 +2,13 @@ package stmgr
 
 import (
 	"context"
-	"github.com/ipfs/go-cid"
-	"golang.org/x/xerrors"
 
-	"github.com/filecoin-project/lotus/build"
+	"github.com/ipfs/go-cid"
 )
 
 func (sm *StateManager) handleStateForks(ctx context.Context, pstate cid.Cid, height, parentH uint64) (_ cid.Cid, err error) {
 	for i := parentH; i < height; i++ {
 		switch i {
-		case build.ForkNoPowerEPSUpdates:
-			pstate, err = sm.forkNoPowerEPS(ctx, pstate)
-			if err != nil {
-				return cid.Undef, xerrors.Errorf("executing state fork in epoch %d: %w", i, err)
-			}
-
-			log.Infof("forkNoPowerEPS state: %s", pstate)
 		}
 	}
 

chain/stmgr/stmgr.go

@@ -18,7 +18,7 @@ import (
 	bls "github.com/filecoin-project/filecoin-ffi"
 	"github.com/ipfs/go-cid"
 	hamt "github.com/ipfs/go-hamt-ipld"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"go.opencensus.io/trace"
 )
 
@@ -139,7 +139,7 @@ func (sm *StateManager) computeTipSetState(ctx context.Context, blks []*types.Bl
 
 	r := store.NewChainRand(sm.cs, cids, blks[0].Height)
 
-	vmi, err := vm.NewVM(pstate, blks[0].Height, r, address.Undef, sm.cs.Blockstore())
+	vmi, err := vm.NewVM(pstate, blks[0].Height, r, address.Undef, sm.cs.Blockstore(), sm.cs.VMSys())
 	if err != nil {
 		return cid.Undef, cid.Undef, xerrors.Errorf("instantiating VM failed: %w", err)
 	}

chain/store/store.go

@@ -23,7 +23,7 @@ import (
 	dstore "github.com/ipfs/go-datastore"
 	hamt "github.com/ipfs/go-hamt-ipld"
 	bstore "github.com/ipfs/go-ipfs-blockstore"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	pubsub "github.com/whyrusleeping/pubsub"
 	"golang.org/x/xerrors"
@@ -51,9 +51,11 @@ type ChainStore struct {
 
 	mmCache *lru.ARCCache
 	tsCache *lru.ARCCache
+
+	vmcalls *types.VMSyscalls
 }
 
-func NewChainStore(bs bstore.Blockstore, ds dstore.Batching) *ChainStore {
+func NewChainStore(bs bstore.Blockstore, ds dstore.Batching, vmcalls *types.VMSyscalls) *ChainStore {
 	c, _ := lru.NewARC(2048)
 	tsc, _ := lru.NewARC(4096)
 	cs := &ChainStore{
@@ -63,6 +65,7 @@ func NewChainStore(bs bstore.Blockstore, ds dstore.Batching) *ChainStore {
 		tipsets:  make(map[uint64][]cid.Cid),
 		mmCache:  c,
 		tsCache:  tsc,
+		vmcalls:  vmcalls,
 	}
 
 	cs.reorgCh = cs.reorgWorker(context.TODO())
@@ -793,6 +796,10 @@ func (cs *ChainStore) Blockstore() bstore.Blockstore {
 	return cs.bs
 }
 
+func (cs *ChainStore) VMSys() *types.VMSyscalls {
+	return cs.vmcalls
+}
+
 func (cs *ChainStore) TryFillTipSet(ts *types.TipSet) (*FullTipSet, error) {
 	var out []*types.FullBlock
 

chain/store/store_test.go

@@ -55,7 +55,7 @@ func BenchmarkGetRandomness(b *testing.B) {
 
 	bs := blockstore.NewBlockstore(bds)
 
-	cs := store.NewChainStore(bs, mds)
+	cs := store.NewChainStore(bs, mds, nil)
 
 	b.ResetTimer()
 

chain/store/weight.go

@@ -60,7 +60,7 @@ func (cs *ChainStore) call(ctx context.Context, msg *types.Message, ts *types.Ti
 
 	r := NewChainRand(cs, ts.Cids(), ts.Height())
 
-	vmi, err := vm.NewVM(bstate, ts.Height(), r, actors.NetworkAddress, cs.bs)
+	vmi, err := vm.NewVM(bstate, ts.Height(), r, actors.NetworkAddress, cs.bs, cs.vmcalls)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to set up vm: %w", err)
 	}

chain/sub/incoming.go

@@ -4,10 +4,11 @@ import (
 	"context"
 	"time"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	connmgr "github.com/libp2p/go-libp2p-core/connmgr"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 
+	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/messagepool"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -33,6 +34,11 @@ func HandleIncomingBlocks(ctx context.Context, bsub *pubsub.Subscription, s *cha
 			continue
 		}
 
+		if len(blk.BlsMessages)+len(blk.SecpkMessages) > build.BlockMessageLimit {
+			log.Warnf("received block with too many messages over pubsub")
+			continue
+		}
+
 		go func() {
 			log.Infof("New block over pubsub: %s", blk.Cid())
 

chain/sync.go

@@ -18,7 +18,7 @@ import (
 	dstore "github.com/ipfs/go-datastore"
 	hamt "github.com/ipfs/go-hamt-ipld"
 	bstore "github.com/ipfs/go-ipfs-blockstore"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p-core/connmgr"
 	"github.com/libp2p/go-libp2p-core/peer"
 	cbg "github.com/whyrusleeping/cbor-gen"
@@ -189,6 +189,10 @@ func (syncer *Syncer) IncomingBlocks(ctx context.Context) (<-chan *types.BlockHe
 }
 
 func (syncer *Syncer) ValidateMsgMeta(fblk *types.FullBlock) error {
+	if msgc := len(fblk.BlsMessages) + len(fblk.SecpkMessages); msgc > build.BlockMessageLimit {
+		return xerrors.Errorf("block %s has too many messages (%d)", fblk.Header.Cid(), msgc)
+	}
+
 	var bcids, scids []cbg.CBORMarshaler
 	for _, m := range fblk.BlsMessages {
 		c := cbg.CborCid(m.Cid())
@@ -295,6 +299,10 @@ func zipTipSetAndMessages(bs amt.Blocks, ts *types.TipSet, allbmsgs []*types.Mes
 			bmsgCids = append(bmsgCids, &c)
 		}
 
+		if msgc := len(bmsgCids) + len(smsgCids); msgc > build.BlockMessageLimit {
+			return nil, fmt.Errorf("block %q has too many messages (%d)", b.Cid(), msgc)
+		}
+
 		mrcid, err := computeMsgMeta(bs, bmsgCids, smsgCids)
 		if err != nil {
 			return nil, err
@@ -533,15 +541,17 @@ func (syncer *Syncer) ValidateBlock(ctx context.Context, b *types.FullBlock) err
 
 		snum := types.BigDiv(mpow, types.NewInt(ssize))
 
-		// FORK START
+		if len(h.EPostProof.Candidates) == 0 {
-		if h.Height > build.ForkCCM {
+			return xerrors.Errorf("no candidates")
-			if len(h.EPostProof.Candidates) == 0 {
-				return xerrors.Errorf("no candidates")
-			}
 		}
-		// FORK END
 
+		wins := make(map[uint64]bool)
 		for _, t := range h.EPostProof.Candidates {
+			if wins[t.ChallengeIndex] {
+				return xerrors.Errorf("block had duplicate epost candidates")
+			}
+			wins[t.ChallengeIndex] = true
+
 			if !types.IsTicketWinner(t.Partial, ssize, snum.Uint64(), tpow) {
 				return xerrors.Errorf("miner created a block but was not a winner")
 			}
@@ -662,13 +672,10 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc
 			SectorChallengeIndex: t.ChallengeIndex,
 		})
 	}
-	// FORK START
+
-	if h.Height > build.ForkCCM {
+	if len(winners) == 0 {
-		if len(winners) == 0 {
+		return xerrors.Errorf("no candidates")
-			return xerrors.Errorf("no candidates")
-		}
 	}
-	// FORK END
 
 	sectorInfo, err := stmgr.GetSectorsForElectionPost(ctx, syncer.sm, baseTs, h.Miner)
 	if err != nil {
@@ -683,7 +690,7 @@ func (syncer *Syncer) VerifyElectionPoStProof(ctx context.Context, h *types.Bloc
 	}
 	hvrf := sha256.Sum256(h.EPostProof.PostRand)
 
-	ok, err := sectorbuilder.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner)
+	ok, err := sectorbuilder.ProofVerifier.VerifyElectionPost(ctx, ssize, *sectorInfo, hvrf[:], h.EPostProof.Proof, winners, h.Miner)
 	if err != nil {
 		return xerrors.Errorf("failed to verify election post: %w", err)
 	}

chain/sync_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p-core/peer"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	"github.com/stretchr/testify/require"

chain/types/bitfield.go

@@ -1,6 +1,7 @@
 package types
 
 import (
+	"errors"
 	"fmt"
 	"io"
 
@@ -9,6 +10,8 @@ import (
 	"golang.org/x/xerrors"
 )
 
+var ErrBitFieldTooMany = errors.New("to many items in RLE")
+
 type BitField struct {
 	rle rlepluslazy.RLE
 
@@ -16,14 +19,23 @@ type BitField struct {
 }
 
 func NewBitField() BitField {
-	rle, err := rlepluslazy.FromBuf([]byte{})
+	bf, err := NewBitFieldFromBytes([]byte{})
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("creating empty rle: %+v", err))
 	}
-	return BitField{
+	return bf
-		rle:  rle,
+}
-		bits: make(map[uint64]struct{}),
+
+func NewBitFieldFromBytes(rle []byte) (BitField, error) {
+	bf := BitField{}
+	rlep, err := rlepluslazy.FromBuf(rle)
+	if err != nil {
+		return BitField{}, xerrors.Errorf("could not decode rle+: %w", err)
 	}
+	bf.rle = rlep
+	bf.bits = make(map[uint64]struct{})
+	return bf, nil
+
 }
 
 func BitFieldFromSet(setBits []uint64) BitField {
@@ -106,7 +118,14 @@ func (bf BitField) Count() (uint64, error) {
 }
 
 // All returns all set bits
-func (bf BitField) All() ([]uint64, error) {
+func (bf BitField) All(max uint64) ([]uint64, error) {
+	c, err := bf.Count()
+	if err != nil {
+		return nil, xerrors.Errorf("count errror: %w", err)
+	}
+	if c > max {
+		return nil, xerrors.Errorf("expected %d, got %d: %w", max, c, ErrBitFieldTooMany)
+	}
 
 	runs, err := bf.sum()
 	if err != nil {
@@ -121,7 +140,14 @@ func (bf BitField) All() ([]uint64, error) {
 	return res, nil
 }
 
-func (bf BitField) AllMap() (map[uint64]bool, error) {
+func (bf BitField) AllMap(max uint64) (map[uint64]bool, error) {
+	c, err := bf.Count()
+	if err != nil {
+		return nil, xerrors.Errorf("count errror: %w", err)
+	}
+	if c > max {
+		return nil, xerrors.Errorf("expected %d, got %d: %w", max, c, ErrBitFieldTooMany)
+	}
 
 	runs, err := bf.sum()
 	if err != nil {

chain/types/blockheader.go

@@ -59,6 +59,8 @@ type BlockHeader struct {
 	Timestamp uint64
 
 	BlockSig *Signature
+
+	ForkSignaling uint64
 }
 
 func (b *BlockHeader) ToStorageBlock() (block.Block, error) {
@@ -216,7 +218,7 @@ func IsTicketWinner(partialTicket []byte, ssizeI uint64, snum uint64, totpow Big
 	return lhs.Cmp(rhs) < 0
 }
 
-func ElectionPostChallengeCount(sectors uint64, faults int) uint64 {
+func ElectionPostChallengeCount(sectors uint64, faults uint64) uint64 {
 	return sectorbuilder.ElectionPostChallengeCount(sectors, faults)
 }
 

chain/types/cbor_gen.go

@@ -5,7 +5,7 @@ import (
 	"io"
 	"math"
 
-	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-cid"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	xerrors "golang.org/x/xerrors"
 )
@@ -19,7 +19,7 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
 		_, err := w.Write(cbg.CborNull)
 		return err
 	}
-	if _, err := w.Write([]byte{140}); err != nil {
+	if _, err := w.Write([]byte{141}); err != nil {
 		return err
 	}
 
@@ -94,6 +94,11 @@ func (t *BlockHeader) MarshalCBOR(w io.Writer) error {
 	if err := t.BlockSig.MarshalCBOR(w); err != nil {
 		return err
 	}
+
+	// t.ForkSignaling (uint64) (uint64)
+	if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.ForkSignaling))); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -108,7 +113,7 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
 		return fmt.Errorf("cbor input should be of type array")
 	}
 
-	if extra != 12 {
+	if extra != 13 {
 		return fmt.Errorf("cbor input had wrong number of fields")
 	}
 
@@ -272,6 +277,16 @@ func (t *BlockHeader) UnmarshalCBOR(r io.Reader) error {
 		}
 
 	}
+	// t.ForkSignaling (uint64) (uint64)
+
+	maj, extra, err = cbg.CborReadHeader(br)
+	if err != nil {
+		return err
+	}
+	if maj != cbg.MajUnsignedInt {
+		return fmt.Errorf("wrong type for uint64 field")
+	}
+	t.ForkSignaling = uint64(extra)
 	return nil
 }
 

chain/types/tipset.go

@@ -8,7 +8,7 @@ import (
 	"sort"
 
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
 )

chain/types/vmcontext.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	"context"
+	"github.com/filecoin-project/go-sectorbuilder"
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/go-amt-ipld"
@@ -46,7 +47,15 @@ type VMContext interface {
 }
 
 type VMSyscalls struct {
-	ValidatePoRep func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError)
+	ValidatePoRep      func(context.Context, address.Address, uint64, []byte, []byte, []byte, []byte, []byte, uint64) (bool, aerrors.ActorError)
+	VerifyFallbackPost func(ctx context.Context,
+		sectorSize uint64,
+		sectorInfo sectorbuilder.SortedPublicSectorInfo,
+		challengeSeed []byte,
+		proof []byte,
+		candidates []sectorbuilder.EPostCandidate,
+		proverID address.Address,
+		faults uint64) (bool, error)
 }
 
 type storageWrapper struct {

chain/validation/applier.go

@@ -2,6 +2,7 @@ package validation
 
 import (
 	"context"
+	"github.com/filecoin-project/go-sectorbuilder"
 
 	vchain "github.com/filecoin-project/chain-validation/pkg/chain"
 	vstate "github.com/filecoin-project/chain-validation/pkg/state"
@@ -32,7 +33,7 @@ func (a *Applier) ApplyMessage(eCtx *vchain.ExecutionContext, state vstate.Wrapp
 	if err != nil {
 		return vchain.MessageReceipt{}, err
 	}
-	lotusVM, err := vm.NewVM(base, eCtx.Epoch, randSrc, minerAddr, st.bs)
+	lotusVM, err := vm.NewVM(base, eCtx.Epoch, randSrc, minerAddr, st.bs, vm.Syscalls(sectorbuilder.ProofVerifier))
 	if err != nil {
 		return vchain.MessageReceipt{}, err
 	}

chain/vm/syscalls.go

@@ -1,14 +1,29 @@
 package vm
 
 import (
+	"context"
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-sectorbuilder"
 	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/aerrors"
 	"github.com/filecoin-project/lotus/chain/types"
+	"go.opencensus.io/trace"
 )
 
 // Actual type is defined in chain/types/vmcontext.go because the VMContext interface is there
 
-func DefaultSyscalls() *types.VMSyscalls {
+func Syscalls(verifier sectorbuilder.Verifier) *types.VMSyscalls {
 	return &types.VMSyscalls{
-		ValidatePoRep: actors.ValidatePoRep,
+		ValidatePoRep: func(ctx context.Context, maddr address.Address, ssize uint64, commD, commR, ticket, proof, seed []byte, sectorID uint64) (bool, actors.ActorError) {
+			_, span := trace.StartSpan(ctx, "ValidatePoRep")
+			defer span.End()
+			ok, err := verifier.VerifySeal(ssize, commR, commD, maddr, ticket, seed, sectorID, proof)
+			if err != nil {
+				return false, aerrors.Absorb(err, 25, "verify seal failed")
+			}
+
+			return ok, nil
+		},
+		VerifyFallbackPost: verifier.VerifyFallbackPost,
 	}
 }

chain/vm/vm.go

@@ -10,7 +10,7 @@ import (
 	cid "github.com/ipfs/go-cid"
 	hamt "github.com/ipfs/go-hamt-ipld"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
@@ -259,7 +259,7 @@ func (bs *gasChargingBlocks) GetBlock(ctx context.Context, c cid.Cid) (block.Blo
 	}
 	blk, err := bs.under.GetBlock(ctx, c)
 	if err != nil {
-		return nil, err
+		return nil, aerrors.Escalate(err, "failed to get block from blockstore")
 	}
 	if err := bs.chargeGas(uint64(len(blk.RawData())) * gasGetPerByte); err != nil {
 		return nil, err
@@ -272,7 +272,10 @@ func (bs *gasChargingBlocks) AddBlock(blk block.Block) error {
 	if err := bs.chargeGas(gasPutObj + uint64(len(blk.RawData()))*gasPutPerByte); err != nil {
 		return err
 	}
-	return bs.under.AddBlock(blk)
+	if err := bs.under.AddBlock(blk); err != nil {
+		return aerrors.Escalate(err, "failed to write data to disk")
+	}
+	return nil
 }
 
 func (vm *VM) makeVMContext(ctx context.Context, sroot cid.Cid, msg *types.Message, origin address.Address, usedGas types.BigInt) *VMContext {
@@ -309,7 +312,7 @@ type VM struct {
 	Syscalls *types.VMSyscalls
 }
 
-func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore) (*VM, error) {
+func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs blockstore.Blockstore, syscalls *types.VMSyscalls) (*VM, error) {
 	buf := bufbstore.NewBufferedBstore(cbs)
 	cst := hamt.CSTFromBstore(buf)
 	state, err := state.LoadStateTree(cst, base)
@@ -325,8 +328,8 @@ func NewVM(base cid.Cid, height uint64, r Rand, maddr address.Address, cbs block
 		blockHeight: height,
 		blockMiner:  maddr,
 		inv:         newInvoker(),
-		rand:        r,
+		rand:        r, // TODO: Probably should be a syscall
-		Syscalls:    DefaultSyscalls(),
+		Syscalls:    syscalls,
 	}, nil
 }
 

chain/wallet/wallet.go

@@ -9,7 +9,7 @@ import (
 
 	bls "github.com/filecoin-project/filecoin-ffi"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/minio/blake2b-simd"
 	"golang.org/x/xerrors"
 

cli/cmd.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"syscall"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/mitchellh/go-homedir"
 	"github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr-net"

cli/wallet.go

@@ -41,7 +41,7 @@ var walletNew = &cli.Command{
 
 		t := cctx.Args().First()
 		if t == "" {
-			t = "bls"
+			t = "secp256k1"
 		}
 
 		nk, err := api.WalletNew(ctx, t)

cmd/lotus-bench/main.go

@@ -17,7 +17,7 @@ import (
 	ffi "github.com/filecoin-project/filecoin-ffi"
 	paramfetch "github.com/filecoin-project/go-paramfetch"
 	"github.com/ipfs/go-datastore"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/mitchellh/go-homedir"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
@@ -188,7 +188,7 @@ func main() {
 
 				log.Info("Running replication...")
 				pieces := []sectorbuilder.PublicPieceInfo{pi}
-				pco, err := sb.SealPreCommit(i, ticket, pieces)
+				pco, err := sb.SealPreCommit(context.TODO(), i, ticket, pieces)
 				if err != nil {
 					return xerrors.Errorf("commit: %w", err)
 				}
@@ -206,14 +206,14 @@ func main() {
 				}
 
 				log.Info("Generating PoRep for sector")
-				proof, err := sb.SealCommit(i, ticket, seed, pieces, pco)
+				proof, err := sb.SealCommit(context.TODO(), i, ticket, seed, pieces, pco)
 				if err != nil {
 					return err
 				}
 
 				sealcommit := time.Now()
 				commD := pi.CommP
-				ok, err := sectorbuilder.VerifySeal(sectorSize, pco.CommR[:], commD[:], maddr, ticket.TicketBytes[:], seed.TicketBytes[:], i, proof)
+				ok, err := sectorbuilder.ProofVerifier.VerifySeal(sectorSize, pco.CommR[:], commD[:], maddr, ticket.TicketBytes[:], seed.TicketBytes[:], i, proof)
 				if err != nil {
 					return err
 				}
@@ -307,7 +307,7 @@ func main() {
 				log.Warn("separate epost calls returned different proof values (this might be bad)")
 			}
 
-			ok, err := sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof1, candidates[:1], maddr)
+			ok, err := sectorbuilder.ProofVerifier.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof1, candidates[:1], maddr)
 			if err != nil {
 				return err
 			}
@@ -317,7 +317,7 @@ func main() {
 
 			verifypost1 := time.Now()
 
-			ok, err = sectorbuilder.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof2, candidates[:1], maddr)
+			ok, err = sectorbuilder.ProofVerifier.VerifyElectionPost(context.TODO(), sectorSize, sinfos, challenge[:], proof2, candidates[:1], maddr)
 			if err != nil {
 				return err
 			}

cmd/lotus-chainwatch/main.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 	"os"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
 

cmd/lotus-fountain/main.go

@@ -11,7 +11,7 @@ import (
 
 	rice "github.com/GeertJohan/go.rice"
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	peer "github.com/libp2p/go-libp2p-peer"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"

cmd/lotus-seal-worker/main.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/mitchellh/go-homedir"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"
 

cmd/lotus-seal-worker/sub.go

@@ -103,7 +103,7 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask)
 
 	switch task.Type {
 	case sectorbuilder.WorkerPreCommit:
-		rspco, err := w.sb.SealPreCommit(task.SectorID, task.SealTicket, task.Pieces)
+		rspco, err := w.sb.SealPreCommit(ctx, task.SectorID, task.SealTicket, task.Pieces)
 		if err != nil {
 			return errRes(xerrors.Errorf("precomitting: %w", err))
 		}
@@ -117,7 +117,7 @@ func (w *worker) processTask(ctx context.Context, task sectorbuilder.WorkerTask)
 			return errRes(xerrors.Errorf("pushing precommited data: %w", err))
 		}
 	case sectorbuilder.WorkerCommit:
-		proof, err := w.sb.SealCommit(task.SectorID, task.SealTicket, task.SealSeed, task.Pieces, task.Rspco)
+		proof, err := w.sb.SealCommit(ctx, task.SectorID, task.SealTicket, task.SealSeed, task.Pieces, task.Rspco)
 		if err != nil {
 			return errRes(xerrors.Errorf("comitting: %w", err))
 		}

cmd/lotus-seed/main.go

@@ -10,8 +10,8 @@ import (
 	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
-	badger "github.com/ipfs/go-ds-badger"
+	badger "github.com/ipfs/go-ds-badger2"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/mitchellh/go-homedir"
 	"golang.org/x/xerrors"
 	"gopkg.in/urfave/cli.v2"

cmd/lotus-seed/seed/seed.go

@@ -13,8 +13,8 @@ import (
 	sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
-	badger "github.com/ipfs/go-ds-badger"
+	badger "github.com/ipfs/go-ds-badger2"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-address"
@@ -71,7 +71,7 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb
 
 		fmt.Printf("sector-id: %d, piece info: %v", sid, pi)
 
-		pco, err := sb.SealPreCommit(sid, ticket, []sectorbuilder.PublicPieceInfo{pi})
+		pco, err := sb.SealPreCommit(context.TODO(), sid, ticket, []sectorbuilder.PublicPieceInfo{pi})
 		if err != nil {
 			return nil, xerrors.Errorf("commit: %w", err)
 		}

cmd/lotus-shed/main.go

@@ -3,7 +3,7 @@ package main
 import (
 	"os"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"gopkg.in/urfave/cli.v2"
 
 	"github.com/filecoin-project/lotus/build"

cmd/lotus-storage-miner/init.go

@@ -14,7 +14,7 @@ import (
 	paramfetch "github.com/filecoin-project/go-paramfetch"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
-	badger "github.com/ipfs/go-ds-badger"
+	badger "github.com/ipfs/go-ds-badger2"
 	"github.com/libp2p/go-libp2p-core/crypto"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/mitchellh/go-homedir"
@@ -22,15 +22,17 @@ import (
 	"gopkg.in/urfave/cli.v2"
 
 	"github.com/filecoin-project/go-address"
-	"github.com/filecoin-project/go-cbor-util"
+	cborutil "github.com/filecoin-project/go-cbor-util"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
 	"github.com/filecoin-project/go-sectorbuilder"
 	lapi "github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
 	"github.com/filecoin-project/lotus/chain/actors"
-	"github.com/filecoin-project/lotus/chain/deals"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/genesis"
+	"github.com/filecoin-project/lotus/markets/utils"
 	"github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node/modules"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
@@ -282,15 +284,21 @@ func migratePreSealMeta(ctx context.Context, api lapi.FullNode, presealDir strin
 
 		dealKey := datastore.NewKey(deals.ProviderDsPrefix).ChildString(proposalCid.String())
 
+		proposal, err := utils.ToSharedStorageDealProposal(&sector.Deal)
+		if err != nil {
+			return err
+		}
 		deal := &deals.MinerDeal{
-			Proposal:    sector.Deal,
+			MinerDeal: storagemarket.MinerDeal{
-			ProposalCid: proposalCid,
+				Proposal:    *proposal,
-			State:       lapi.DealComplete,
+				ProposalCid: proposalCid,
-			Ref:         proposalCid, // TODO: This is super wrong, but there
+				State:       lapi.DealComplete,
-			// are no params for CommP CIDs, we can't recover unixfs cid easily,
+				Ref:         proposalCid, // TODO: This is super wrong, but there
-			// and this isn't even used after the deal enters Complete state
+				// are no params for CommP CIDs, we can't recover unixfs cid easily,
-			DealID:   dealID,
+				// and this isn't even used after the deal enters Complete state
-			SectorID: sector.SectorID,
+				DealID:   dealID,
+				SectorID: sector.SectorID,
+			},
 		}
 
 		b, err = cborutil.Dump(deal)

cmd/lotus-storage-miner/main.go

@@ -3,7 +3,7 @@ package main
 import (
 	"os"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"go.opencensus.io/trace"
 	"gopkg.in/urfave/cli.v2"
 

cmd/lotus-townhall/main.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"strings"
 	"time"
 
 	rice "github.com/GeertJohan/go.rice"
@@ -16,17 +15,20 @@ import (
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	"github.com/libp2p/go-libp2p"
 	"github.com/libp2p/go-libp2p-core/peer"
-	pnet "github.com/libp2p/go-libp2p-pnet"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
 
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/node/modules/lp2p"
 )
 
 var topic = "/fil/headnotifs/"
 
 func init() {
 	genBytes := build.MaybeGenesis()
+	if len(genBytes) == 0 {
+		topic = ""
+		return
+	}
+
 	bs := blockstore.NewBlockstore(datastore.NewMapDatastore())
 
 	c, err := car.LoadCar(bs, bytes.NewReader(genBytes))
@@ -49,17 +51,16 @@ var upgrader = websocket.Upgrader{
 }
 
 func main() {
-	ctx := context.Background()
+	if topic == "" {
-
+		fmt.Println("FATAL: No genesis found")
-	protec, err := pnet.NewProtector(strings.NewReader(lp2p.LotusKey))
+		return
-	if err != nil {
-		panic(err)
 	}
 
+	ctx := context.Background()
+
 	host, err := libp2p.New(
 		ctx,
 		libp2p.Defaults,
-		libp2p.PrivateNetwork(protec),
 	)
 	if err != nil {
 		panic(err)

cmd/lotus/rpc.go

@@ -17,7 +17,7 @@ import (
 	"github.com/filecoin-project/lotus/node/impl"
 
 	"github.com/ipfs/go-cid"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr-net"
 	"golang.org/x/xerrors"

datatransfer/dagservice_impl.go

@@ -1,78 +0,0 @@
-package datatransfer
-
-import (
-	"context"
-	"reflect"
-
-	"github.com/ipfs/go-cid"
-	ipldformat "github.com/ipfs/go-ipld-format"
-	"github.com/ipfs/go-merkledag"
-	ipld "github.com/ipld/go-ipld-prime"
-	"github.com/libp2p/go-libp2p-core/peer"
-	"golang.org/x/xerrors"
-)
-
-// This file implements a VERY simple, incomplete version of the data transfer
-// module that allows us to make the necessary insertions of data transfer
-// functionality into the storage market
-// It does not:
-// -- actually validate requests
-// -- support Push requests
-// -- support multiple subscribers
-// -- do any actual network coordination or use Graphsync
-
-type dagserviceImpl struct {
-	dag        ipldformat.DAGService
-	subscriber Subscriber
-}
-
-// NewDAGServiceDataTransfer returns a data transfer manager based on
-// an IPLD DAGService
-func NewDAGServiceDataTransfer(dag ipldformat.DAGService) Manager {
-	return &dagserviceImpl{dag, nil}
-}
-
-// RegisterVoucherType registers a validator for the given voucher type
-// will error if voucher type does not implement voucher
-// or if there is a voucher type registered with an identical identifier
-func (impl *dagserviceImpl) RegisterVoucherType(voucherType reflect.Type, validator RequestValidator) error {
-	return nil
-}
-
-// open a data transfer that will send data to the recipient peer and
-// transfer parts of the piece that match the selector
-func (impl *dagserviceImpl) OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, Selector ipld.Node) (ChannelID, error) {
-	return ChannelID{}, xerrors.Errorf("not implemented")
-}
-
-// open a data transfer that will request data from the sending peer and
-// transfer parts of the piece that match the selector
-func (impl *dagserviceImpl) OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, Selector ipld.Node) (ChannelID, error) {
-	ctx, cancel := context.WithCancel(ctx)
-	go func() {
-		defer cancel()
-		err := merkledag.FetchGraph(ctx, baseCid, impl.dag)
-		var event Event
-		if err != nil {
-			event = Error
-		} else {
-			event = Complete
-		}
-		impl.subscriber(event, ChannelState{Channel: Channel{voucher: voucher}})
-	}()
-	return ChannelID{}, nil
-}
-
-// close an open channel (effectively a cancel)
-func (impl *dagserviceImpl) CloseDataTransferChannel(x ChannelID) {}
-
-// get status of a transfer
-func (impl *dagserviceImpl) TransferChannelStatus(x ChannelID) Status { return ChannelNotFoundError }
-
-// get notified when certain types of events happen
-func (impl *dagserviceImpl) SubscribeToEvents(subscriber Subscriber) {
-	impl.subscriber = subscriber
-}
-
-// get all in progress transfers
-func (impl *dagserviceImpl) InProgressChannels() map[ChannelID]ChannelState { return nil }

datatransfer/types.go

@@ -1,173 +0,0 @@
-package datatransfer
-
-import (
-	"context"
-	"reflect"
-
-	"github.com/ipfs/go-cid"
-	ipld "github.com/ipld/go-ipld-prime"
-	"github.com/libp2p/go-libp2p-core/peer"
-)
-
-// Voucher is used to validate
-// a data transfer request against the underlying storage or retrieval deal
-// that precipitated it. The only requirement is a voucher can read and write
-// from bytes, and has a string identifier type
-type Voucher interface {
-	// ToBytes converts the Voucher to raw bytes
-	ToBytes() ([]byte, error)
-	// FromBytes reads a Voucher from raw bytes
-	FromBytes([]byte) error
-	// Identifier is a unique string identifier for this voucher type
-	Identifier() string
-}
-
-// Status is the status of transfer for a given channel
-type Status int
-
-const (
-	// Ongoing means the data transfer is in progress
-	Ongoing Status = iota
-
-	// Completed means the data transfer is completed successfully
-	Completed
-
-	// Failed means the data transfer failed
-	Failed
-
-	// ChannelNotFoundError means the searched for data transfer does not exist
-	ChannelNotFoundError
-)
-
-// TransferID is an identifier for a data transfer, shared between
-// request/responder and unique to the requester
-type TransferID uint64
-
-// ChannelID is a unique identifier for a channel, distinct by both the other
-// party's peer ID + the transfer ID
-type ChannelID struct {
-	to peer.ID
-	id TransferID
-}
-
-// Channel represents all the parameters for a single data transfer
-type Channel struct {
-	// an identifier for this channel shared by request and responder, set by requester through protocol
-	transferID TransferID
-	// base CID for the piece being transferred
-	baseCid cid.Cid
-	// portion of Piece to return, specified by an IPLD selector
-	selector ipld.Node
-	// used to verify this channel
-	voucher Voucher
-	// the party that is sending the data (not who initiated the request)
-	sender peer.ID
-	// the party that is receiving the data (not who initiated the request)
-	recipient peer.ID
-	// expected amount of data to be transferred
-	totalSize uint64
-}
-
-// TransferID returns the transfer id for this channel
-func (c Channel) TransferID() TransferID { return c.transferID }
-
-// BaseCID returns the CID that is at the root of this data transfer
-func (c Channel) BaseCID() cid.Cid { return c.baseCid }
-
-// Selector returns the IPLD selector for this data transfer (represented as
-// an IPLD node)
-func (c Channel) Selector() ipld.Node { return c.selector }
-
-// Voucher returns the voucher for this data transfer
-func (c Channel) Voucher() Voucher { return c.voucher }
-
-// Sender returns the peer id for the node that is sending data
-func (c Channel) Sender() peer.ID { return c.sender }
-
-// Recipient returns the peer id for the node that is receiving data
-func (c Channel) Recipient() peer.ID { return c.recipient }
-
-// TotalSize returns the total size for the data being transferred
-func (c Channel) TotalSize() uint64 { return c.totalSize }
-
-// ChannelState is immutable channel data plus mutable state
-type ChannelState struct {
-	Channel
-	// total bytes sent from this node (0 if receiver)
-	sent uint64
-	// total bytes received by this node (0 if sender)
-	received uint64
-}
-
-// Sent returns the number of bytes sent
-func (c ChannelState) Sent() uint64 { return c.sent }
-
-// Received returns the number of bytes received
-func (c ChannelState) Received() uint64 { return c.received }
-
-// Event is a name for an event that occurs on a data transfer channel
-type Event int
-
-const (
-	// Open is an event occurs when a channel is first opened
-	Open Event = iota
-
-	// Progress is an event that gets emitted every time more data is transferred
-	Progress
-
-	// Error is an event that emits when an error occurs in a data transfer
-	Error
-
-	// Complete is emitted when a data transfer is complete
-	Complete
-)
-
-// Subscriber is a callback that is called when events are emitted
-type Subscriber func(event Event, channelState ChannelState)
-
-// RequestValidator is an interface implemented by the client of the
-// data transfer module to validate requests
-type RequestValidator interface {
-	// ValidatePush validates a push request received from the peer that will send data
-	ValidatePush(
-		sender peer.ID,
-		voucher Voucher,
-		baseCid cid.Cid,
-		selector ipld.Node) error
-	// ValidatePull validates a pull request received from the peer that will receive data
-	ValidatePull(
-		receiver peer.ID,
-		voucher Voucher,
-		baseCid cid.Cid,
-		selector ipld.Node) error
-}
-
-// Manager is the core interface presented by all implementations of
-// of the data transfer sub system
-type Manager interface {
-	// RegisterVoucherType registers a validator for the given voucher type
-	// will error if voucher type does not implement voucher
-	// or if there is a voucher type registered with an identical identifier
-	RegisterVoucherType(voucherType reflect.Type, validator RequestValidator) error
-
-	// open a data transfer that will send data to the recipient peer and
-	// open a data transfer that will send data to the recipient peer and
-	// transfer parts of the piece that match the selector
-	OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error)
-
-	// open a data transfer that will request data from the sending peer and
-	// transfer parts of the piece that match the selector
-	OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error)
-
-	// close an open channel (effectively a cancel)
-	CloseDataTransferChannel(x ChannelID)
-
-	// get status of a transfer
-	TransferChannelStatus(x ChannelID) Status
-
-	// get notified when certain types of events happen
-	SubscribeToEvents(subscriber Subscriber)
-
-	// get all in progress transfers
-	InProgressChannels() map[ChannelID]ChannelState
-}

documentation/en/join-testnet.md

@@ -47,16 +47,16 @@ lotus sync wait
 
 ## Create your first address
 
-Initialize a wallet using BLS signature formats:
+Initialize a new wallet:
 
 ```sh
-lotus wallet new bls
+lotus wallet new 
 ```
 
 Here is an example of the response:
 
 ```sh
-t3vhfme4qfvegqaz7m7q6o6afjcs67n6kpzv7t2eozio4chwpafwa2y4l7zhwd5eom7jmihzdg4s52dpvnclza
+t1aswwvjsae63tcrniz6x5ykvsuotlgkvlulnqpsi
 ```
 
 - Visit the [faucet](https://lotus-faucet.kittyhawk.wtf/funds.html) to add funds.

gen/main.go

@@ -9,10 +9,8 @@ import (
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/blocksync"
-	"github.com/filecoin-project/lotus/chain/deals"
 	"github.com/filecoin-project/lotus/chain/types"
 	"github.com/filecoin-project/lotus/paych"
-	"github.com/filecoin-project/lotus/retrieval"
 	"github.com/filecoin-project/lotus/storage"
 )
 
@@ -59,21 +57,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	err = gen.WriteTupleEncodersToFile("./retrieval/cbor_gen.go", "retrieval",
-		retrieval.RetParams{},
-
-		retrieval.Query{},
-		retrieval.QueryResponse{},
-		retrieval.Unixfs0Offer{},
-		retrieval.DealProposal{},
-		retrieval.DealResponse{},
-		retrieval.Block{},
-	)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
 	err = gen.WriteTupleEncodersToFile("./chain/blocksync/cbor_gen.go", "blocksync",
 		blocksync.BlockSyncRequest{},
 		blocksync.BlockSyncResponse{},
@@ -138,22 +121,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	err = gen.WriteTupleEncodersToFile("./chain/deals/cbor_gen.go", "deals",
-		deals.AskRequest{},
-		deals.AskResponse{},
-		deals.Proposal{},
-		deals.Response{},
-		deals.SignedResponse{},
-		deals.ClientDealProposal{},
-		deals.ClientDeal{},
-		deals.MinerDeal{},
-		deals.StorageDataTransferVoucher{},
-	)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-
 	err = gen.WriteMapEncodersToFile("./storage/cbor_gen.go", "storage",
 		storage.SealTicket{},
 		storage.SealSeed{},

go.mod

@@ -15,12 +15,13 @@ require (
 	github.com/filecoin-project/go-amt-ipld v0.0.0-20191205011053-79efc22d6cdc
 	github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
+	github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce
+	github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8
 	github.com/filecoin-project/go-paramfetch v0.0.1
-	github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013
+	github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689
-	github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5
+	github.com/filecoin-project/go-statestore v0.1.0
 	github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1
 	github.com/go-ole/go-ole v1.2.4 // indirect
-	github.com/google/go-cmp v0.3.1 // indirect
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.1
 	github.com/hashicorp/go-multierror v1.0.0
@@ -29,15 +30,15 @@ require (
 	github.com/ipfs/go-bitswap v0.1.8
 	github.com/ipfs/go-block-format v0.0.2
 	github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c
-	github.com/ipfs/go-car v0.0.2
+	github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1
 	github.com/ipfs/go-cid v0.0.4
-	github.com/ipfs/go-datastore v0.1.1
+	github.com/ipfs/go-datastore v0.3.1
-	github.com/ipfs/go-ds-badger v0.0.7
+	github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521
 	github.com/ipfs/go-filestore v0.0.2
 	github.com/ipfs/go-fs-lock v0.0.1
+	github.com/ipfs/go-graphsync v0.0.4
 	github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1
 	github.com/ipfs/go-ipfs-blockstore v0.1.1
-	github.com/ipfs/go-ipfs-blocksutil v0.0.1
 	github.com/ipfs/go-ipfs-chunker v0.0.1
 	github.com/ipfs/go-ipfs-ds-help v0.0.1
 	github.com/ipfs/go-ipfs-exchange-interface v0.0.1
@@ -47,11 +48,10 @@ require (
 	github.com/ipfs/go-ipld-cbor v0.0.3
 	github.com/ipfs/go-ipld-format v0.0.2
 	github.com/ipfs/go-log v1.0.1
-	github.com/ipfs/go-log/v2 v2.0.2 // indirect
+	github.com/ipfs/go-log/v2 v2.0.2
 	github.com/ipfs/go-merkledag v0.2.4
 	github.com/ipfs/go-path v0.0.7
 	github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb
-	github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b
 	github.com/libp2p/go-libp2p v0.4.2
 	github.com/libp2p/go-libp2p-circuit v0.1.4
 	github.com/libp2p/go-libp2p-connmgr v0.1.0
@@ -61,7 +61,6 @@ require (
 	github.com/libp2p/go-libp2p-mplex v0.2.1
 	github.com/libp2p/go-libp2p-peer v0.2.0
 	github.com/libp2p/go-libp2p-peerstore v0.1.4
-	github.com/libp2p/go-libp2p-pnet v0.1.0
 	github.com/libp2p/go-libp2p-pubsub v0.2.3
 	github.com/libp2p/go-libp2p-quic-transport v0.1.1
 	github.com/libp2p/go-libp2p-record v0.1.1
@@ -80,14 +79,12 @@ require (
 	github.com/multiformats/go-multiaddr-net v0.1.1
 	github.com/multiformats/go-multihash v0.0.10
 	github.com/multiformats/go-varint v0.0.2
-	github.com/onsi/ginkgo v1.9.0 // indirect
-	github.com/onsi/gomega v1.6.0 // indirect
 	github.com/opentracing/opentracing-go v1.1.0
 	github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a
 	github.com/prometheus/common v0.2.0
 	github.com/stretchr/testify v1.4.0
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba
-	github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0
+	github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe
 	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
 	github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d
 	go.opencensus.io v0.22.2

go.sum

@@ -7,6 +7,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkBy
 github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
+github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg=
 github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
 github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ=
@@ -14,6 +16,8 @@ github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtix
 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee h1:8doiS7ib3zi6/K172oDhSKU0dJ/miJramo9NITOMyZQ=
 github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee/go.mod h1:W0GbEAA4uFNYOGG2cJpmFJ04E6SD1NLELPYZB57/7AY=
 github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
+github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
@@ -34,6 +38,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8=
 github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
 github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
+github.com/btcsuite/btcd v0.0.0-20190629003639-c26ffa870fd8/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
 github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3 h1:A/EVblehb75cUgXA5njHPn0kLAsykn6mJGz7rnmW5W0=
 github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
@@ -44,6 +49,8 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa
 github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -53,7 +60,10 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-semver v0.2.1-0.20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0=
 github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis=
 github.com/daaku/go.zipexe v1.0.0 h1:VSOgZtH418pH9L16hC/JrgSNJbbAL26pj7lmD1+CGdY=
@@ -62,12 +72,14 @@ github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0=
-github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
 github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
 github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
 github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
+github.com/dgraph-io/badger/v2 v2.0.1 h1:+D6dhIqC6jIeCclnxMHqk4HPuXgrRN5UfBsLR4dNQ3A=
+github.com/dgraph-io/badger/v2 v2.0.1/go.mod h1:YoRSIp1LmAJ7zH7tZwRvjNMUYLxB4wl3ebYkaIruZ04=
+github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e h1:aeUNgwup7PnDOBAD1BOKAqzb/W/NksOj6r3dwKKuqfg=
+github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs=
 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
@@ -95,16 +107,23 @@ github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMX
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ=
+github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY=
+github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww=
+github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8 h1:g3oodvSz+Ou+ObwcVBB2wyt8SHdWpwzMiNJ19U1zZNA=
+github.com/filecoin-project/go-fil-markets v0.0.0-20200114015428-74d100f305f8/go.mod h1:c8NTjvFVy1Ud02mmGDjOiMeawY2t6ALfrrdvAB01FQc=
 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878 h1:YicJT9xhPzZ1SBGiJFNUCkfwqK/G9vFyY1ytKBSjNJA=
 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
 github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
+github.com/filecoin-project/go-paramfetch v0.0.0-20200102181131-b20d579f2878/go.mod h1:40kI2Gv16mwcRsHptI3OAV4nlOEU7wVDc4RgMylNFjU=
 github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyCXUE0rimz4L7ghoE=
 github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
-github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013 h1:OGpRq3HRxyrxZJtbNKCOsb5YTmc+RBLLwdAgwZfkRnY=
+github.com/filecoin-project/go-sectorbuilder v0.0.1 h1:yiLSEprWA1E43DFTSCXLSuCstYuDKiI6RCXiYz4GaRs=
-github.com/filecoin-project/go-sectorbuilder v0.0.0-20200107152336-0cbb2c483013/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
+github.com/filecoin-project/go-sectorbuilder v0.0.1/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
-github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5 h1:NZXq90YlfakSmB2/84dGr0AVmKYFA97+yyViBIgTFbk=
+github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689 h1:2cT5bhm/5I0RY+HBIPdRRrtjCwLj33Qx6DHRs9TCslY=
-github.com/filecoin-project/go-statestore v0.0.0-20200102200712-1f63c701c1e5/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
+github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200114015900-4103afa82689/go.mod h1:3OZ4E3B2OuwhJjtxR4r7hPU9bCfB+A+hm4alLEsaeDc=
+github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ=
+github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/gbrlsnchs/jwt/v3 v3.0.0-beta.1 h1:EzDjxMg43q1tA2c0MV3tNbaontnHLplHyFF6M5KiVP0=
@@ -130,9 +149,12 @@ github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
-github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
@@ -154,6 +176,8 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU=
 github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
 github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE=
+github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c h1:+MSf4NEnLCYZoAgK6fqwc7NH88nM8haFSxKGUGIG3vA=
+github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
@@ -188,8 +212,8 @@ github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbR
 github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c h1:lN5IQA07VtLiTLAp/Scezp1ljFhXErC6yq4O1cu+yJ0=
 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
-github.com/ipfs/go-car v0.0.2 h1:j02lzgeijorstzoMl3nQmvvb8wjJUVCiOAl8XEwYMCQ=
+github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 h1:Nq8xEW+2KZq7IkRlkOh0rTEUI8FgunhMoLj5EMkJzbQ=
-github.com/ipfs/go-car v0.0.2/go.mod h1:60pzeu308k5kVFHzq0HIi2kPtITgor+1ll1xuGk5JwQ=
+github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1/go.mod h1:rmd887mJxQRDfndfDEY3Liyx8gQVyfFFRSHdsnDSAlk=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
@@ -201,18 +225,24 @@ github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAK
 github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
 github.com/ipfs/go-datastore v0.1.1 h1:F4k0TkTAZGLFzBOrVKDAvch6JZtuN4NHkfdcEZL50aI=
 github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
+github.com/ipfs/go-datastore v0.3.1 h1:SS1t869a6cctoSYmZXUk8eL6AzVXgASmKIWFNQkQ1jU=
+github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
 github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
 github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
 github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
 github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s=
 github.com/ipfs/go-ds-badger v0.0.7 h1:NMyh88Q50HG6/S2YD58DLkq0c0/ZQPMbSojONH+PRf4=
 github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk=
+github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521 h1:GSygNXXIRXFVntMR3Yr30imf40H9cYDAw72Qb0za2nE=
+github.com/ipfs/go-ds-badger2 v0.0.0-20200108185345-7f650e6b2521/go.mod h1:oDKWqmQbLyyml3mgMtHgB7qAf6cQOEsJPRLFzfwKQ5Q=
 github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
 github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8=
 github.com/ipfs/go-filestore v0.0.2 h1:pcYwpjtXXwirtbjBXKVJM9CTa9F7/8v1EkfnDaHTO3s=
 github.com/ipfs/go-filestore v0.0.2/go.mod h1:KnZ41qJsCt2OX2mxZS0xsK3Psr0/oB93HMMssLujjVc=
 github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0=
 github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y=
+github.com/ipfs/go-graphsync v0.0.4 h1:iF98+J8pcqvEb48IM0TemqeGARsCDtwQ73P9ejMZIuU=
+github.com/ipfs/go-graphsync v0.0.4/go.mod h1:6UACBjfOXEa8rQL3Q/JpZpWS0nZDCLx134WUkjrmFpQ=
 github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1 h1:0xpCaaXvN8bPyws3ObiCn7G0KIfgbS9E132eL57dHx8=
 github.com/ipfs/go-hamt-ipld v0.0.14-0.20191218031521-b2c774a54db1/go.mod h1:8yRx0xLUps1Xq8ZDnIwIVdQRp7JjA55gGvCiRHT91Vk=
 github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08=
@@ -255,8 +285,12 @@ github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9
 github.com/ipfs/go-log v1.0.0 h1:BW3LQIiZzpNyolt84yvKNCd3FU+AK4VDw1hnHR+1aiI=
 github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA=
 github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo=
+github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo=
+github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I=
 github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I=
 github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM=
+github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM=
+github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
 github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
 github.com/ipfs/go-log/v2 v2.0.2 h1:xguurydRdfKMJjKyxNXNU8lYP0VZH1NUwJRwUorjuEw=
 github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0=
@@ -279,8 +313,10 @@ github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb h1:tmWYgjltxwM7PD
 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
-github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b h1:ACSEK4f1SDQC+FJ4B4pqHFW14d7kEW2ufwXA/c7eLP0=
+github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 h1:fASnkvtR+SmB2y453RxmDD3Uvd4LonVUgFGk9JoDaZs=
-github.com/ipld/go-ipld-prime v0.0.2-0.20191025154717-8dff1cbec43b/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
+github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
+github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 h1:lSip43rAdyGA+yRQuy6ju0ucZkWpYc1F2CTQtZTVW/4=
+github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
 github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
@@ -291,6 +327,8 @@ github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+
 github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs=
 github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc=
 github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA=
+github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c h1:uUx61FiAa1GI6ZmVd2wf2vULeQZIKG66eybjNXKYCz4=
+github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c/go.mod h1:sdx1xVM9UuLw1tXnhJWN3piypTUO3vCIHYmG15KE/dU=
 github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2 h1:vhC1OXXiT9R2pczegwz6moDvuRpggaroAXhPIseh57A=
 github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs=
 github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
@@ -329,6 +367,8 @@ github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40J
 github.com/libp2p/go-conn-security-multistream v0.0.2/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE=
 github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0=
 github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc=
+github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk=
+github.com/libp2p/go-eventbus v0.0.3/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk=
 github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvMfzQ=
 github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4=
 github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s=
@@ -336,6 +376,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ
 github.com/libp2p/go-libp2p v0.0.30/go.mod h1:XWT8FGHlhptAv1+3V/+J5mEpzyui/5bvFsNuWYs611A=
 github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM=
 github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8=
+github.com/libp2p/go-libp2p v0.2.1/go.mod h1:HZbtEOrgZN4F1fGZVvkV+930Wx3DkqlpBlO8dIoZWds=
+github.com/libp2p/go-libp2p v0.3.0/go.mod h1:J7DPB1+zB5VLc8v/kKSD8+u2cbyIGI0Dh/Pf3Wprt+0=
 github.com/libp2p/go-libp2p v0.4.2 h1:p0cthB0jDNHO4gH2HzS8/nAMMXbfUlFHs0jwZ4U+F2g=
 github.com/libp2p/go-libp2p v0.4.2/go.mod h1:MNmgUxUw5pMsdOzMlT0EE7oKjRasl+WyVwM0IBlpKgQ=
 github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE=
@@ -344,10 +386,12 @@ github.com/libp2p/go-libp2p-autonat v0.1.1 h1:WLBZcIRsjZlWdAZj9CiBSvU2wQXoUOiS1Z
 github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE=
 github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc=
 github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro=
+github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg=
 github.com/libp2p/go-libp2p-blankhost v0.1.4 h1:I96SWjR4rK9irDHcHq3XHN6hawCRTPUADzkJacgZLvk=
 github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU=
 github.com/libp2p/go-libp2p-circuit v0.0.9/go.mod h1:uU+IBvEQzCu953/ps7bYzC/D/R0Ho2A9LfKVVCatlqU=
 github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8=
+github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8=
 github.com/libp2p/go-libp2p-circuit v0.1.4 h1:Phzbmrg3BkVzbqd4ZZ149JxCuUWu2wZcXf/Kr6hZJj8=
 github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU=
 github.com/libp2p/go-libp2p-connmgr v0.1.0 h1:vp0t0F0EuT3rrlTtnMnIyyzCnly7nIlRoEbhJpgp0qU=
@@ -356,6 +400,8 @@ github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGc
 github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco=
 github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE=
 github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I=
+github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE=
+github.com/libp2p/go-libp2p-core v0.0.9/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE=
 github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI=
 github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0=
 github.com/libp2p/go-libp2p-core v0.2.4 h1:Et6ykkTwI6PU44tr8qUF9k43vP0aduMNniShAbUJJw8=
@@ -401,11 +447,11 @@ github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMg
 github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20=
 github.com/libp2p/go-libp2p-peerstore v0.0.6/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20=
 github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY=
+github.com/libp2p/go-libp2p-peerstore v0.1.2/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI=
+github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+84VV9glZq1M=
 github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI=
 github.com/libp2p/go-libp2p-peerstore v0.1.4 h1:d23fvq5oYMJ/lkkbO4oTwBp/JP+I/1m5gZJobNXCE/k=
 github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs=
-github.com/libp2p/go-libp2p-pnet v0.1.0 h1:kRUES28dktfnHNIRW4Ro78F7rKBHBiw5MJpl0ikrLIA=
-github.com/libp2p/go-libp2p-pnet v0.1.0/go.mod h1:ZkyZw3d0ZFOex71halXRihWf9WH/j3OevcJdTmD0lyE=
 github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s=
 github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk=
 github.com/libp2p/go-libp2p-pubsub v0.2.3 h1:qJRnRnM7Z4xnHb4i6EBb3DKQXRPgtFWlKP4AmfJudLQ=
@@ -423,11 +469,15 @@ github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu
 github.com/libp2p/go-libp2p-routing-helpers v0.1.0/go.mod h1:oUs0h39vNwYtYXnQWOTU5BaafbedSyWCCal3gqHuoOQ=
 github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVpm+j2/S2A5UqYb0=
 github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
+github.com/libp2p/go-libp2p-secio v0.1.1/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
+github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng=
 github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
 github.com/libp2p/go-libp2p-secio v0.2.1 h1:eNWbJTdyPA7NxhP7J3c5lT97DC5d+u+IldkgCYFTPVA=
 github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
 github.com/libp2p/go-libp2p-swarm v0.0.6/go.mod h1:s5GZvzg9xXe8sbeESuFpjt8CJPTCa8mhEusweJqyFy8=
 github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4=
+github.com/libp2p/go-libp2p-swarm v0.1.1/go.mod h1:4NVJaLwq/dr5kEq79Jo6pMin7ZFwLx73ln1FTefR91Q=
+github.com/libp2p/go-libp2p-swarm v0.2.0/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU=
 github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ=
 github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU=
 github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E=
@@ -471,6 +521,7 @@ github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FW
 github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA=
 github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4=
 github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs=
+github.com/libp2p/go-stream-muxer v0.0.1 h1:Ce6e2Pyu+b5MC1k3eeFtAax0pW4gc6MosYSLV05UeLw=
 github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14=
 github.com/libp2p/go-stream-muxer v0.1.0/go.mod h1:8JAVsjeRBCWwPoZeH0W1imLOcriqXJyFvB0mR4A04sQ=
 github.com/libp2p/go-stream-muxer-multistream v0.1.1/go.mod h1:zmGdfkQ1AzOECIAcccoL8L//laqawOsO03zX8Sa+eGw=
@@ -541,6 +592,7 @@ github.com/multiformats/go-multiaddr v0.1.1 h1:rVAztJYMhCQ7vEFr8FvxW3mS+HF2eY/oP
 github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
 github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
+github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA=
 github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0=
 github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q=
@@ -554,6 +606,7 @@ github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmr
 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
 github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
+github.com/multiformats/go-multihash v0.0.6/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM=
 github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM=
@@ -609,9 +662,14 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM=
 github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -620,6 +678,7 @@ github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHei
 github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
@@ -637,6 +696,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -644,7 +704,10 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli/v2 v2.0.0 h1:+HU9SCbu8GnEUFtIBfuUNXN39ofWViIEJIp6SURMpCg=
+github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
@@ -659,8 +722,11 @@ github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CH
 github.com/whyrusleeping/cbor-gen v0.0.0-20190910031516-c1cbffdb01bb/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
 github.com/whyrusleeping/cbor-gen v0.0.0-20190917003517-d78d67427694/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
 github.com/whyrusleeping/cbor-gen v0.0.0-20191116002219-891f55cd449d/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
+github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
 github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 h1:efb/4CnrubzNGqQOeHErxyQ6rIsJb7GcgeSDF7fqWeI=
 github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
+github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe h1:n7En1uyDtknjLRDXebWlPGJoHvwL8AkNcSQzuOoYYYQ=
+github.com/whyrusleeping/cbor-gen v0.0.0-20200106232624-282db0d37dbe/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k=
@@ -685,6 +751,8 @@ github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7V
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50=
 go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
 go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -725,6 +793,8 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -751,11 +821,15 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -782,6 +856,7 @@ golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -806,9 +881,10 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs=
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361 h1:RIIXAeV6GvDBuADKumTODatUqANFZ+5BPMnzsy4hulY=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200108195415-316d2f248479 h1:csuS+MHeEA2eWhyjQCMaPMq4z1+/PohkBSjJZHSIbOE=
 golang.org/x/tools v0.0.0-20200108195415-316d2f248479/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -827,6 +903,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=

lib/auth/handler.go

@@ -5,7 +5,7 @@ import (
 	"net/http"
 	"strings"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/api/apistruct"

lib/jsonrpc/client.go

@@ -12,7 +12,7 @@ import (
 	"sync/atomic"
 
 	"github.com/gorilla/websocket"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"go.opencensus.io/trace"
 	"go.opencensus.io/trace/propagation"
 	"golang.org/x/xerrors"

lib/lotuslog/levels.go

@@ -1,6 +1,6 @@
 package lotuslog
 
-import logging "github.com/ipfs/go-log"
+import logging "github.com/ipfs/go-log/v2"
 
 func SetupLogLevels() {
 	logging.SetLogLevel("*", "INFO")

lib/tarutil/systar.go

@@ -8,7 +8,7 @@ import (
 	"os"
 	"path/filepath"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 )
 
 var log = logging.Logger("tarutil")

markets/retrievaladapter/client.go

@@ -0,0 +1,49 @@
+package retrievaladapter
+
+import (
+	"context"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount"
+	retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types"
+	"github.com/filecoin-project/lotus/markets/utils"
+	payapi "github.com/filecoin-project/lotus/node/impl/paych"
+	"github.com/filecoin-project/lotus/paych"
+)
+
+type retrievalClientNode struct {
+	pmgr   *paych.Manager
+	payapi payapi.PaychAPI
+}
+
+// NewRetrievalClientNode returns a new node adapter for a retrieval client that talks to the
+// Lotus Node
+func NewRetrievalClientNode(pmgr *paych.Manager, payapi payapi.PaychAPI) retrievalmarket.RetrievalClientNode {
+	return &retrievalClientNode{pmgr: pmgr, payapi: payapi}
+}
+
+// GetOrCreatePaymentChannel sets up a new payment channel if one does not exist
+// between a client and a miner and insures the client has the given amount of funds available in the channel
+func (rcn *retrievalClientNode) GetOrCreatePaymentChannel(ctx context.Context, clientAddress address.Address, minerAddress address.Address, clientFundsAvailable retrievaltoken.TokenAmount) (address.Address, error) {
+	paych, _, err := rcn.pmgr.GetPaych(ctx, clientAddress, minerAddress, utils.FromSharedTokenAmount(clientFundsAvailable))
+	return paych, err
+}
+
+// Allocate late creates a lane within a payment channel so that calls to
+// CreatePaymentVoucher will automatically make vouchers only for the difference
+// in total
+func (rcn *retrievalClientNode) AllocateLane(paymentChannel address.Address) (uint64, error) {
+	return rcn.pmgr.AllocateLane(paymentChannel)
+}
+
+// CreatePaymentVoucher creates a new payment voucher in the given lane for a
+// given payment channel so that all the payment vouchers in the lane add up
+// to the given amount (so the payment voucher will be for the difference)
+func (rcn *retrievalClientNode) CreatePaymentVoucher(ctx context.Context, paymentChannel address.Address, amount retrievaltoken.TokenAmount, lane uint64) (*retrievaltypes.SignedVoucher, error) {
+	voucher, err := rcn.payapi.PaychVoucherCreate(ctx, paymentChannel, utils.FromSharedTokenAmount(amount), lane)
+	if err != nil {
+		return nil, err
+	}
+	return utils.ToSharedSignedVoucher(voucher)
+}

markets/retrievaladapter/provider.go

@@ -0,0 +1,48 @@
+package retrievaladapter
+
+import (
+	"context"
+
+	"github.com/ipfs/go-cid"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	retrievaltoken "github.com/filecoin-project/go-fil-markets/shared/tokenamount"
+	retrievaltypes "github.com/filecoin-project/go-fil-markets/shared/types"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/markets/utils"
+	"github.com/filecoin-project/lotus/storage/sectorblocks"
+)
+
+type retrievalProviderNode struct {
+	sectorBlocks *sectorblocks.SectorBlocks
+	full         api.FullNode
+}
+
+// NewRetrievalProviderNode returns a new node adapter for a retrieval provider that talks to the
+// Lotus Node
+func NewRetrievalProviderNode(sectorBlocks *sectorblocks.SectorBlocks, full api.FullNode) retrievalmarket.RetrievalProviderNode {
+	return &retrievalProviderNode{sectorBlocks, full}
+}
+
+func (rpn *retrievalProviderNode) GetPieceSize(pieceCid []byte) (uint64, error) {
+	asCid, err := cid.Cast(pieceCid)
+	if err != nil {
+		return 0, err
+	}
+	return rpn.sectorBlocks.GetSize(asCid)
+}
+
+func (rpn *retrievalProviderNode) SealedBlockstore(approveUnseal func() error) blockstore.Blockstore {
+	return rpn.sectorBlocks.SealedBlockstore(approveUnseal)
+}
+
+func (rpn *retrievalProviderNode) SavePaymentVoucher(ctx context.Context, paymentChannel address.Address, voucher *retrievaltypes.SignedVoucher, proof []byte, expectedAmount retrievaltoken.TokenAmount) (retrievaltoken.TokenAmount, error) {
+	localVoucher, err := utils.FromSharedSignedVoucher(voucher)
+	if err != nil {
+		return retrievaltoken.FromInt(0), err
+	}
+	added, err := rpn.full.PaychVoucherAdd(ctx, paymentChannel, localVoucher, proof, utils.FromSharedTokenAmount(expectedAmount))
+	return utils.ToSharedTokenAmount(added), err
+}

markets/storageadapter/client.go

@@ -0,0 +1,341 @@
+package storageadapter
+
+// this file implements storagemarket.StorageClientNode
+
+import (
+	"bytes"
+	"context"
+
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	cborutil "github.com/filecoin-project/go-cbor-util"
+	"github.com/filecoin-project/go-fil-markets/shared/tokenamount"
+	sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/build"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/events"
+	"github.com/filecoin-project/lotus/chain/market"
+	"github.com/filecoin-project/lotus/chain/stmgr"
+	"github.com/filecoin-project/lotus/chain/store"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/markets/utils"
+	"github.com/filecoin-project/lotus/node/impl/full"
+)
+
+type ClientNodeAdapter struct {
+	full.StateAPI
+	full.ChainAPI
+	full.MpoolAPI
+
+	sm *stmgr.StateManager
+	cs *store.ChainStore
+	fm *market.FundMgr
+	ev *events.Events
+}
+
+type clientApi struct {
+	full.ChainAPI
+	full.StateAPI
+}
+
+func NewClientNodeAdapter(state full.StateAPI, chain full.ChainAPI, mpool full.MpoolAPI, sm *stmgr.StateManager, cs *store.ChainStore, fm *market.FundMgr) storagemarket.StorageClientNode {
+	return &ClientNodeAdapter{
+		StateAPI: state,
+		ChainAPI: chain,
+		MpoolAPI: mpool,
+
+		sm: sm,
+		cs: cs,
+		fm: fm,
+		ev: events.NewEvents(context.TODO(), &clientApi{chain, state}),
+	}
+}
+
+func (n *ClientNodeAdapter) ListStorageProviders(ctx context.Context) ([]*storagemarket.StorageProviderInfo, error) {
+	ts, err := n.ChainHead(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	addresses, err := n.StateListMiners(ctx, ts)
+	if err != nil {
+		return nil, err
+	}
+
+	var out []*storagemarket.StorageProviderInfo
+
+	for _, addr := range addresses {
+		workerAddr, err := n.StateMinerWorker(ctx, addr, ts)
+		if err != nil {
+			return nil, err
+		}
+
+		sectorSize, err := n.StateMinerSectorSize(ctx, addr, ts)
+		if err != nil {
+			return nil, err
+		}
+
+		peerId, err := n.StateMinerPeerID(ctx, addr, ts)
+		if err != nil {
+			return nil, err
+		}
+		storageProviderInfo := utils.NewStorageProviderInfo(addr, workerAddr, sectorSize, peerId)
+		out = append(out, &storageProviderInfo)
+	}
+
+	return out, nil
+}
+
+func (n *ClientNodeAdapter) ListClientDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) {
+	allDeals, err := n.StateMarketDeals(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var out []storagemarket.StorageDeal
+
+	for _, deal := range allDeals {
+		storageDeal := utils.FromOnChainDeal(deal)
+		if storageDeal.Client == addr {
+			out = append(out, storageDeal)
+		}
+	}
+
+	return out, nil
+}
+
+func (n *ClientNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) {
+	return n.ChainHead(ctx)
+}
+
+// Adds funds with the StorageMinerActor for a storage participant.  Used by both providers and clients.
+func (n *ClientNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error {
+	// (Provider Node API)
+	smsg, err := n.MpoolPushMessage(ctx, &types.Message{
+		To:       actors.StorageMarketAddress,
+		From:     addr,
+		Value:    utils.FromSharedTokenAmount(amount),
+		GasPrice: types.NewInt(0),
+		GasLimit: types.NewInt(1000000),
+		Method:   actors.SMAMethods.AddBalance,
+	})
+	if err != nil {
+		return err
+	}
+
+	r, err := n.StateWaitMsg(ctx, smsg.Cid())
+	if err != nil {
+		return err
+	}
+
+	if r.Receipt.ExitCode != 0 {
+		return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode)
+	}
+
+	return nil
+}
+
+func (n *ClientNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error {
+	return n.fm.EnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amount))
+}
+
+func (n *ClientNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) {
+	bal, err := n.StateMarketBalance(ctx, addr, nil)
+	if err != nil {
+		return storagemarket.Balance{}, err
+	}
+
+	return utils.ToSharedBalance(bal), nil
+}
+
+// ValidatePublishedDeal validates that the provided deal has appeared on chain and references the same ClientDeal
+// returns the Deal id if there is no error
+func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal storagemarket.ClientDeal) (uint64, error) {
+	log.Infow("DEAL ACCEPTED!")
+
+	pubmsg, err := c.cs.GetMessage(*deal.PublishMessage)
+	if err != nil {
+		return 0, xerrors.Errorf("getting deal pubsish message: %w", err)
+	}
+
+	pw, err := stmgr.GetMinerWorker(ctx, c.sm, nil, deal.Proposal.Provider)
+	if err != nil {
+		return 0, xerrors.Errorf("getting miner worker failed: %w", err)
+	}
+
+	if pubmsg.From != pw {
+		return 0, xerrors.Errorf("deal wasn't published by storage provider: from=%s, provider=%s", pubmsg.From, deal.Proposal.Provider)
+	}
+
+	if pubmsg.To != actors.StorageMarketAddress {
+		return 0, xerrors.Errorf("deal publish message wasn't set to StorageMarket actor (to=%s)", pubmsg.To)
+	}
+
+	if pubmsg.Method != actors.SMAMethods.PublishStorageDeals {
+		return 0, xerrors.Errorf("deal publish message called incorrect method (method=%s)", pubmsg.Method)
+	}
+
+	var params actors.PublishStorageDealsParams
+	if err := params.UnmarshalCBOR(bytes.NewReader(pubmsg.Params)); err != nil {
+		return 0, err
+	}
+
+	dealIdx := -1
+	for i, storageDeal := range params.Deals {
+		// TODO: make it less hacky
+		sd := storageDeal
+		eq, err := cborutil.Equals(&deal.Proposal, &sd)
+		if err != nil {
+			return 0, err
+		}
+		if eq {
+			dealIdx = i
+			break
+		}
+	}
+
+	if dealIdx == -1 {
+		return 0, xerrors.Errorf("deal publish didn't contain our deal (message cid: %s)", deal.PublishMessage)
+	}
+
+	// TODO: timeout
+	_, ret, err := c.sm.WaitForMessage(ctx, *deal.PublishMessage)
+	if err != nil {
+		return 0, xerrors.Errorf("waiting for deal publish message: %w", err)
+	}
+	if ret.ExitCode != 0 {
+		return 0, xerrors.Errorf("deal publish failed: exit=%d", ret.ExitCode)
+	}
+
+	var res actors.PublishStorageDealResponse
+	if err := res.UnmarshalCBOR(bytes.NewReader(ret.Return)); err != nil {
+		return 0, err
+	}
+
+	return res.DealIDs[dealIdx], nil
+}
+
+func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId uint64, cb storagemarket.DealSectorCommittedCallback) error {
+	checkFunc := func(ts *types.TipSet) (done bool, more bool, err error) {
+		sd, err := stmgr.GetStorageDeal(ctx, c.StateManager, dealId, ts)
+		if err != nil {
+			// TODO: This may be fine for some errors
+			return false, false, xerrors.Errorf("failed to look up deal on chain: %w", err)
+		}
+
+		if sd.ActivationEpoch > 0 {
+			cb(nil)
+			return true, false, nil
+		}
+
+		return false, true, nil
+	}
+
+	called := func(msg *types.Message, rec *types.MessageReceipt, ts *types.TipSet, curH uint64) (more bool, err error) {
+		defer func() {
+			if err != nil {
+				cb(xerrors.Errorf("handling applied event: %w", err))
+			}
+		}()
+
+		if msg == nil {
+			log.Error("timed out waiting for deal activation... what now?")
+			return false, nil
+		}
+
+		sd, err := stmgr.GetStorageDeal(ctx, c.StateManager, dealId, ts)
+		if err != nil {
+			return false, xerrors.Errorf("failed to look up deal on chain: %w", err)
+		}
+
+		if sd.ActivationEpoch == 0 {
+			return false, xerrors.Errorf("deal wasn't active: deal=%d, parentState=%s, h=%d", dealId, ts.ParentState(), ts.Height())
+		}
+
+		log.Infof("Storage deal %d activated at epoch %d", dealId, sd.ActivationEpoch)
+
+		cb(nil)
+
+		return false, nil
+	}
+
+	revert := func(ctx context.Context, ts *types.TipSet) error {
+		log.Warn("deal activation reverted; TODO: actually handle this!")
+		// TODO: Just go back to DealSealing?
+		return nil
+	}
+
+	matchEvent := func(msg *types.Message) (bool, error) {
+		if msg.To != provider {
+			return false, nil
+		}
+
+		if msg.Method != actors.MAMethods.ProveCommitSector {
+			return false, nil
+		}
+
+		var params actors.SectorProveCommitInfo
+		if err := params.UnmarshalCBOR(bytes.NewReader(msg.Params)); err != nil {
+			return false, err
+		}
+
+		var found bool
+		for _, dealID := range params.DealIDs {
+			if dealID == dealId {
+				found = true
+				break
+			}
+		}
+
+		return found, nil
+	}
+
+	if err := c.ev.Called(checkFunc, called, revert, 3, build.SealRandomnessLookbackLimit, matchEvent); err != nil {
+		return xerrors.Errorf("failed to set up called handler")
+	}
+
+	return nil
+}
+
+func (n *ClientNodeAdapter) SignProposal(ctx context.Context, signer address.Address, proposal *storagemarket.StorageDealProposal) error {
+	localProposal, err := utils.FromSharedStorageDealProposal(proposal)
+	if err != nil {
+		return err
+	}
+	err = api.SignWith(ctx, n.Wallet.Sign, signer, localProposal)
+	if err != nil {
+		return err
+	}
+	signature, err := utils.ToSharedSignature(localProposal.ProposerSignature)
+	if err != nil {
+		return err
+	}
+	proposal.ProposerSignature = signature
+	return nil
+}
+
+func (n *ClientNodeAdapter) GetDefaultWalletAddress(ctx context.Context) (address.Address, error) {
+	addr, err := n.Wallet.GetDefault()
+	return addr, err
+}
+
+func (n *ClientNodeAdapter) ValidateAskSignature(ask *sharedtypes.SignedStorageAsk) error {
+	tss := n.cs.GetHeaviestTipSet().ParentState()
+
+	w, err := stmgr.GetMinerWorkerRaw(context.TODO(), n.StateManager, tss, ask.Ask.Miner)
+	if err != nil {
+		return xerrors.Errorf("failed to get worker for miner in ask", err)
+	}
+
+	sigb, err := cborutil.Dump(ask.Ask)
+	if err != nil {
+		return xerrors.Errorf("failed to re-serialize ask")
+	}
+
+	return ask.Signature.Verify(w, sigb)
+}
+
+var _ storagemarket.StorageClientNode = &ClientNodeAdapter{}

markets/storageadapter/provider.go

@@ -0,0 +1,209 @@
+package storageadapter
+
+// this file implements storagemarket.StorageProviderNode
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/ipfs/go-cid"
+	logging "github.com/ipfs/go-log"
+	unixfile "github.com/ipfs/go-unixfs/file"
+	"golang.org/x/xerrors"
+
+	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/shared/tokenamount"
+	sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/lib/padreader"
+	"github.com/filecoin-project/lotus/markets/utils"
+	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/filecoin-project/lotus/storage/sectorblocks"
+)
+
+var log = logging.Logger("provideradapter")
+
+type ProviderNodeAdapter struct {
+	api.FullNode
+
+	// this goes away with the data transfer module
+	dag dtypes.StagingDAG
+
+	secb *sectorblocks.SectorBlocks
+}
+
+func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode {
+	return &ProviderNodeAdapter{
+		FullNode: full,
+		dag:      dag,
+		secb:     secb,
+	}
+}
+
+func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemarket.MinerDeal) (storagemarket.DealID, cid.Cid, error) {
+	log.Info("publishing deal")
+
+	worker, err := n.StateMinerWorker(ctx, deal.Proposal.Provider, nil)
+	if err != nil {
+		return 0, cid.Undef, err
+	}
+
+	localProposal, err := utils.FromSharedStorageDealProposal(&deal.Proposal)
+	if err != nil {
+		return 0, cid.Undef, err
+	}
+	params, err := actors.SerializeParams(&actors.PublishStorageDealsParams{
+		Deals: []actors.StorageDealProposal{*localProposal},
+	})
+
+	if err != nil {
+		return 0, cid.Undef, xerrors.Errorf("serializing PublishStorageDeals params failed: ", err)
+	}
+
+	// TODO: We may want this to happen after fetching data
+	smsg, err := n.MpoolPushMessage(ctx, &types.Message{
+		To:       actors.StorageMarketAddress,
+		From:     worker,
+		Value:    types.NewInt(0),
+		GasPrice: types.NewInt(0),
+		GasLimit: types.NewInt(1000000),
+		Method:   actors.SMAMethods.PublishStorageDeals,
+		Params:   params,
+	})
+	if err != nil {
+		return 0, cid.Undef, err
+	}
+	r, err := n.StateWaitMsg(ctx, smsg.Cid())
+	if err != nil {
+		return 0, cid.Undef, err
+	}
+	if r.Receipt.ExitCode != 0 {
+		return 0, cid.Undef, xerrors.Errorf("publishing deal failed: exit %d", r.Receipt.ExitCode)
+	}
+	var resp actors.PublishStorageDealResponse
+	if err := resp.UnmarshalCBOR(bytes.NewReader(r.Receipt.Return)); err != nil {
+		return 0, cid.Undef, err
+	}
+	if len(resp.DealIDs) != 1 {
+		return 0, cid.Undef, xerrors.Errorf("got unexpected number of DealIDs from")
+	}
+
+	return storagemarket.DealID(resp.DealIDs[0]), smsg.Cid(), nil
+}
+
+func (n *ProviderNodeAdapter) OnDealComplete(ctx context.Context, deal storagemarket.MinerDeal, piecePath string) (uint64, error) {
+	root, err := n.dag.Get(ctx, deal.Ref)
+	if err != nil {
+		return 0, xerrors.Errorf("failed to get file root for deal: %s", err)
+	}
+
+	// TODO: abstract this away into ReadSizeCloser + implement different modes
+	node, err := unixfile.NewUnixfsFile(ctx, n.dag, root)
+	if err != nil {
+		return 0, xerrors.Errorf("cannot open unixfs file: %s", err)
+	}
+
+	uf, ok := node.(sectorblocks.UnixfsReader)
+	if !ok {
+		// we probably got directory, unsupported for now
+		return 0, xerrors.Errorf("unsupported unixfs file type")
+	}
+
+	// TODO: uf.Size() is user input, not trusted
+	// This won't be useful / here after we migrate to putting CARs into sectors
+	size, err := uf.Size()
+	if err != nil {
+		return 0, xerrors.Errorf("getting unixfs file size: %w", err)
+	}
+	if padreader.PaddedSize(uint64(size)) != deal.Proposal.PieceSize {
+		return 0, xerrors.Errorf("deal.Proposal.PieceSize didn't match padded unixfs file size")
+	}
+
+	sectorID, err := n.secb.AddUnixfsPiece(ctx, uf, deal.DealID)
+	if err != nil {
+		return 0, xerrors.Errorf("AddPiece failed: %s", err)
+	}
+	log.Warnf("New Sector: %d (deal %d)", sectorID, deal.DealID)
+
+	return sectorID, nil
+}
+
+func (n *ProviderNodeAdapter) ListProviderDeals(ctx context.Context, addr address.Address) ([]storagemarket.StorageDeal, error) {
+	allDeals, err := n.StateMarketDeals(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var out []storagemarket.StorageDeal
+
+	for _, deal := range allDeals {
+		sharedDeal := utils.FromOnChainDeal(deal)
+		if sharedDeal.Provider == addr {
+			out = append(out, sharedDeal)
+		}
+	}
+
+	return out, nil
+}
+
+func (n *ProviderNodeAdapter) GetMinerWorker(ctx context.Context, miner address.Address) (address.Address, error) {
+	addr, err := n.StateMinerWorker(ctx, miner, nil)
+	return addr, err
+}
+
+func (n *ProviderNodeAdapter) SignBytes(ctx context.Context, signer address.Address, b []byte) (*sharedtypes.Signature, error) {
+	localSignature, err := n.WalletSign(ctx, signer, b)
+	if err != nil {
+		return nil, err
+	}
+	return utils.ToSharedSignature(localSignature)
+}
+
+func (n *ProviderNodeAdapter) EnsureFunds(ctx context.Context, addr address.Address, amt tokenamount.TokenAmount) error {
+	return n.MarketEnsureAvailable(ctx, addr, utils.FromSharedTokenAmount(amt))
+}
+
+func (n *ProviderNodeAdapter) MostRecentStateId(ctx context.Context) (storagemarket.StateKey, error) {
+	return n.ChainHead(ctx)
+}
+
+// Adds funds with the StorageMinerActor for a storage participant.  Used by both providers and clients.
+func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address, amount tokenamount.TokenAmount) error {
+	// (Provider Node API)
+	smsg, err := n.MpoolPushMessage(ctx, &types.Message{
+		To:       actors.StorageMarketAddress,
+		From:     addr,
+		Value:    utils.FromSharedTokenAmount(amount),
+		GasPrice: types.NewInt(0),
+		GasLimit: types.NewInt(1000000),
+		Method:   actors.SMAMethods.AddBalance,
+	})
+	if err != nil {
+		return err
+	}
+
+	r, err := n.StateWaitMsg(ctx, smsg.Cid())
+	if err != nil {
+		return err
+	}
+
+	if r.Receipt.ExitCode != 0 {
+		return xerrors.Errorf("adding funds to storage miner market actor failed: exit %d", r.Receipt.ExitCode)
+	}
+
+	return nil
+}
+
+func (n *ProviderNodeAdapter) GetBalance(ctx context.Context, addr address.Address) (storagemarket.Balance, error) {
+	bal, err := n.StateMarketBalance(ctx, addr, nil)
+	if err != nil {
+		return storagemarket.Balance{}, err
+	}
+
+	return utils.ToSharedBalance(bal), nil
+}
+
+var _ storagemarket.StorageProviderNode = &ProviderNodeAdapter{}

markets/utils/converters.go

@@ -0,0 +1,174 @@
+package utils
+
+import (
+	"bytes"
+
+	peer "github.com/libp2p/go-libp2p-peer"
+
+	"github.com/filecoin-project/go-address"
+	sharedamount "github.com/filecoin-project/go-fil-markets/shared/tokenamount"
+	sharedtypes "github.com/filecoin-project/go-fil-markets/shared/types"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/types"
+)
+
+func FromSharedTokenAmount(in sharedamount.TokenAmount) types.BigInt {
+	return types.BigInt{Int: in.Int}
+}
+
+func ToSharedTokenAmount(in types.BigInt) sharedamount.TokenAmount {
+	return sharedamount.TokenAmount{Int: in.Int}
+}
+
+func ToSharedSignedVoucher(in *types.SignedVoucher) (*sharedtypes.SignedVoucher, error) {
+	var encoded bytes.Buffer
+	err := in.MarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	var out sharedtypes.SignedVoucher
+	err = out.UnmarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func FromSharedSignedVoucher(in *sharedtypes.SignedVoucher) (*types.SignedVoucher, error) {
+	var encoded bytes.Buffer
+	err := in.MarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	var out types.SignedVoucher
+	err = out.UnmarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func ToSharedSignature(in *types.Signature) (*sharedtypes.Signature, error) {
+	var encoded bytes.Buffer
+	err := in.MarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	var out sharedtypes.Signature
+	err = out.UnmarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func FromSharedSignature(in *sharedtypes.Signature) (*types.Signature, error) {
+	var encoded bytes.Buffer
+	err := in.MarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	var out types.Signature
+	err = out.UnmarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func ToSharedStorageAsk(in *types.SignedStorageAsk) (*sharedtypes.SignedStorageAsk, error) {
+	var encoded bytes.Buffer
+	err := in.MarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	var out sharedtypes.SignedStorageAsk
+	err = out.UnmarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func FromSignedStorageAsk(in *sharedtypes.SignedStorageAsk) (*types.SignedStorageAsk, error) {
+	var encoded bytes.Buffer
+	err := in.MarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	var out types.SignedStorageAsk
+	err = out.UnmarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func NewStorageProviderInfo(address address.Address, miner address.Address, sectorSize uint64, peer peer.ID) storagemarket.StorageProviderInfo {
+	return storagemarket.StorageProviderInfo{
+		Address:    address,
+		Worker:     miner,
+		SectorSize: sectorSize,
+		PeerID:     peer,
+	}
+}
+
+func FromOnChainDeal(deal actors.OnChainDeal) storagemarket.StorageDeal {
+	return storagemarket.StorageDeal{
+		PieceRef:             deal.PieceRef,
+		PieceSize:            deal.PieceSize,
+		Client:               deal.Client,
+		Provider:             deal.Provider,
+		StoragePricePerEpoch: ToSharedTokenAmount(deal.StoragePricePerEpoch),
+		StorageCollateral:    ToSharedTokenAmount(deal.StorageCollateral),
+		ActivationEpoch:      deal.ActivationEpoch,
+	}
+}
+
+func ToOnChainDeal(deal storagemarket.StorageDeal) actors.OnChainDeal {
+	return actors.OnChainDeal{
+		PieceRef:             deal.PieceRef,
+		PieceSize:            deal.PieceSize,
+		Client:               deal.Client,
+		Provider:             deal.Provider,
+		StoragePricePerEpoch: FromSharedTokenAmount(deal.StoragePricePerEpoch),
+		StorageCollateral:    FromSharedTokenAmount(deal.StorageCollateral),
+		ActivationEpoch:      deal.ActivationEpoch,
+	}
+}
+
+func ToSharedBalance(balance actors.StorageParticipantBalance) storagemarket.Balance {
+	return storagemarket.Balance{
+		Locked:    ToSharedTokenAmount(balance.Locked),
+		Available: ToSharedTokenAmount(balance.Available),
+	}
+}
+
+func ToSharedStorageDealProposal(proposal *actors.StorageDealProposal) (*storagemarket.StorageDealProposal, error) {
+	var encoded bytes.Buffer
+	err := proposal.MarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	var out storagemarket.StorageDealProposal
+	err = out.UnmarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func FromSharedStorageDealProposal(proposal *storagemarket.StorageDealProposal) (*actors.StorageDealProposal, error) {
+	var encoded bytes.Buffer
+	err := proposal.MarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	var out actors.StorageDealProposal
+	err = out.UnmarshalCBOR(&encoded)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}

miner/miner.go

@@ -12,13 +12,11 @@ import (
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/types"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"go.opencensus.io/trace"
 	"golang.org/x/xerrors"
 )
 
-const MaxMessagesPerBlock = 4000
-
 var log = logging.Logger("miner")
 
 type waitFunc func(ctx context.Context, baseTime uint64) error
@@ -363,6 +361,11 @@ func (m *Miner) createBlock(base *MiningBase, addr address.Address, ticket *type
 		return nil, xerrors.Errorf("message filtering failed: %w", err)
 	}
 
+	if len(msgs) > build.BlockMessageLimit {
+		log.Error("selectMessages returned too many messages: ", len(msgs))
+		msgs = msgs[:build.BlockMessageLimit]
+	}
+
 	uts := base.ts.MinTimestamp() + uint64(build.BlockDelay*(base.nullRounds+1))
 
 	nheight := base.ts.Height() + base.nullRounds + 1
@@ -383,12 +386,13 @@ func countFrom(msgs []*types.SignedMessage, from address.Address) (out int) {
 }
 
 func selectMessages(ctx context.Context, al actorLookup, base *MiningBase, msgs []*types.SignedMessage) ([]*types.SignedMessage, error) {
-	out := make([]*types.SignedMessage, 0, len(msgs))
+	out := make([]*types.SignedMessage, 0, build.BlockMessageLimit)
 	inclNonces := make(map[address.Address]uint64)
 	inclBalances := make(map[address.Address]types.BigInt)
 	inclCount := make(map[address.Address]int)
 
 	for _, msg := range msgs {
+
 		if msg.Message.To == address.Undef {
 			log.Warnf("message in mempool had bad 'To' address")
 			continue
@@ -426,7 +430,7 @@ func selectMessages(ctx context.Context, al actorLookup, base *MiningBase, msgs
 		inclCount[from]++
 
 		out = append(out, msg)
-		if len(out) >= MaxMessagesPerBlock {
+		if len(out) >= build.BlockMessageLimit {
 			break
 		}
 	}

node/builder.go

@@ -19,10 +19,14 @@ import (
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
+
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/blocksync"
-	"github.com/filecoin-project/lotus/chain/deals"
 	"github.com/filecoin-project/lotus/chain/gen"
 	"github.com/filecoin-project/lotus/chain/market"
 	"github.com/filecoin-project/lotus/chain/messagepool"
@@ -30,7 +34,9 @@ import (
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/chain/vm"
 	"github.com/filecoin-project/lotus/chain/wallet"
+	"github.com/filecoin-project/lotus/markets/storageadapter"
 	"github.com/filecoin-project/lotus/miner"
 	"github.com/filecoin-project/lotus/node/config"
 	"github.com/filecoin-project/lotus/node/hello"
@@ -43,8 +49,6 @@ import (
 	"github.com/filecoin-project/lotus/node/repo"
 	"github.com/filecoin-project/lotus/paych"
 	"github.com/filecoin-project/lotus/peermgr"
-	"github.com/filecoin-project/lotus/retrieval"
-	"github.com/filecoin-project/lotus/retrieval/discovery"
 	"github.com/filecoin-project/lotus/storage"
 	"github.com/filecoin-project/lotus/storage/sectorblocks"
 )
@@ -56,7 +60,6 @@ type special struct{ id int }
 //nolint:golint
 var (
 	DefaultTransportsKey = special{0} // Libp2p option
-	PNetKey              = special{1} // Option + multiret
 	DiscoveryHandlerKey  = special{2} // Private type
 	AddrsFactoryKey      = special{3} // Libp2p option
 	SmuxTransportKey     = special{4} // Libp2p option
@@ -140,7 +143,6 @@ func libp2p() Option {
 		Override(new(peerstore.Peerstore), pstoremem.NewPeerstore),
 
 		Override(DefaultTransportsKey, lp2p.DefaultTransports),
-		Override(PNetKey, lp2p.PNet),
 
 		Override(new(lp2p.RawHost), lp2p.Host),
 		Override(new(host.Host), lp2p.RoutedHost),
@@ -193,6 +195,8 @@ func Online() Option {
 
 			Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages),
 
+			Override(new(sectorbuilder.Verifier), sectorbuilder.ProofVerifier),
+			Override(new(*types.VMSyscalls), vm.Syscalls),
 			Override(new(*store.ChainStore), modules.ChainStore),
 			Override(new(*stmgr.StateManager), stmgr.NewStateManager),
 			Override(new(*wallet.Wallet), wallet.NewWallet),
@@ -220,14 +224,15 @@ func Online() Option {
 			Override(RunPeerMgrKey, modules.RunPeerMgr),
 			Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks),
 
-			Override(new(*discovery.Local), discovery.NewLocal),
+			Override(new(*discovery.Local), modules.NewLocalDiscovery),
-			Override(new(discovery.PeerResolver), modules.RetrievalResolver),
+			Override(new(retrievalmarket.PeerResolver), modules.RetrievalResolver),
 
-			Override(new(*retrieval.Client), retrieval.NewClient),
+			Override(new(retrievalmarket.RetrievalClient), modules.RetrievalClient),
 			Override(new(dtypes.ClientDealStore), modules.NewClientDealStore),
 			Override(new(dtypes.ClientDataTransfer), modules.NewClientDAGServiceDataTransfer),
-			Override(new(*deals.ClientRequestValidator), deals.NewClientRequestValidator),
+			Override(new(*deals.ClientRequestValidator), modules.NewClientRequestValidator),
-			Override(new(*deals.Client), deals.NewClient),
+			Override(new(storagemarket.StorageClient), modules.StorageClient),
+			Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter),
 			Override(RegisterClientValidatorKey, modules.RegisterClientValidator),
 			Override(RunDealClientKey, modules.RunDealClient),
 
@@ -238,18 +243,20 @@ func Online() Option {
 
 		// Storage miner
 		ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner },
-			Override(new(*sectorbuilder.SectorBuilder), modules.SectorBuilder),
+			Override(new(sectorbuilder.Interface), modules.SectorBuilder),
 			Override(new(*sectorblocks.SectorBlocks), sectorblocks.NewSectorBlocks),
 			Override(new(storage.TicketFn), modules.SealTicketGen),
 			Override(new(*storage.Miner), modules.StorageMiner),
 
+			Override(new(dtypes.StagingBlockstore), modules.StagingBlockstore),
 			Override(new(dtypes.StagingDAG), modules.StagingDAG),
-
+			Override(new(dtypes.StagingGraphsync), modules.StagingGraphsync),
-			Override(new(*retrieval.Miner), retrieval.NewMiner),
+			Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider),
 			Override(new(dtypes.ProviderDealStore), modules.NewProviderDealStore),
 			Override(new(dtypes.ProviderDataTransfer), modules.NewProviderDAGServiceDataTransfer),
-			Override(new(*deals.ProviderRequestValidator), deals.NewProviderRequestValidator),
+			Override(new(*deals.ProviderRequestValidator), modules.NewProviderRequestValidator),
-			Override(new(*deals.Provider), deals.NewProvider),
+			Override(new(storagemarket.StorageProvider), modules.StorageProvider),
+			Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter),
 			Override(RegisterProviderValidatorKey, modules.RegisterProviderValidator),
 			Override(HandleRetrievalKey, modules.HandleRetrieval),
 			Override(GetParamsKey, modules.GetParams),
@@ -372,6 +379,7 @@ func Repo(r repo.Repo) Option {
 			Override(new(dtypes.ClientFilestore), modules.ClientFstore),
 			Override(new(dtypes.ClientBlockstore), modules.ClientBlockstore),
 			Override(new(dtypes.ClientDAG), modules.ClientDAG),
+			Override(new(dtypes.ClientGraphsync), modules.ClientGraphsync),
 
 			Override(new(ci.PrivKey), lp2p.PrivKey),
 			Override(new(ci.PubKey), ci.PrivKey.GetPublic),

node/fxlog.go

@@ -1,7 +1,7 @@
 package node
 
 import (
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 
 	"go.uber.org/fx"
 )

node/hello/hello.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p-core/host"
 	inet "github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"

node/impl/client/client.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"io"
@@ -17,22 +18,24 @@ import (
 	files "github.com/ipfs/go-ipfs-files"
 	ipld "github.com/ipfs/go-ipld-format"
 	"github.com/ipfs/go-merkledag"
+	unixfile "github.com/ipfs/go-unixfs/file"
 	"github.com/ipfs/go-unixfs/importer/balanced"
 	ihelper "github.com/ipfs/go-unixfs/importer/helpers"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"go.uber.org/fx"
 
 	"github.com/filecoin-project/go-address"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-fil-markets/shared/tokenamount"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/build"
-	"github.com/filecoin-project/lotus/chain/deals"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
+	"github.com/filecoin-project/lotus/markets/utils"
 	"github.com/filecoin-project/lotus/node/impl/full"
 	"github.com/filecoin-project/lotus/node/impl/paych"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
-	"github.com/filecoin-project/lotus/retrieval"
-	"github.com/filecoin-project/lotus/retrieval/discovery"
 )
 
 type API struct {
@@ -43,9 +46,9 @@ type API struct {
 	full.WalletAPI
 	paych.PaychAPI
 
-	DealClient   *deals.Client
+	SMDealClient storagemarket.StorageClient
-	RetDiscovery discovery.PeerResolver
+	RetDiscovery retrievalmarket.PeerResolver
-	Retrieval    *retrieval.Client
+	Retrieval    retrievalmarket.RetrievalClient
 	Chain        *store.ChainStore
 
 	LocalDAG   dtypes.ClientDAG
@@ -71,28 +74,26 @@ func (a *API) ClientStartDeal(ctx context.Context, data cid.Cid, addr address.Ad
 	if err != nil {
 		return nil, xerrors.Errorf("failed getting miner worker: %w", err)
 	}
+	providerInfo := utils.NewStorageProviderInfo(miner, mw, 0, pid)
+	result, err := a.SMDealClient.ProposeStorageDeal(
+		ctx,
+		addr,
+		&providerInfo,
+		data,
+		storagemarket.Epoch(math.MaxUint64),
+		storagemarket.Epoch(blocksDuration),
+		utils.ToSharedTokenAmount(epochPrice),
+		tokenamount.Empty)
 
-	proposal := deals.ClientDealProposal{
-		Data:               data,
-		PricePerEpoch:      epochPrice,
-		ProposalExpiration: math.MaxUint64, // TODO: set something reasonable
-		Duration:           blocksDuration,
-		Client:             addr,
-		ProviderAddress:    miner,
-		MinerWorker:        mw,
-		MinerID:            pid,
-	}
-
-	c, err := a.DealClient.Start(ctx, proposal)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to start deal: %w", err)
 	}
 
-	return &c, nil
+	return &result.ProposalCid, nil
 }
 
 func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
-	deals, err := a.DealClient.List()
+	deals, err := a.SMDealClient.ListInProgressDeals(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -107,7 +108,7 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
 			PieceRef: v.Proposal.PieceRef,
 			Size:     v.Proposal.PieceSize,
 
-			PricePerEpoch: v.Proposal.StoragePricePerEpoch,
+			PricePerEpoch: utils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch),
 			Duration:      v.Proposal.Duration,
 		}
 	}
@@ -116,17 +117,18 @@ func (a *API) ClientListDeals(ctx context.Context) ([]api.DealInfo, error) {
 }
 
 func (a *API) ClientGetDealInfo(ctx context.Context, d cid.Cid) (*api.DealInfo, error) {
-	v, err := a.DealClient.GetDeal(d)
+	v, err := a.SMDealClient.GetInProgressDeal(ctx, d)
 	if err != nil {
 		return nil, err
 	}
+
 	return &api.DealInfo{
 		ProposalCid:   v.ProposalCid,
 		State:         v.State,
 		Provider:      v.Proposal.Provider,
 		PieceRef:      v.Proposal.PieceRef,
 		Size:          v.Proposal.PieceSize,
-		PricePerEpoch: v.Proposal.StoragePricePerEpoch,
+		PricePerEpoch: utils.FromSharedTokenAmount(v.Proposal.StoragePricePerEpoch),
 		Duration:      v.Proposal.Duration,
 	}, nil
 }
@@ -153,7 +155,18 @@ func (a *API) ClientFindData(ctx context.Context, root cid.Cid) ([]api.QueryOffe
 
 	out := make([]api.QueryOffer, len(peers))
 	for k, p := range peers {
-		out[k] = a.Retrieval.Query(ctx, p, root)
+		queryResponse, err := a.Retrieval.Query(ctx, p, root.Bytes(), retrievalmarket.QueryParams{})
+		if err != nil {
+			out[k] = api.QueryOffer{Err: err.Error(), Miner: p.Address, MinerPeerID: p.ID}
+		} else {
+			out[k] = api.QueryOffer{
+				Root:        root,
+				Size:        queryResponse.Size,
+				MinPrice:    utils.FromSharedTokenAmount(queryResponse.PieceRetrievalPrice()),
+				Miner:       p.Address, // TODO: check
+				MinerPeerID: p.ID,
+			}
+		}
 	}
 
 	return out, nil
@@ -263,20 +276,54 @@ func (a *API) ClientRetrieve(ctx context.Context, order api.RetrievalOrder, path
 		order.MinerPeerID = pid
 	}
 
-	outFile, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0777)
+	retrievalResult := make(chan error, 1)
-	if err != nil {
+
-		return err
+	unsubscribe := a.Retrieval.SubscribeToEvents(func(event retrievalmarket.ClientEvent, state retrievalmarket.ClientDealState) {
+		if bytes.Equal(state.PieceCID, order.Root.Bytes()) {
+			switch event {
+			case retrievalmarket.ClientEventError:
+				retrievalResult <- xerrors.New("Retrieval Error")
+			case retrievalmarket.ClientEventComplete:
+				retrievalResult <- nil
+			}
+		}
+	})
+
+	a.Retrieval.Retrieve(
+		ctx,
+		order.Root.Bytes(),
+		retrievalmarket.NewParamsV0(types.BigDiv(order.Total, types.NewInt(order.Size)).Int, 0, 0),
+		utils.ToSharedTokenAmount(order.Total),
+		order.MinerPeerID,
+		order.Client,
+		order.Miner)
+	select {
+	case <-ctx.Done():
+		return xerrors.New("Retrieval Timed Out")
+	case err := <-retrievalResult:
+		if err != nil {
+			return xerrors.Errorf("RetrieveUnixfs: %w", err)
+		}
 	}
 
-	err = a.Retrieval.RetrieveUnixfs(ctx, order.Root, order.Size, order.Total, order.MinerPeerID, order.Client, order.Miner, outFile)
+	unsubscribe()
-	if err != nil {
-		_ = outFile.Close()
-		return xerrors.Errorf("RetrieveUnixfs: %w", err)
-	}
 
-	return outFile.Close()
+	nd, err := a.LocalDAG.Get(ctx, order.Root)
+	if err != nil {
+		return xerrors.Errorf("ClientRetrieve: %w", err)
+	}
+	file, err := unixfile.NewUnixfsFile(ctx, a.LocalDAG, nd)
+	if err != nil {
+		return xerrors.Errorf("ClientRetrieve: %w", err)
+	}
+	return files.WriteTo(file, path)
 }
 
 func (a *API) ClientQueryAsk(ctx context.Context, p peer.ID, miner address.Address) (*types.SignedStorageAsk, error) {
-	return a.DealClient.QueryAsk(ctx, p, miner)
+	info := utils.NewStorageProviderInfo(miner, address.Undef, 0, p)
+	signedAsk, err := a.SMDealClient.GetAsk(ctx, info)
+	if err != nil {
+		return nil, err
+	}
+	return utils.FromSignedStorageAsk(signedAsk)
 }

node/impl/full.go

@@ -1,7 +1,7 @@
 package impl
 
 import (
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 
 	"github.com/filecoin-project/lotus/node/impl/client"
 	"github.com/filecoin-project/lotus/node/impl/market"

node/impl/storminer.go

@@ -3,12 +3,13 @@ package impl
 import (
 	"context"
 	"encoding/json"
-	"github.com/filecoin-project/lotus/api/apistruct"
 	"io"
 	"mime"
 	"net/http"
 	"os"
 
+	"github.com/filecoin-project/lotus/api/apistruct"
+
 	"github.com/gorilla/mux"
 	files "github.com/ipfs/go-ipfs-files"
 
@@ -25,7 +26,7 @@ type StorageMinerAPI struct {
 	CommonAPI
 
 	SectorBuilderConfig *sectorbuilder.Config
-	SectorBuilder       *sectorbuilder.SectorBuilder
+	SectorBuilder       sectorbuilder.Interface
 	SectorBlocks        *sectorblocks.SectorBlocks
 
 	Miner      *storage.Miner

node/modules/chain.go

@@ -73,8 +73,8 @@ func ChainBlockservice(bs dtypes.ChainBlockstore, rem dtypes.ChainExchange) dtyp
 	return blockservice.New(bs, rem)
 }
 
-func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS) *store.ChainStore {
+func ChainStore(lc fx.Lifecycle, bs dtypes.ChainBlockstore, ds dtypes.MetadataDS, syscalls *types.VMSyscalls) *store.ChainStore {
-	chain := store.NewChainStore(bs, ds)
+	chain := store.NewChainStore(bs, ds, syscalls)
 
 	if err := chain.Load(); err != nil {
 		log.Warnf("loading chain state from disk: %s", err)

node/modules/client.go

@@ -5,25 +5,36 @@ import (
 	"path/filepath"
 	"reflect"
 
+	"github.com/filecoin-project/go-data-transfer/impl/graphsync"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
+	"github.com/filecoin-project/go-fil-markets/retrievalmarket/discovery"
+	retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl"
+	"github.com/filecoin-project/go-fil-markets/storagemarket"
+	deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
+	storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
 	"github.com/filecoin-project/go-statestore"
-	"github.com/filecoin-project/lotus/node/modules/helpers"
 	"github.com/ipfs/go-bitswap"
 	"github.com/ipfs/go-bitswap/network"
-	"github.com/libp2p/go-libp2p-core/host"
-	"github.com/libp2p/go-libp2p-core/routing"
-
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/namespace"
 	"github.com/ipfs/go-filestore"
+	graphsync "github.com/ipfs/go-graphsync/impl"
+	"github.com/ipfs/go-graphsync/ipldbridge"
+	gsnet "github.com/ipfs/go-graphsync/network"
+	"github.com/ipfs/go-graphsync/storeutil"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	"github.com/ipfs/go-merkledag"
+	"github.com/libp2p/go-libp2p-core/host"
+	"github.com/libp2p/go-libp2p-core/routing"
 	"go.uber.org/fx"
 
-	"github.com/filecoin-project/lotus/chain/deals"
+	"github.com/filecoin-project/lotus/markets/retrievaladapter"
-	"github.com/filecoin-project/lotus/datatransfer"
+	payapi "github.com/filecoin-project/lotus/node/impl/paych"
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
+	"github.com/filecoin-project/lotus/node/modules/helpers"
 	"github.com/filecoin-project/lotus/node/repo"
+	"github.com/filecoin-project/lotus/paych"
 )
 
 func ClientFstore(r repo.LockedRepo) (dtypes.ClientFilestore, error) {
@@ -49,15 +60,15 @@ func ClientBlockstore(fstore dtypes.ClientFilestore) dtypes.ClientBlockstore {
 // request validator with the data transfer module as the validator for
 // StorageDataTransferVoucher types
 func RegisterClientValidator(crv *deals.ClientRequestValidator, dtm dtypes.ClientDataTransfer) {
-	if err := dtm.RegisterVoucherType(reflect.TypeOf(deals.StorageDataTransferVoucher{}), crv); err != nil {
+	if err := dtm.RegisterVoucherType(reflect.TypeOf(&deals.StorageDataTransferVoucher{}), crv); err != nil {
 		panic(err)
 	}
 }
 
 // NewClientDAGServiceDataTransfer returns a data transfer manager that just
 // uses the clients's Client DAG service for transfers
-func NewClientDAGServiceDataTransfer(dag dtypes.ClientDAG) dtypes.ClientDataTransfer {
+func NewClientDAGServiceDataTransfer(h host.Host, gs dtypes.ClientGraphsync) dtypes.ClientDataTransfer {
-	return datatransfer.NewDAGServiceDataTransfer(dag)
+	return graphsyncimpl.NewGraphSyncDataTransfer(h, gs)
 }
 
 // NewClientDealStore creates a statestore for the client to store its deals
@@ -65,6 +76,7 @@ func NewClientDealStore(ds dtypes.MetadataDS) dtypes.ClientDealStore {
 	return statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))
 }
 
+// ClientDAG is a DAGService for the ClientBlockstore
 func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, rt routing.Routing, h host.Host) dtypes.ClientDAG {
 	bitswapNetwork := network.NewFromIpfsHost(h, rt)
 	exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, ibs)
@@ -80,3 +92,29 @@ func ClientDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlocks
 
 	return dag
 }
+
+// ClientGraphsync creates a graphsync instance which reads and writes blocks
+// to the ClientBlockstore
+func ClientGraphsync(mctx helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.ClientBlockstore, h host.Host) dtypes.ClientGraphsync {
+	graphsyncNetwork := gsnet.NewFromLibp2pHost(h)
+	ipldBridge := ipldbridge.NewIPLDBridge()
+	loader := storeutil.LoaderForBlockstore(ibs)
+	storer := storeutil.StorerForBlockstore(ibs)
+	gs := graphsync.New(helpers.LifecycleCtx(mctx, lc), graphsyncNetwork, ipldBridge, loader, storer)
+
+	return gs
+}
+
+func NewClientRequestValidator(deals dtypes.ClientDealStore) *storageimpl.ClientRequestValidator {
+	return storageimpl.NewClientRequestValidator(deals)
+}
+
+func StorageClient(h host.Host, dag dtypes.ClientDAG, dataTransfer dtypes.ClientDataTransfer, discovery *discovery.Local, deals dtypes.ClientDealStore, scn storagemarket.StorageClientNode) storagemarket.StorageClient {
+	return storageimpl.NewClient(h, dag, dataTransfer, discovery, deals, scn)
+}
+
+// RetrievalClient creates a new retrieval client attached to the client blockstore
+func RetrievalClient(h host.Host, bs dtypes.ClientBlockstore, pmgr *paych.Manager, payapi payapi.PaychAPI) retrievalmarket.RetrievalClient {
+	adapter := retrievaladapter.NewRetrievalClientNode(pmgr, payapi)
+	return retrievalimpl.NewClient(h, bs, adapter)
+}

node/modules/core.go

@@ -13,7 +13,7 @@ import (
 	"github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/filecoin-project/lotus/node/repo"
 	"github.com/gbrlsnchs/jwt/v3"
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p-core/peerstore"
 	record "github.com/libp2p/go-libp2p-record"
 	"golang.org/x/xerrors"

node/modules/dtypes/storage.go

@@ -4,12 +4,13 @@ import (
 	bserv "github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-filestore"
+	"github.com/ipfs/go-graphsync"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	exchange "github.com/ipfs/go-ipfs-exchange-interface"
 	ipld "github.com/ipfs/go-ipld-format"
 
+	"github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-statestore"
-	"github.com/filecoin-project/lotus/datatransfer"
 )
 
 // MetadataDS stores metadata
@@ -26,6 +27,7 @@ type ChainBlockService bserv.BlockService
 type ClientFilestore *filestore.Filestore
 type ClientBlockstore blockstore.Blockstore
 type ClientDAG ipld.DAGService
+type ClientGraphsync graphsync.GraphExchange
 type ClientDealStore *statestore.StateStore
 
 // ClientDataTransfer is a data transfer manager for the client
@@ -37,3 +39,5 @@ type ProviderDealStore *statestore.StateStore
 type ProviderDataTransfer datatransfer.Manager
 
 type StagingDAG ipld.DAGService
+type StagingBlockstore blockstore.Blockstore
+type StagingGraphsync graphsync.GraphExchange

node/modules/lp2p/host.go

@@ -79,6 +79,7 @@ func DHTRouting(client bool) interface{} {
 			dhtopts.Client(client),
 			dhtopts.Datastore(dstore),
 			dhtopts.Validator(validator),
+			dhtopts.Protocols("/lotus/kad/1.0.0"),
 		)
 
 		if err != nil {

node/modules/lp2p/libp2p.go

@@ -8,7 +8,7 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	"golang.org/x/xerrors"
 
-	logging "github.com/ipfs/go-log"
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/libp2p/go-libp2p"
 	connmgr "github.com/libp2p/go-libp2p-connmgr"
 	"github.com/libp2p/go-libp2p-core/crypto"

node/modules/lp2p/pnet.go

@@ -1,63 +0,0 @@
-package lp2p
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/libp2p/go-libp2p"
-	pnet "github.com/libp2p/go-libp2p-pnet"
-)
-
-var LotusKey = "/key/swarm/psk/1.0.0/\n/base16/\n20c72388e6299c7bbc1b501fdcc8abe4f89f798e9b93b2d2bc02e3c29b6a088e"
-
-type PNetFingerprint []byte
-
-func PNet() (opts Libp2pOpts, fp PNetFingerprint, err error) {
-	protec, err := pnet.NewProtector(strings.NewReader(LotusKey))
-	if err != nil {
-		return opts, nil, fmt.Errorf("failed to configure private network: %s", err)
-	}
-	fp = protec.Fingerprint()
-
-	opts.Opts = append(opts.Opts, libp2p.PrivateNetwork(protec))
-	return opts, fp, nil
-}
-
-/*
-func PNetChecker(repo repo.Repo, ph host.Host, lc fx.Lifecycle) error {
-	// TODO: better check?
-	swarmkey, err := repo.SwarmKey()
-	if err != nil || swarmkey == nil {
-		return err
-	}
-
-	done := make(chan struct{})
-	lc.Append(fx.Hook{
-		OnStart: func(_ context.Context) error {
-			go func() {
-				t := time.NewTicker(30 * time.Second)
-				defer t.Stop()
-
-				<-t.C // swallow one tick
-				for {
-					select {
-					case <-t.C:
-						if len(ph.Network().Peers()) == 0 {
-							log.Warn("We are in private network and have no peers.")
-							log.Warn("This might be configuration mistake.")
-						}
-					case <-done:
-						return
-					}
-				}
-			}()
-			return nil
-		},
-		OnStop: func(_ context.Context) error {
-			close(done)
-			return nil
-		},
-	})
-	return nil
-}
-*/