Merge branch 'next' into ntwk-calibration

api/api_full.go

@@ -614,6 +614,7 @@ type StartDealParams struct {
 	Miner              address.Address
 	EpochPrice         types.BigInt
 	MinBlocksDuration  uint64
+	ProviderCollateral big.Int
 	DealStartEpoch     abi.ChainEpoch
 	FastRetrieval      bool
 	VerifiedDeal       bool

cli/client.go

@@ -311,6 +311,10 @@ var clientDealCmd = &cli.Command{
 			Usage: "indicate that the deal counts towards verified client total",
 			Value: false,
 		},
+		&cli.StringFlag{
+			Name:  "provider-collateral",
+			Usage: "specify the requested provider collateral the miner should put up",
+		},
 		&CidBaseFlag,
 	},
 	Action: func(cctx *cli.Context) error {
@@ -351,6 +355,15 @@ var clientDealCmd = &cli.Command{
 			return err
 		}
 
+		var provCol big.Int
+		if pcs := cctx.String("provider-collateral"); pcs != "" {
+			pc, err := big.FromString(pcs)
+			if err != nil {
+				return fmt.Errorf("failed to parse provider-collateral: %w", err)
+			}
+			provCol = pc
+		}
+
 		if abi.ChainEpoch(dur) < build.MinDealDuration {
 			return xerrors.Errorf("minimum deal duration is %d blocks", build.MinDealDuration)
 		}
@@ -423,6 +436,7 @@ var clientDealCmd = &cli.Command{
 			DealStartEpoch:     abi.ChainEpoch(cctx.Int64("start-epoch")),
 			FastRetrieval:      cctx.Bool("fast-retrieval"),
 			VerifiedDeal:       isVerified,
+			ProviderCollateral: provCol,
 		})
 		if err != nil {
 			return err

cmd/lotus-chainwatch/processor/common_actors.go

@@ -1,8 +1,8 @@
 package processor
 
 import (
+	"bytes"
 	"context"
-	"fmt"
 	"time"
 
 	"golang.org/x/sync/errgroup"
@@ -12,7 +12,12 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/events/state"
+	"github.com/filecoin-project/lotus/chain/types"
+	cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
+	_init "github.com/filecoin-project/specs-actors/actors/builtin/init"
+	"github.com/filecoin-project/specs-actors/actors/util/adt"
+	typegen "github.com/whyrusleeping/cbor-gen"
 )
 
 func (p *Processor) setupCommonActors() error {
@@ -141,49 +146,41 @@ func (p Processor) storeActorAddresses(ctx context.Context, actors map[cid.Cid]A
 	addressToID[builtin.StorageMarketActorAddr] = builtin.StorageMarketActorAddr
 	addressToID[builtin.VerifiedRegistryActorAddr] = builtin.VerifiedRegistryActorAddr
 	addressToID[builtin.BurntFundsActorAddr] = builtin.BurntFundsActorAddr
-
+	initActor, err := p.node.StateGetActor(ctx, builtin.InitActorAddr, types.EmptyTSK)
-	addressesToUpdate := []UpdateAddresses{}
-
-	pred := state.NewStatePredicates(p.node)
-	for _, act := range actors[builtin.InitActorCodeID] {
-		for _, info := range act {
-			changed, val, err := pred.OnInitActorChange(pred.OnAddressMapChange())(ctx, info.parentTsKey, info.tsKey)
-			if err != nil {
-				return err
-			}
-			if !changed {
-				continue
-			}
-			changes := val.(*state.InitActorAddressChanges)
-			for _, add := range changes.Added {
-				addressToID[add.PK] = add.ID
-			}
-			// we'll need to update any addresses that were modified, this indicates a reorg.
-			for _, mod := range changes.Modified {
-				addressesToUpdate = append(addressesToUpdate, UpdateAddresses{
-					Old: mod.From,
-					New: mod.To,
-				})
-			}
-		}
-	}
-
-	updateTx, err := p.db.Begin()
 	if err != nil {
 		return err
 	}
 
-	for _, updates := range addressesToUpdate {
+	initActorRaw, err := p.node.ChainReadObj(ctx, initActor.Head)
-		if _, err := updateTx.Exec(
+	if err != nil {
-			fmt.Sprintf("update id_address_map set id=%s, address=%s where id=%s and address=%s", updates.New.ID, updates.New.PK, updates.Old.ID, updates.Old.PK),
-		); err != nil {
-			return err
-		}
-	}
-	if err := updateTx.Commit(); err != nil {
 		return err
 	}
 
+	var initActorState _init.State
+	if err := initActorState.UnmarshalCBOR(bytes.NewReader(initActorRaw)); err != nil {
+		return err
+	}
+	ctxStore := cw_util.NewAPIIpldStore(ctx, p.node)
+	addrMap, err := adt.AsMap(ctxStore, initActorState.AddressMap)
+	if err != nil {
+		return err
+	}
+	// gross..
+	var actorID typegen.CborInt
+	if err := addrMap.ForEach(&actorID, func(key string) error {
+		longAddr, err := address.NewFromBytes([]byte(key))
+		if err != nil {
+			return err
+		}
+		shortAddr, err := address.NewIDAddress(uint64(actorID))
+		if err != nil {
+			return err
+		}
+		addressToID[longAddr] = shortAddr
+		return nil
+	}); err != nil {
+		return err
+	}
 	tx, err := p.db.Begin()
 	if err != nil {
 		return err
@@ -215,8 +212,10 @@ create temp table iam (like id_address_map excluding constraints) on commit drop
 		return err
 	}
 
-	if _, err := tx.Exec(`insert into id_address_map select * from iam on conflict (id) do nothing`); err != nil {
+	// HACK until chain watch can handle reorgs we need to update this table when ID -> PubKey mappings change
-		return xerrors.Errorf("actor put: %w", err)
+	if _, err := tx.Exec(`insert into id_address_map select * from iam on conflict (id) do update set address = EXCLUDED.address`); err != nil {
+		log.Warnw("Failed to update id_address_map table, this is a known issue")
+		return nil
 	}
 
 	return tx.Commit()

cmd/lotus-chainwatch/processor/miner.go

@@ -319,7 +319,7 @@ func (p *Processor) storeMinerPreCommitInfo(ctx context.Context, miners []minerA
 
 		changes, err := p.getMinerPreCommitChanges(ctx, m)
 		if err != nil {
-			if strings.Contains(err.Error(), "address not found") {
+			if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
 				continue
 			} else {
 				return err
@@ -439,7 +439,7 @@ func (p *Processor) storeMinerSectorInfo(ctx context.Context, miners []minerActo
 	for _, m := range miners {
 		changes, err := p.getMinerSectorChanges(ctx, m)
 		if err != nil {
-			if strings.Contains(err.Error(), "address not found") {
+			if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
 				continue
 			} else {
 				return err
@@ -518,7 +518,7 @@ func (p *Processor) getMinerPartitionsDifferences(ctx context.Context, miners []
 		m := m
 		grp.Go(func() error {
 			if err := p.diffMinerPartitions(ctx, m, events); err != nil {
-				if strings.Contains(err.Error(), "address not found") {
+				if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
 					return nil
 				}
 				return err
@@ -873,7 +873,7 @@ func (p *Processor) storeMinersActorInfoState(ctx context.Context, miners []mine
 	for _, m := range miners {
 		mi, err := p.node.StateMinerInfo(ctx, m.common.addr, m.common.tsKey)
 		if err != nil {
-			if strings.Contains(err.Error(), "address not found") {
+			if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
 				continue
 			} else {
 				return err

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
 	github.com/filecoin-project/go-data-transfer v0.6.1
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
-	github.com/filecoin-project/go-fil-markets v0.5.5
+	github.com/filecoin-project/go-fil-markets v0.5.6-0.20200814021159-7be996ed8ccb
 	github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
 	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6

go.sum

@@ -240,8 +240,8 @@ github.com/filecoin-project/go-data-transfer v0.6.1 h1:EA6X8fSiBRNVVwKm5pA7+njZn
 github.com/filecoin-project/go-data-transfer v0.6.1/go.mod h1:uRYBRKVBVM12CSusBtVrzDHkVw/3DKZpkxKJVP1Ydas=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
-github.com/filecoin-project/go-fil-markets v0.5.5 h1:6bDUXXWgMfhHLdp6KJJGfffOwM4lA7I8yJ9bSFLgnbQ=
+github.com/filecoin-project/go-fil-markets v0.5.6-0.20200814021159-7be996ed8ccb h1:eCLqJb1tmhMCWUFAfJuSyyv/qLrqiAhICLjhUcbi4x8=
-github.com/filecoin-project/go-fil-markets v0.5.5/go.mod h1:9Sbm+N/WW2QpcmeDgEcQo7BJMPDbDpfHOvsYS9kT7zs=
+github.com/filecoin-project/go-fil-markets v0.5.6-0.20200814021159-7be996ed8ccb/go.mod h1:SJApXAKr5jyGpbzDEOhvemui0pih7hhT8r2MXJxCP1E=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM=
 github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI=
@@ -254,6 +254,8 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261/g
 github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
 github.com/filecoin-project/go-statemachine v0.0.0-20200730031800-c3336614d2a7 h1:KAF3WM/xSnl6G6RHX8vDJthg4+e4PSgBh72//6c6Qvc=
 github.com/filecoin-project/go-statemachine v0.0.0-20200730031800-c3336614d2a7/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
+github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 h1:Jbburj7Ih2iaJ/o5Q9A+EAeTabME6YII7FLi9SKUf5c=
+github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
 github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ=
 github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg=

markets/storageadapter/client.go

@@ -5,6 +5,7 @@ package storageadapter
 import (
 	"bytes"
 	"context"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 
 	"golang.org/x/xerrors"
 
@@ -224,13 +225,15 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor
 	return res.IDs[dealIdx], nil
 }
 
+const clientOverestimation = 2
+
 func (c *ClientNodeAdapter) DealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, isVerified bool) (abi.TokenAmount, abi.TokenAmount, error) {
 	bounds, err := c.StateDealProviderCollateralBounds(ctx, size, isVerified, types.EmptyTSK)
 	if err != nil {
 		return abi.TokenAmount{}, abi.TokenAmount{}, err
 	}
 
-	return bounds.Min, bounds.Max, nil
+	return big.Mul(bounds.Min, big.NewInt(clientOverestimation)), bounds.Max, nil
 }
 
 func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId abi.DealID, cb storagemarket.DealSectorCommittedCallback) error {

node/impl/client/client.go

@@ -36,7 +36,6 @@ import (
 	"github.com/filecoin-project/go-multistore"
 	"github.com/filecoin-project/go-padreader"
 	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
@@ -157,7 +156,7 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams)
 		StartEpoch:    dealStart,
 		EndEpoch:      calcDealExpiration(params.MinBlocksDuration, md, dealStart),
 		Price:         params.EpochPrice,
-		Collateral:    big.Zero(),
+		Collateral:    params.ProviderCollateral,
 		Rt:            rt,
 		FastRetrieval: params.FastRetrieval,
 		VerifiedDeal:  params.VerifiedDeal,