Merge remote-tracking branch 'origin/next' into feat/fix-rename-miner-withdraw

api/api_common.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
+	ma "github.com/multiformats/go-multiaddr"
 
 	"github.com/filecoin-project/go-jsonrpc/auth"
 
@@ -28,6 +29,7 @@ type Common interface {
 	NetDisconnect(context.Context, peer.ID) error
 	NetFindPeer(context.Context, peer.ID) (peer.AddrInfo, error)
 	NetPubsubScores(context.Context) ([]PubsubScore, error)
+	NetAutoNatStatus(context.Context) (NatInfo, error)
 
 	// MethodGroup: Common
 
@@ -65,3 +67,8 @@ type Version struct {
 func (v Version) String() string {
 	return fmt.Sprintf("%s+api%s", v.Version, v.APIVersion.String())
 }
+
+type NatInfo struct {
+	Reachability network.Reachability
+	PublicAddr   ma.Multiaddr
+}

api/api_full.go

@@ -319,7 +319,6 @@ type FullNode interface {
 	StateSectorExpiration(context.Context, address.Address, abi.SectorNumber, types.TipSetKey) (*SectorExpiration, error)
 	// StateSectorPartition finds deadline/partition with the specified sector
 	StateSectorPartition(ctx context.Context, maddr address.Address, sectorNumber abi.SectorNumber, tok types.TipSetKey) (*SectorLocation, error)
-	StatePledgeCollateral(context.Context, types.TipSetKey) (types.BigInt, error)
 	// StateSearchMsg searches for a message in the chain, and returns its receipt and the tipset where it was executed
 	StateSearchMsg(context.Context, cid.Cid) (*MsgLookup, error)
 	// StateWaitMsg looks back in the chain for a message. If not found, it blocks until the
@@ -610,14 +609,15 @@ type MethodCall struct {
 }
 
 type StartDealParams struct {
-	Data              *storagemarket.DataRef
+	Data               *storagemarket.DataRef
-	Wallet            address.Address
+	Wallet             address.Address
-	Miner             address.Address
+	Miner              address.Address
-	EpochPrice        types.BigInt
+	EpochPrice         types.BigInt
-	MinBlocksDuration uint64
+	MinBlocksDuration  uint64
-	DealStartEpoch    abi.ChainEpoch
+	ProviderCollateral big.Int
-	FastRetrieval     bool
+	DealStartEpoch     abi.ChainEpoch
-	VerifiedDeal      bool
+	FastRetrieval      bool
+	VerifiedDeal       bool
 }
 
 type IpldObject struct {

api/apistruct/struct.go

@@ -49,6 +49,7 @@ type CommonStruct struct {
 		NetDisconnect    func(context.Context, peer.ID) error                          `perm:"write"`
 		NetFindPeer      func(context.Context, peer.ID) (peer.AddrInfo, error)         `perm:"read"`
 		NetPubsubScores  func(context.Context) ([]api.PubsubScore, error)              `perm:"read"`
+		NetAutoNatStatus func(context.Context) (api.NatInfo, error)                    `perm:"read"`
 
 		ID      func(context.Context) (peer.ID, error)     `perm:"read"`
 		Version func(context.Context) (api.Version, error) `perm:"read"`
@@ -163,7 +164,6 @@ type FullNodeStruct struct {
 		StateReplay                        func(context.Context, types.TipSetKey, cid.Cid) (*api.InvocResult, error)                                           `perm:"read"`
 		StateGetActor                      func(context.Context, address.Address, types.TipSetKey) (*types.Actor, error)                                       `perm:"read"`
 		StateReadState                     func(context.Context, address.Address, types.TipSetKey) (*api.ActorState, error)                                    `perm:"read"`
-		StatePledgeCollateral              func(context.Context, types.TipSetKey) (types.BigInt, error)                                                        `perm:"read"`
 		StateWaitMsg                       func(ctx context.Context, cid cid.Cid, confidence uint64) (*api.MsgLookup, error)                                   `perm:"read"`
 		StateSearchMsg                     func(context.Context, cid.Cid) (*api.MsgLookup, error)                                                              `perm:"read"`
 		StateListMiners                    func(context.Context, types.TipSetKey) ([]address.Address, error)                                                   `perm:"read"`
@@ -331,6 +331,7 @@ func (c *CommonStruct) AuthNew(ctx context.Context, perms []auth.Permission) ([]
 func (c *CommonStruct) NetPubsubScores(ctx context.Context) ([]api.PubsubScore, error) {
 	return c.Internal.NetPubsubScores(ctx)
 }
+
 func (c *CommonStruct) NetConnectedness(ctx context.Context, pid peer.ID) (network.Connectedness, error) {
 	return c.Internal.NetConnectedness(ctx, pid)
 }
@@ -355,6 +356,10 @@ func (c *CommonStruct) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInf
 	return c.Internal.NetFindPeer(ctx, p)
 }
 
+func (c *CommonStruct) NetAutoNatStatus(ctx context.Context) (api.NatInfo, error) {
+	return c.Internal.NetAutoNatStatus(ctx)
+}
+
 // ID implements API.ID
 func (c *CommonStruct) ID(ctx context.Context) (peer.ID, error) {
 	return c.Internal.ID(ctx)
@@ -732,10 +737,6 @@ func (c *FullNodeStruct) StateReadState(ctx context.Context, addr address.Addres
 	return c.Internal.StateReadState(ctx, addr, tsk)
 }
 
-func (c *FullNodeStruct) StatePledgeCollateral(ctx context.Context, tsk types.TipSetKey) (types.BigInt, error) {
-	return c.Internal.StatePledgeCollateral(ctx, tsk)
-}
-
 func (c *FullNodeStruct) StateWaitMsg(ctx context.Context, msgc cid.Cid, confidence uint64) (*api.MsgLookup, error) {
 	return c.Internal.StateWaitMsg(ctx, msgc, confidence)
 }

chain/messagepool/block_proba.go

@@ -1,20 +1,29 @@
 package messagepool
 
-import "math"
+import (
+	"math"
+	"sync"
+)
+
+var noWinnersProbCache []float64
+var noWinnersProbOnce sync.Once
 
 func noWinnersProb() []float64 {
-	poissPdf := func(x float64) float64 {
+	noWinnersProbOnce.Do(func() {
-		const Mu = 5
+		poissPdf := func(x float64) float64 {
-		lg, _ := math.Lgamma(x + 1)
+			const Mu = 5
-		result := math.Exp((math.Log(Mu) * x) - lg - Mu)
+			lg, _ := math.Lgamma(x + 1)
-		return result
+			result := math.Exp((math.Log(Mu) * x) - lg - Mu)
-	}
+			return result
+		}
 
-	out := make([]float64, 0, MaxBlocks)
+		out := make([]float64, 0, MaxBlocks)
-	for i := 0; i < MaxBlocks; i++ {
+		for i := 0; i < MaxBlocks; i++ {
-		out = append(out, poissPdf(float64(i)))
+			out = append(out, poissPdf(float64(i)))
-	}
+		}
-	return out
+		noWinnersProbCache = out
+	})
+	return noWinnersProbCache
 }
 
 func binomialCoefficient(n, k float64) float64 {

chain/messagepool/selection.go

@@ -22,15 +22,17 @@ var bigBlockGasLimit = big.NewInt(build.BlockGasLimit)
 const MaxBlocks = 15
 
 type msgChain struct {
-	msgs      []*types.SignedMessage
+	msgs         []*types.SignedMessage
-	gasReward *big.Int
+	gasReward    *big.Int
-	gasLimit  int64
+	gasLimit     int64
-	gasPerf   float64
+	gasPerf      float64
-	effPerf   float64
+	effPerf      float64
-	valid     bool
+	bp           float64
-	merged    bool
+	parentOffset float64
-	next      *msgChain
+	valid        bool
-	prev      *msgChain
+	merged       bool
+	next         *msgChain
+	prev         *msgChain
 }
 
 func (mp *MessagePool) SelectMessages(ts *types.TipSet, tq float64) ([]*types.SignedMessage, error) {
@@ -179,8 +181,20 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64
 			}
 
 			chain.merged = true
+			// adjust the effective pefromance for all subsequent chains
+			if next := chain.next; next != nil && next.effPerf > 0 {
+				next.effPerf += next.parentOffset
+				for next = next.next; next != nil && next.effPerf > 0; next = next.next {
+					next.setEffPerf()
+				}
+			}
 			result = append(result, chain.msgs...)
 			gasLimit -= chainGasLimit
+
+			// resort to account for already merged chains and effective performance adjustments
+			sort.Slice(chains[i+1:], func(i, j int) bool {
+				return chains[i].BeforeEffective(chains[j])
+			})
 			continue
 		}
 
@@ -810,18 +824,13 @@ func (mc *msgChain) Trim(gasLimit int64, mp *MessagePool, baseFee types.BigInt,
 		mc.gasReward = new(big.Int).Sub(mc.gasReward, gasReward)
 		mc.gasLimit -= mc.msgs[i].Message.GasLimit
 		if mc.gasLimit > 0 {
-			bp := 1.0
-			if mc.gasPerf != 0 { // prevent div by 0
-				bp = mc.effPerf / mc.gasPerf
-			}
-
 			mc.gasPerf = mp.getGasPerf(mc.gasReward, mc.gasLimit)
-
+			if mc.bp != 0 {
-			if mc.effPerf != 0 { // keep effPerf 0 if it is 0
+				mc.setEffPerf()
-				mc.effPerf = bp * mc.gasPerf
 			}
 		} else {
 			mc.gasPerf = 0
+			mc.effPerf = 0
 		}
 		i--
 	}
@@ -849,7 +858,19 @@ func (mc *msgChain) Invalidate() {
 }
 
 func (mc *msgChain) SetEffectivePerf(bp float64) {
-	mc.effPerf = mc.gasPerf * bp
+	mc.bp = bp
+	mc.setEffPerf()
+}
+
+func (mc *msgChain) setEffPerf() {
+	effPerf := mc.gasPerf * mc.bp
+	if effPerf > 0 && mc.prev != nil {
+		effPerfWithParent := (effPerf*float64(mc.gasLimit) + mc.prev.effPerf*float64(mc.prev.gasLimit)) / float64(mc.gasLimit+mc.prev.gasLimit)
+		mc.parentOffset = effPerf - effPerfWithParent
+		effPerf = effPerfWithParent
+	}
+	mc.effPerf = effPerf
+
 }
 
 func (mc *msgChain) SetNullEffectivePerf() {
@@ -861,7 +882,8 @@ func (mc *msgChain) SetNullEffectivePerf() {
 }
 
 func (mc *msgChain) BeforeEffective(other *msgChain) bool {
-	return mc.effPerf > other.effPerf ||
+	// move merged chains to the front so we can discard them earlier
+	return (mc.merged && !other.merged) || mc.effPerf > other.effPerf ||
 		(mc.effPerf == other.effPerf && mc.gasPerf > other.gasPerf) ||
 		(mc.effPerf == other.effPerf && mc.gasPerf == other.gasPerf && mc.gasReward.Cmp(other.gasReward) > 0)
 }

chain/messagepool/selection_test.go

@@ -755,9 +755,9 @@ func TestOptimalMessageSelection2(t *testing.T) {
 	nMessages := int(5 * build.BlockGasLimit / gasLimit)
 	for i := 0; i < nMessages; i++ {
 		bias := (nMessages - i) / 3
-		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(10000+i%3+bias))
+		m := makeTestMessage(w1, a1, a2, uint64(i), gasLimit, uint64(200000+i%3+bias))
 		mustAdd(t, mp, m)
-		m = makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(1+i%3+bias))
+		m = makeTestMessage(w2, a2, a1, uint64(i), gasLimit, uint64(190000+i%3+bias))
 		mustAdd(t, mp, m)
 	}
 
@@ -771,16 +771,26 @@ func TestOptimalMessageSelection2(t *testing.T) {
 		t.Fatalf("expected %d messages, but got %d", expectedMsgs, len(msgs))
 	}
 
-	nextNonce := uint64(0)
+	var nFrom1, nFrom2 int
+	var nextNonce1, nextNonce2 uint64
 	for _, m := range msgs {
-		if m.Message.From != a2 {
+		if m.Message.From == a1 {
-			t.Fatal("expected message from a2")
+			if m.Message.Nonce != nextNonce1 {
+				t.Fatalf("expected nonce %d but got %d", nextNonce1, m.Message.Nonce)
+			}
+			nextNonce1++
+			nFrom1++
+		} else {
+			if m.Message.Nonce != nextNonce2 {
+				t.Fatalf("expected nonce %d but got %d", nextNonce2, m.Message.Nonce)
+			}
+			nextNonce2++
+			nFrom2++
 		}
+	}
 
-		if m.Message.Nonce != nextNonce {
+	if nFrom1 > nFrom2 {
-			t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce)
+		t.Fatalf("expected more messages from a2 than a1; nFrom1=%d nFrom2=%d", nFrom1, nFrom2)
-		}
-		nextNonce++
 	}
 }
 
@@ -824,8 +834,8 @@ func TestOptimalMessageSelection3(t *testing.T) {
 	nMessages := int(build.BlockGasLimit/gasLimit) + 1
 	for i := 0; i < nMessages; i++ {
 		for j := 0; j < nActors; j++ {
-			bias := (nActors-j)*nMessages + (nMessages+2-i)/(3*nActors) + i%3
+			premium := 500000 + 20000*(nActors-j) + (nMessages+2-i)/(3*nActors) + i%3
-			m := makeTestMessage(wallets[j], actors[j], actors[j%nActors], uint64(i), gasLimit, uint64(1+bias))
+			m := makeTestMessage(wallets[j], actors[j], actors[j%nActors], uint64(i), gasLimit, uint64(premium))
 			mustAdd(t, mp, m)
 		}
 	}
@@ -840,30 +850,33 @@ func TestOptimalMessageSelection3(t *testing.T) {
 		t.Fatalf("expected %d messages, but got %d", expectedMsgs, len(msgs))
 	}
 
-	nextNonce := uint64(0)
+	whoIs := func(a address.Address) int {
-	a := actors[len(actors)/2-1]
+		for i, aa := range actors {
-	for _, m := range msgs {
+			if a == aa {
-		if m.Message.From != a {
+				return i
-			who := 0
-			for i, a := range actors {
-				if a == m.Message.From {
-					who = i
-					break
-				}
 			}
-			t.Fatalf("expected message from last actor, but got from %d instead", who)
+		}
+		return -1
+	}
+
+	nonces := make([]uint64, nActors)
+	for _, m := range msgs {
+		who := whoIs(m.Message.From)
+		if who < 3 {
+			t.Fatalf("got message from %dth actor", who)
 		}
 
+		nextNonce := nonces[who]
 		if m.Message.Nonce != nextNonce {
 			t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce)
 		}
-		nextNonce++
+		nonces[who]++
 	}
 }
 
-func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) {
+func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand, getPremium func() uint64) (float64, float64) {
-	// in this test we use 100 actors and send 10 blocks of messages.
+	// in this test we use 300 actors and send 10 blocks of messages.
-	// actors send with an exponentially decreasing premium.
+	// actors send with an randomly distributed premium dictated by the getPremium function.
 	// a number of miners select with varying ticket quality and we compare the
 	// capacity and rewards of greedy selection -vs- optimal selection
 
@@ -906,33 +919,37 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) {
 	for i := 0; i < nMessages; i++ {
 		from := rng.Intn(nActors)
 		to := rng.Intn(nActors)
-		premium := 20000*math.Exp(-3.*rand.Float64()) + 5000
 		nonce := nonces[from]
 		nonces[from]++
-		m := makeTestMessage(wallets[from], actors[from], actors[to], uint64(nonce), gasLimit, uint64(premium))
+		premium := getPremium()
+		m := makeTestMessage(wallets[from], actors[from], actors[to], nonce, gasLimit, premium)
 		mustAdd(t, mp, m)
 	}
 
+	logging.SetLogLevel("messagepool", "error")
+
 	// 1. greedy selection
 	greedyMsgs, err := mp.selectMessagesGreedy(ts, ts)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// 2. optimal selection
+	capacityBoost := 0.0
-	minersRand := rng.Float64()
+	rewardBoost := 0.0
-	winerProba := noWinnersProb()
+	const runs = 1
-	i := 0
+	for i := 0; i < runs; i++ {
-	for ; i < MaxBlocks && minersRand > 0; i++ {
+		// 2. optimal selection
-		minersRand -= winerProba[i]
+		minersRand := rng.Float64()
-	}
+		winerProba := noWinnersProb()
-	nMiners := i
+		i := 0
-	if nMiners == 0 {
+		for ; i < MaxBlocks && minersRand > 0; i++ {
-		nMiners = 1
+			minersRand -= winerProba[i]
-	}
+		}
+		nMiners := i
+		if nMiners == 0 {
+			nMiners = 1
+		}
 
-	logging.SetLogLevel("messagepool", "error")
-	for i := 0; i < 1; i++ {
 		optMsgs := make(map[cid.Cid]*types.SignedMessage)
 		for j := 0; j < nMiners; j++ {
 			tq := rng.Float64()
@@ -945,9 +962,12 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) {
 			}
 		}
 
+		boost := float64(len(optMsgs)) / float64(len(greedyMsgs))
+		capacityBoost += boost
+
 		t.Logf("nMiners: %d", nMiners)
-		t.Logf("greedy capacity %d, optimal capacity %d (x%.1f)", len(greedyMsgs),
+		t.Logf("greedy capacity %d, optimal capacity %d (x %.1f )", len(greedyMsgs),
-			len(optMsgs), float64(len(optMsgs))/float64(len(greedyMsgs)))
+			len(optMsgs), boost)
 		if len(greedyMsgs) > len(optMsgs) {
 			t.Fatal("greedy capacity higher than optimal capacity; wtf")
 		}
@@ -965,20 +985,68 @@ func testCompetitiveMessageSelection(t *testing.T, rng *rand.Rand) {
 		nMinersBig := big.NewInt(int64(nMiners))
 		greedyAvgReward, _ := new(big.Rat).SetFrac(greedyReward, nMinersBig).Float64()
 		optimalAvgReward, _ := new(big.Rat).SetFrac(optReward, nMinersBig).Float64()
-		t.Logf("greedy reward: %.0f, optimal reward: %.0f (x%.1f)", greedyAvgReward,
-			optimalAvgReward, optimalAvgReward/greedyAvgReward)
 
-		if greedyReward.Cmp(optReward) > 0 {
+		boost = optimalAvgReward / greedyAvgReward
-			t.Fatal("greedy reward raw higher than optimal reward; booh")
+		rewardBoost += boost
-		}
+		t.Logf("greedy reward: %.0f, optimal reward: %.0f (x %.1f )", greedyAvgReward,
+			optimalAvgReward, boost)
+
 	}
+
+	capacityBoost /= runs
+	rewardBoost /= runs
+	t.Logf("Average capacity boost: %f", capacityBoost)
+	t.Logf("Average reward boost: %f", rewardBoost)
+
 	logging.SetLogLevel("messagepool", "info")
+
+	return capacityBoost, rewardBoost
 }
 
-func TestCompetitiveMessageSelection(t *testing.T) {
+func makeExpPremiumDistribution(rng *rand.Rand) func() uint64 {
-	seeds := []int64{1947, 1976, 2020, 2100, 10000}
+	return func() uint64 {
-	for _, seed := range seeds {
+		premium := 20000*math.Exp(-3.*rng.Float64()) + 5000
-		t.Log("running competitve message selection with seed", seed)
+		return uint64(premium)
-		testCompetitiveMessageSelection(t, rand.New(rand.NewSource(seed)))
 	}
 }
+
+func makeZipfPremiumDistribution(rng *rand.Rand) func() uint64 {
+	zipf := rand.NewZipf(rng, 1.001, 1, 40000)
+	return func() uint64 {
+		return zipf.Uint64() + 10000
+	}
+}
+
+func TestCompetitiveMessageSelectionExp(t *testing.T) {
+	var capacityBoost, rewardBoost float64
+	seeds := []int64{1947, 1976, 2020, 2100, 10000, 143324, 432432, 131, 32, 45}
+	for _, seed := range seeds {
+		t.Log("running competitive message selection with Exponential premium distribution and seed", seed)
+		rng := rand.New(rand.NewSource(seed))
+		cb, rb := testCompetitiveMessageSelection(t, rng, makeExpPremiumDistribution(rng))
+		capacityBoost += cb
+		rewardBoost += rb
+	}
+
+	capacityBoost /= float64(len(seeds))
+	rewardBoost /= float64(len(seeds))
+	t.Logf("Average capacity boost across all seeds: %f", capacityBoost)
+	t.Logf("Average reward boost across all seeds: %f", rewardBoost)
+}
+
+func TestCompetitiveMessageSelectionZipf(t *testing.T) {
+	var capacityBoost, rewardBoost float64
+	seeds := []int64{1947, 1976, 2020, 2100, 10000, 143324, 432432, 131, 32, 45}
+	for _, seed := range seeds {
+		t.Log("running competitive message selection with Zipf premium distribution and seed", seed)
+		rng := rand.New(rand.NewSource(seed))
+		cb, rb := testCompetitiveMessageSelection(t, rng, makeZipfPremiumDistribution(rng))
+		capacityBoost += cb
+		rewardBoost += rb
+	}
+
+	capacityBoost /= float64(len(seeds))
+	rewardBoost /= float64(len(seeds))
+	t.Logf("Average capacity boost across all seeds: %f", capacityBoost)
+	t.Logf("Average reward boost across all seeds: %f", rewardBoost)
+}

cli/client.go

@@ -311,6 +311,10 @@ var clientDealCmd = &cli.Command{
 			Usage: "indicate that the deal counts towards verified client total",
 			Value: false,
 		},
+		&cli.StringFlag{
+			Name:  "provider-collateral",
+			Usage: "specify the requested provider collateral the miner should put up",
+		},
 		&CidBaseFlag,
 	},
 	Action: func(cctx *cli.Context) error {
@@ -351,6 +355,15 @@ var clientDealCmd = &cli.Command{
 			return err
 		}
 
+		var provCol big.Int
+		if pcs := cctx.String("provider-collateral"); pcs != "" {
+			pc, err := big.FromString(pcs)
+			if err != nil {
+				return fmt.Errorf("failed to parse provider-collateral: %w", err)
+			}
+			provCol = pc
+		}
+
 		if abi.ChainEpoch(dur) < build.MinDealDuration {
 			return xerrors.Errorf("minimum deal duration is %d blocks", build.MinDealDuration)
 		}
@@ -415,14 +428,15 @@ var clientDealCmd = &cli.Command{
 		}
 
 		proposal, err := api.ClientStartDeal(ctx, &lapi.StartDealParams{
-			Data:              ref,
+			Data:               ref,
-			Wallet:            a,
+			Wallet:             a,
-			Miner:             miner,
+			Miner:              miner,
-			EpochPrice:        types.BigInt(price),
+			EpochPrice:         types.BigInt(price),
-			MinBlocksDuration: uint64(dur),
+			MinBlocksDuration:  uint64(dur),
-			DealStartEpoch:    abi.ChainEpoch(cctx.Int64("start-epoch")),
+			DealStartEpoch:     abi.ChainEpoch(cctx.Int64("start-epoch")),
-			FastRetrieval:     cctx.Bool("fast-retrieval"),
+			FastRetrieval:      cctx.Bool("fast-retrieval"),
-			VerifiedDeal:      isVerified,
+			VerifiedDeal:       isVerified,
+			ProviderCollateral: provCol,
 		})
 		if err != nil {
 			return err

cli/net.go

@@ -24,6 +24,7 @@ var netCmd = &cli.Command{
 		NetId,
 		netFindPeer,
 		netScores,
+		NetReachability,
 	},
 }
 
@@ -202,3 +203,28 @@ var netFindPeer = &cli.Command{
 		return nil
 	},
 }
+
+var NetReachability = &cli.Command{
+	Name:  "reachability",
+	Usage: "Print information about reachability from the internet",
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := GetAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := ReqContext(cctx)
+
+		i, err := api.NetAutoNatStatus(ctx)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("AutoNAT status: ", i.Reachability.String())
+		if i.PublicAddr != nil {
+			fmt.Println("Public address: ", i.PublicAddr.String())
+		}
+		return nil
+	},
+}

cli/state.go

@@ -52,7 +52,6 @@ var stateCmd = &cli.Command{
 		statePowerCmd,
 		stateSectorsCmd,
 		stateActiveSectorsCmd,
-		statePledgeCollateralCmd,
 		stateListActorsCmd,
 		stateListMinersCmd,
 		stateCircSupplyCmd,
@@ -386,33 +385,6 @@ var stateReplaySetCmd = &cli.Command{
 	},
 }
 
-var statePledgeCollateralCmd = &cli.Command{
-	Name:  "pledge-collateral",
-	Usage: "Get minimum miner pledge collateral",
-	Action: func(cctx *cli.Context) error {
-		api, closer, err := GetFullNodeAPI(cctx)
-		if err != nil {
-			return err
-		}
-		defer closer()
-
-		ctx := ReqContext(cctx)
-
-		ts, err := LoadTipSet(ctx, cctx, api)
-		if err != nil {
-			return err
-		}
-
-		coll, err := api.StatePledgeCollateral(ctx, ts.Key())
-		if err != nil {
-			return err
-		}
-
-		fmt.Println(types.FIL(coll))
-		return nil
-	},
-}
-
 var stateGetDealSetCmd = &cli.Command{
 	Name:      "get-deal",
 	Usage:     "View on-chain deal info",

cmd/lotus-chainwatch/processor/common_actors.go

@@ -1,8 +1,8 @@
 package processor
 
 import (
+	"bytes"
 	"context"
-	"fmt"
 	"time"
 
 	"golang.org/x/sync/errgroup"
@@ -12,7 +12,12 @@ import (
 
 	"github.com/filecoin-project/go-address"
 	"github.com/filecoin-project/lotus/chain/events/state"
+	"github.com/filecoin-project/lotus/chain/types"
+	cw_util "github.com/filecoin-project/lotus/cmd/lotus-chainwatch/util"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
+	_init "github.com/filecoin-project/specs-actors/actors/builtin/init"
+	"github.com/filecoin-project/specs-actors/actors/util/adt"
+	typegen "github.com/whyrusleeping/cbor-gen"
 )
 
 func (p *Processor) setupCommonActors() error {
@@ -141,49 +146,41 @@ func (p Processor) storeActorAddresses(ctx context.Context, actors map[cid.Cid]A
 	addressToID[builtin.StorageMarketActorAddr] = builtin.StorageMarketActorAddr
 	addressToID[builtin.VerifiedRegistryActorAddr] = builtin.VerifiedRegistryActorAddr
 	addressToID[builtin.BurntFundsActorAddr] = builtin.BurntFundsActorAddr
-
+	initActor, err := p.node.StateGetActor(ctx, builtin.InitActorAddr, types.EmptyTSK)
-	addressesToUpdate := []UpdateAddresses{}
-
-	pred := state.NewStatePredicates(p.node)
-	for _, act := range actors[builtin.InitActorCodeID] {
-		for _, info := range act {
-			changed, val, err := pred.OnInitActorChange(pred.OnAddressMapChange())(ctx, info.parentTsKey, info.tsKey)
-			if err != nil {
-				return err
-			}
-			if !changed {
-				continue
-			}
-			changes := val.(*state.InitActorAddressChanges)
-			for _, add := range changes.Added {
-				addressToID[add.PK] = add.ID
-			}
-			// we'll need to update any addresses that were modified, this indicates a reorg.
-			for _, mod := range changes.Modified {
-				addressesToUpdate = append(addressesToUpdate, UpdateAddresses{
-					Old: mod.From,
-					New: mod.To,
-				})
-			}
-		}
-	}
-
-	updateTx, err := p.db.Begin()
 	if err != nil {
 		return err
 	}
 
-	for _, updates := range addressesToUpdate {
+	initActorRaw, err := p.node.ChainReadObj(ctx, initActor.Head)
-		if _, err := updateTx.Exec(
+	if err != nil {
-			fmt.Sprintf("update id_address_map set id=%s, address=%s where id=%s and address=%s", updates.New.ID, updates.New.PK, updates.Old.ID, updates.Old.PK),
-		); err != nil {
-			return err
-		}
-	}
-	if err := updateTx.Commit(); err != nil {
 		return err
 	}
 
+	var initActorState _init.State
+	if err := initActorState.UnmarshalCBOR(bytes.NewReader(initActorRaw)); err != nil {
+		return err
+	}
+	ctxStore := cw_util.NewAPIIpldStore(ctx, p.node)
+	addrMap, err := adt.AsMap(ctxStore, initActorState.AddressMap)
+	if err != nil {
+		return err
+	}
+	// gross..
+	var actorID typegen.CborInt
+	if err := addrMap.ForEach(&actorID, func(key string) error {
+		longAddr, err := address.NewFromBytes([]byte(key))
+		if err != nil {
+			return err
+		}
+		shortAddr, err := address.NewIDAddress(uint64(actorID))
+		if err != nil {
+			return err
+		}
+		addressToID[longAddr] = shortAddr
+		return nil
+	}); err != nil {
+		return err
+	}
 	tx, err := p.db.Begin()
 	if err != nil {
 		return err
@@ -215,8 +212,10 @@ create temp table iam (like id_address_map excluding constraints) on commit drop
 		return err
 	}
 
-	if _, err := tx.Exec(`insert into id_address_map select * from iam on conflict (id) do nothing`); err != nil {
+	// HACK until chain watch can handle reorgs we need to update this table when ID -> PubKey mappings change
-		return xerrors.Errorf("actor put: %w", err)
+	if _, err := tx.Exec(`insert into id_address_map select * from iam on conflict (id) do update set address = EXCLUDED.address`); err != nil {
+		log.Warnw("Failed to update id_address_map table, this is a known issue")
+		return nil
 	}
 
 	return tx.Commit()

cmd/lotus-chainwatch/processor/miner.go

@@ -319,7 +319,7 @@ func (p *Processor) storeMinerPreCommitInfo(ctx context.Context, miners []minerA
 
 		changes, err := p.getMinerPreCommitChanges(ctx, m)
 		if err != nil {
-			if strings.Contains(err.Error(), "address not found") {
+			if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
 				continue
 			} else {
 				return err
@@ -439,7 +439,7 @@ func (p *Processor) storeMinerSectorInfo(ctx context.Context, miners []minerActo
 	for _, m := range miners {
 		changes, err := p.getMinerSectorChanges(ctx, m)
 		if err != nil {
-			if strings.Contains(err.Error(), "address not found") {
+			if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
 				continue
 			} else {
 				return err
@@ -518,7 +518,7 @@ func (p *Processor) getMinerPartitionsDifferences(ctx context.Context, miners []
 		m := m
 		grp.Go(func() error {
 			if err := p.diffMinerPartitions(ctx, m, events); err != nil {
-				if strings.Contains(err.Error(), "address not found") {
+				if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
 					return nil
 				}
 				return err
@@ -873,7 +873,7 @@ func (p *Processor) storeMinersActorInfoState(ctx context.Context, miners []mine
 	for _, m := range miners {
 		mi, err := p.node.StateMinerInfo(ctx, m.common.addr, m.common.tsKey)
 		if err != nil {
-			if strings.Contains(err.Error(), "address not found") {
+			if strings.Contains(err.Error(), types.ErrActorNotFound.Error()) {
 				continue
 			} else {
 				return err

cmd/lotus-fountain/main.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/filecoin-project/sector-storage/ffiwrapper"
 	"github.com/filecoin-project/specs-actors/actors/abi"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 
@@ -339,13 +340,6 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	collateral, err := h.api.StatePledgeCollateral(r.Context(), types.EmptyTSK)
-	if err != nil {
-		w.WriteHeader(400)
-		w.Write([]byte(err.Error()))
-		return
-	}
-
 	smsg, err := h.api.MpoolPushMessage(h.ctx, &types.Message{
 		Value: types.BigInt(h.sendPerRequest),
 		From:  h.from,
@@ -380,7 +374,7 @@ func (h *handler) mkminer(w http.ResponseWriter, r *http.Request) {
 	createStorageMinerMsg := &types.Message{
 		To:    builtin.StoragePowerActorAddr,
 		From:  h.from,
-		Value: types.BigAdd(collateral, types.BigDiv(collateral, types.NewInt(100))),
+		Value: big.Zero(),
 
 		Method: builtin.MethodsPower.CreateMiner,
 		Params: params,

cmd/lotus-storage-miner/info_all.go

@@ -63,6 +63,11 @@ var infoAllCmd = &cli.Command{
 			return err
 		}
 
+		fmt.Println("\n#: Reachability")
+		if err := lcli.NetReachability.Action(cctx); err != nil {
+			return err
+		}
+
 		// Very Verbose info
 		fmt.Println("\n#: Peers")
 		if err := lcli.NetPeers.Action(cctx); err != nil {
@@ -89,6 +94,11 @@ var infoAllCmd = &cli.Command{
 			return err
 		}
 
+		fmt.Println("\n#: Retrieval Deals")
+		if err := retrievalDealsListCmd.Action(cctx); err != nil {
+			return err
+		}
+
 		fmt.Println("\n#: Sector List")
 		if err := sectorsListCmd.Action(cctx); err != nil {
 			return err

cmd/lotus-storage-miner/init.go

@@ -7,6 +7,7 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -614,11 +615,6 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID,
 		return address.Undef, err
 	}
 
-	collateral, err := api.StatePledgeCollateral(ctx, types.EmptyTSK)
-	if err != nil {
-		return address.Undef, err
-	}
-
 	spt, err := ffiwrapper.SealProofTypeFromSectorSize(abi.SectorSize(ssize))
 	if err != nil {
 		return address.Undef, err
@@ -637,7 +633,7 @@ func createStorageMiner(ctx context.Context, api lapi.FullNode, peerid peer.ID,
 	createStorageMinerMsg := &types.Message{
 		To:    builtin.StoragePowerActorAddr,
 		From:  owner,
-		Value: types.BigAdd(collateral, types.BigDiv(collateral, types.NewInt(100))),
+		Value: big.Zero(),
 
 		Method: builtin.MethodsPower.CreateMiner,
 		Params: params,

extern/sector-storage/sched_watch.go

@@ -87,11 +87,14 @@ func (sh *scheduler) runWorkerWatcher() {
 			}
 
 			log.Warnf("worker %d dropped", wid)
-			select {
+			// send in a goroutine to avoid a deadlock between workerClosing / watchClosing
-			case sh.workerClosing <- wid:
+			go func() {
-			case <-sh.closing:
+				select {
-				return
+				case sh.workerClosing <- wid:
-			}
+				case <-sh.closing:
+					return
+				}
+			}()
 		}
 	}
 }

extern/sector-storage/stats.go

@@ -3,8 +3,8 @@ package sectorstorage
 import "github.com/filecoin-project/sector-storage/storiface"
 
 func (m *Manager) WorkerStats() map[uint64]storiface.WorkerStats {
-	m.sched.workersLk.Lock()
+	m.sched.workersLk.RLock()
-	defer m.sched.workersLk.Unlock()
+	defer m.sched.workersLk.RUnlock()
 
 	out := map[uint64]storiface.WorkerStats{}
 
@@ -22,8 +22,8 @@ func (m *Manager) WorkerStats() map[uint64]storiface.WorkerStats {
 }
 
 func (m *Manager) WorkerJobs() map[uint64][]storiface.WorkerJob {
-	m.sched.workersLk.Lock()
+	m.sched.workersLk.RLock()
-	defer m.sched.workersLk.Unlock()
+	defer m.sched.workersLk.RUnlock()
 
 	out := map[uint64][]storiface.WorkerJob{}
 

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03
 	github.com/filecoin-project/go-data-transfer v0.6.1
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f
-	github.com/filecoin-project/go-fil-markets v0.5.5
+	github.com/filecoin-project/go-fil-markets v0.5.6-0.20200814021159-7be996ed8ccb
 	github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24
 	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6

go.sum

@@ -240,8 +240,8 @@ github.com/filecoin-project/go-data-transfer v0.6.1 h1:EA6X8fSiBRNVVwKm5pA7+njZn
 github.com/filecoin-project/go-data-transfer v0.6.1/go.mod h1:uRYBRKVBVM12CSusBtVrzDHkVw/3DKZpkxKJVP1Ydas=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f h1:GxJzR3oRIMTPtpZ0b7QF8FKPK6/iPAc7trhlL5k/g+s=
 github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ=
-github.com/filecoin-project/go-fil-markets v0.5.5 h1:6bDUXXWgMfhHLdp6KJJGfffOwM4lA7I8yJ9bSFLgnbQ=
+github.com/filecoin-project/go-fil-markets v0.5.6-0.20200814021159-7be996ed8ccb h1:eCLqJb1tmhMCWUFAfJuSyyv/qLrqiAhICLjhUcbi4x8=
-github.com/filecoin-project/go-fil-markets v0.5.5/go.mod h1:9Sbm+N/WW2QpcmeDgEcQo7BJMPDbDpfHOvsYS9kT7zs=
+github.com/filecoin-project/go-fil-markets v0.5.6-0.20200814021159-7be996ed8ccb/go.mod h1:SJApXAKr5jyGpbzDEOhvemui0pih7hhT8r2MXJxCP1E=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24 h1:Jc7vkplmZYVuaEcSXGHDwefvZIdoyyaoGDLqSr8Svms=
 github.com/filecoin-project/go-jsonrpc v0.1.1-0.20200602181149-522144ab4e24/go.mod h1:j6zV//WXIIY5kky873Q3iIKt/ViOE8rcijovmpxrXzM=
 github.com/filecoin-project/go-multistore v0.0.3 h1:vaRBY4YiA2UZFPK57RNuewypB8u0DzzQwqsL0XarpnI=
@@ -254,6 +254,8 @@ github.com/filecoin-project/go-paramfetch v0.0.2-0.20200701152213-3e0f0afdc261/g
 github.com/filecoin-project/go-statemachine v0.0.0-20200714194326-a77c3ae20989/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
 github.com/filecoin-project/go-statemachine v0.0.0-20200730031800-c3336614d2a7 h1:KAF3WM/xSnl6G6RHX8vDJthg4+e4PSgBh72//6c6Qvc=
 github.com/filecoin-project/go-statemachine v0.0.0-20200730031800-c3336614d2a7/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
+github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370 h1:Jbburj7Ih2iaJ/o5Q9A+EAeTabME6YII7FLi9SKUf5c=
+github.com/filecoin-project/go-statemachine v0.0.0-20200813232949-df9b130df370/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig=
 github.com/filecoin-project/go-statestore v0.1.0 h1:t56reH59843TwXHkMcwyuayStBIiWBRilQjQ+5IiwdQ=
 github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg=

markets/storageadapter/client.go

@@ -5,6 +5,7 @@ package storageadapter
 import (
 	"bytes"
 	"context"
+	"github.com/filecoin-project/specs-actors/actors/abi/big"
 
 	"golang.org/x/xerrors"
 
@@ -224,13 +225,15 @@ func (c *ClientNodeAdapter) ValidatePublishedDeal(ctx context.Context, deal stor
 	return res.IDs[dealIdx], nil
 }
 
+const clientOverestimation = 2
+
 func (c *ClientNodeAdapter) DealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, isVerified bool) (abi.TokenAmount, abi.TokenAmount, error) {
 	bounds, err := c.StateDealProviderCollateralBounds(ctx, size, isVerified, types.EmptyTSK)
 	if err != nil {
 		return abi.TokenAmount{}, abi.TokenAmount{}, err
 	}
 
-	return bounds.Min, bounds.Max, nil
+	return big.Mul(bounds.Min, big.NewInt(clientOverestimation)), bounds.Max, nil
 }
 
 func (c *ClientNodeAdapter) OnDealSectorCommitted(ctx context.Context, provider address.Address, dealId abi.DealID, cb storagemarket.DealSectorCommittedCallback) error {

node/config/def.go

@@ -154,8 +154,8 @@ func DefaultStorageMiner() *StorageMiner {
 		},
 
 		Fees: MinerFeeConfig{
-			MaxPreCommitGasFee:  types.FIL(types.FromFil(1)),
+			MaxPreCommitGasFee:  types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05
-			MaxCommitGasFee:     types.FIL(types.FromFil(1)),
+			MaxCommitGasFee:     types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))),
 			MaxWindowPoStGasFee: types.FIL(types.FromFil(50)),
 		},
 

node/impl/client/client.go

@@ -36,7 +36,6 @@ import (
 	"github.com/filecoin-project/go-multistore"
 	"github.com/filecoin-project/go-padreader"
 	"github.com/filecoin-project/specs-actors/actors/abi"
-	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/miner"
 
 	marketevents "github.com/filecoin-project/lotus/markets/loggers"
@@ -157,7 +156,7 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams)
 		StartEpoch:    dealStart,
 		EndEpoch:      calcDealExpiration(params.MinBlocksDuration, md, dealStart),
 		Price:         params.EpochPrice,
-		Collateral:    big.Zero(),
+		Collateral:    params.ProviderCollateral,
 		Rt:            rt,
 		FastRetrieval: params.FastRetrieval,
 		VerifiedDeal:  params.VerifiedDeal,

node/impl/common/common.go

@@ -12,6 +12,7 @@ import (
 	"github.com/libp2p/go-libp2p-core/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 	swarm "github.com/libp2p/go-libp2p-swarm"
+	basichost "github.com/libp2p/go-libp2p/p2p/host/basic"
 	ma "github.com/multiformats/go-multiaddr"
 	"go.uber.org/fx"
 	"golang.org/x/xerrors"
@@ -28,6 +29,7 @@ type CommonAPI struct {
 	fx.In
 
 	APISecret    *dtypes.APIAlg
+	RawHost      lp2p.RawHost
 	Host         host.Host
 	Router       lp2p.BaseIpfsRouting
 	Sk           *dtypes.ScoreKeeper
@@ -113,6 +115,23 @@ func (a *CommonAPI) NetFindPeer(ctx context.Context, p peer.ID) (peer.AddrInfo,
 	return a.Router.FindPeer(ctx, p)
 }
 
+func (a *CommonAPI) NetAutoNatStatus(ctx context.Context) (i api.NatInfo, err error) {
+	autonat := a.RawHost.(*basichost.BasicHost).AutoNat
+
+	var maddr ma.Multiaddr
+	if autonat.Status() == network.ReachabilityPublic {
+		maddr, err = autonat.PublicAddr()
+		if err != nil {
+			return api.NatInfo{}, err
+		}
+	}
+
+	return api.NatInfo{
+		Reachability: autonat.Status(),
+		PublicAddr:   maddr,
+	}, nil
+}
+
 func (a *CommonAPI) ID(context.Context) (peer.ID, error) {
 	return a.Host.ID(), nil
 }

node/impl/full/gas.go

@@ -3,6 +3,7 @@ package full
 import (
 	"context"
 	"math"
+	"math/rand"
 	"sort"
 
 	"github.com/filecoin-project/lotus/build"
@@ -26,7 +27,7 @@ type GasAPI struct {
 	Mpool *messagepool.MessagePool
 }
 
-const MinGasPremium = 10e3
+const MinGasPremium = 100e3
 const MaxSpendOnFeeDenom = 100
 
 func (a *GasAPI) GasEstimateFeeCap(ctx context.Context, msg *types.Message, maxqueueblks int64,
@@ -111,6 +112,7 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64,
 	at := build.BlockGasTarget * int64(blocks) / 2
 	prev := big.Zero()
 
+	premium := big.Zero()
 	for _, price := range prices {
 		at -= price.limit
 		if at > 0 {
@@ -122,17 +124,27 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64,
 			return types.BigAdd(price.price, big.NewInt(1)), nil
 		}
 
-		return types.BigAdd(big.Div(types.BigAdd(price.price, prev), types.NewInt(2)), big.NewInt(1)), nil
+		premium = types.BigAdd(big.Div(types.BigAdd(price.price, prev), types.NewInt(2)), big.NewInt(1))
 	}
 
-	switch nblocksincl {
+	if types.BigCmp(premium, big.Zero()) == 0 {
-	case 1:
+		switch nblocksincl {
-		return types.NewInt(2 * MinGasPremium), nil
+		case 1:
-	case 2:
+			premium = types.NewInt(2 * MinGasPremium)
-		return types.NewInt(1.5 * MinGasPremium), nil
+		case 2:
-	default:
+			premium = types.NewInt(1.5 * MinGasPremium)
-		return types.NewInt(MinGasPremium), nil
+		default:
+			premium = types.NewInt(MinGasPremium)
+		}
 	}
+
+	// add some noise to normalize behaviour of message selection
+	const precision = 32
+	// mean 1, stddev 0.005 => 95% within +-1%
+	noise := 1 + rand.NormFloat64()*0.005
+	premium = types.BigMul(premium, types.NewInt(uint64(noise*(1<<precision))))
+	premium = types.BigDiv(premium, types.NewInt(1<<precision))
+	return premium, nil
 }
 
 func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, _ types.TipSetKey) (int64, error) {

node/impl/full/mpool.go

@@ -114,7 +114,7 @@ func capGasFee(msg *types.Message, maxFee abi.TokenAmount) {
 		return
 	}
 
-	gl := types.BigMul(msg.GasPremium, types.NewInt(uint64(msg.GasLimit)))
+	gl := types.NewInt(uint64(msg.GasLimit))
 	totalFee := types.BigMul(msg.GasFeeCap, gl)
 	minerFee := types.BigMul(msg.GasPremium, gl)
 

node/impl/full/state.go

@@ -258,37 +258,6 @@ func (a *StateAPI) StateMinerPower(ctx context.Context, addr address.Address, ts
 	}, nil
 }
 
-func (a *StateAPI) StatePledgeCollateral(ctx context.Context, tsk types.TipSetKey) (types.BigInt, error) {
-	/*ts, err := a.Chain.GetTipSetFromKey(tsk)
-	if err != nil {
-		return types.EmptyInt, xerrors.Errorf("loading tipset %s: %w", tsk, err)
-	}
-
-	param, err := actors.SerializeParams(&actors.PledgeCollateralParams{Size: types.NewInt(0)})
-	if err != nil {
-		return types.NewInt(0), err
-	}
-
-	ret, aerr := a.StateManager.Call(ctx, &types.Message{
-		From:   actors.StoragePowerAddress,
-		To:     actors.StoragePowerAddress,
-		Method: actors.SPAMethods.PledgeCollateralForSize,
-
-		Params: param,
-	}, ts)
-	if aerr != nil {
-		return types.NewInt(0), xerrors.Errorf("failed to get miner worker addr: %w", err)
-	}
-
-	if ret.MsgRct.ExitCode != 0 {
-		return types.NewInt(0), xerrors.Errorf("failed to get miner worker addr (exit code %d)", ret.MsgRct.ExitCode)
-	}
-
-	return types.BigFromBytes(ret.Return), nil*/
-	log.Error("TODO StatePledgeCollateral")
-	return big.Zero(), nil
-}
-
 func (a *StateAPI) StateCall(ctx context.Context, msg *types.Message, tsk types.TipSetKey) (*api.InvocResult, error) {
 	ts, err := a.Chain.GetTipSetFromKey(tsk)
 	if err != nil {

paychmgr/paych_test.go

@@ -507,6 +507,15 @@ func TestAddVoucherNextLane(t *testing.T) {
 	require.NoError(t, err)
 	require.EqualValues(t, ci.NextLane, 3)
 
+	// Allocate a lane (should be lane 3)
+	lane, err := mgr.AllocateLane(ch)
+	require.NoError(t, err)
+	require.EqualValues(t, lane, 3)
+
+	ci, err = mgr.GetChannelInfo(ch)
+	require.NoError(t, err)
+	require.EqualValues(t, ci.NextLane, 4)
+
 	// Add a voucher in lane 1
 	voucherLane = uint64(1)
 	sv = testCreateVoucher(t, ch, voucherLane, nonce, voucherAmount, fromKeyPrivate)
@@ -515,17 +524,89 @@ func TestAddVoucherNextLane(t *testing.T) {
 
 	ci, err = mgr.GetChannelInfo(ch)
 	require.NoError(t, err)
-	require.EqualValues(t, ci.NextLane, 3)
+	require.EqualValues(t, ci.NextLane, 4)
 
-	// Add a voucher in lane 5
+	// Add a voucher in lane 7
-	voucherLane = uint64(5)
+	voucherLane = uint64(7)
 	sv = testCreateVoucher(t, ch, voucherLane, nonce, voucherAmount, fromKeyPrivate)
 	_, err = mgr.AddVoucher(ctx, ch, sv, nil, minDelta)
 	require.NoError(t, err)
 
 	ci, err = mgr.GetChannelInfo(ch)
 	require.NoError(t, err)
-	require.EqualValues(t, ci.NextLane, 6)
+	require.EqualValues(t, ci.NextLane, 8)
+}
+
+func TestAllocateLane(t *testing.T) {
+	ctx := context.Background()
+
+	// Set up a manager with a single payment channel
+	mgr, ch, _ := testSetupMgrWithChannel(ctx, t)
+
+	// First lane should be 0
+	lane, err := mgr.AllocateLane(ch)
+	require.NoError(t, err)
+	require.EqualValues(t, lane, 0)
+
+	// Next lane should be 1
+	lane, err = mgr.AllocateLane(ch)
+	require.NoError(t, err)
+	require.EqualValues(t, lane, 1)
+}
+
+func TestAllocateLaneWithExistingLaneState(t *testing.T) {
+	ctx := context.Background()
+
+	_, fromKeyPublic := testGenerateKeyPair(t)
+
+	ch := tutils.NewIDAddr(t, 100)
+	from := tutils.NewSECP256K1Addr(t, string(fromKeyPublic))
+	to := tutils.NewSECP256K1Addr(t, "secpTo")
+	fromAcct := tutils.NewActorAddr(t, "fromAct")
+	toAcct := tutils.NewActorAddr(t, "toAct")
+
+	mock := newMockManagerAPI()
+	mock.setAccountState(fromAcct, account.State{Address: from})
+	mock.setAccountState(toAcct, account.State{Address: to})
+
+	store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore()))
+
+	actorBalance := big.NewInt(10)
+	toSend := big.NewInt(1)
+	laneStates := map[uint64]paych.LaneState{
+		2: {
+			Nonce:    1,
+			Redeemed: big.NewInt(4),
+		},
+	}
+
+	act := &types.Actor{
+		Code:    builtin.AccountActorCodeID,
+		Head:    cid.Cid{},
+		Nonce:   0,
+		Balance: actorBalance,
+	}
+
+	lsCid, err := mock.storeLaneStates(laneStates)
+	require.NoError(t, err)
+	mock.setPaychState(ch, act, paych.State{
+		From:            fromAcct,
+		To:              toAcct,
+		ToSend:          toSend,
+		SettlingAt:      abi.ChainEpoch(0),
+		MinSettleHeight: abi.ChainEpoch(0),
+		LaneStates:      lsCid,
+	})
+
+	mgr, err := newManager(store, mock)
+	require.NoError(t, err)
+
+	err = mgr.TrackInboundChannel(ctx, ch)
+	require.NoError(t, err)
+
+	lane, err := mgr.AllocateLane(ch)
+	require.NoError(t, err)
+	require.EqualValues(t, 3, lane)
 }
 
 func TestAddVoucherProof(t *testing.T) {
@@ -575,23 +656,6 @@ func TestAddVoucherProof(t *testing.T) {
 	require.Len(t, ci.Vouchers[0].Proof, 1)
 }
 
-func TestAllocateLane(t *testing.T) {
-	ctx := context.Background()
-
-	// Set up a manager with a single payment channel
-	mgr, ch, _ := testSetupMgrWithChannel(ctx, t)
-
-	// First lane should be 0
-	lane, err := mgr.AllocateLane(ch)
-	require.NoError(t, err)
-	require.EqualValues(t, lane, 0)
-
-	// Next lane should be 1
-	lane, err = mgr.AllocateLane(ch)
-	require.NoError(t, err)
-	require.EqualValues(t, lane, 1)
-}
-
 func TestNextNonceForLane(t *testing.T) {
 	ctx := context.Background()
 

paychmgr/state.go

@@ -2,7 +2,6 @@ package paychmgr
 
 import (
 	"context"
-	"errors"
 
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 
@@ -78,18 +77,15 @@ func (ca *stateAccessor) nextLaneFromState(ctx context.Context, st *paych.State)
 		return 0, nil
 	}
 
-	nextID := int64(0)
+	maxID := int64(0)
-	stopErr := errors.New("stop")
 	if err := laneStates.ForEach(nil, func(i int64) error {
-		if nextID < i {
+		if i > maxID {
-			// We've found a hole. Stop here.
+			maxID = i
-			return stopErr
 		}
-		nextID++
 		return nil
-	}); err != nil && err != stopErr {
+	}); err != nil {
 		return 0, err
 	}
 
-	return uint64(nextID), nil
+	return uint64(maxID + 1), nil
 }

tools/stats/metrics.go

@@ -16,6 +16,7 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
+	"golang.org/x/xerrors"
 
 	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
@@ -104,7 +105,8 @@ func InfluxNewBatch() (client.BatchPoints, error) {
 }
 
 func NewPoint(name string, value interface{}) models.Point {
-	pt, _ := models.NewPoint(name, models.Tags{}, map[string]interface{}{"value": value}, build.Clock.Now())
+	pt, _ := models.NewPoint(name, models.Tags{},
+		map[string]interface{}{"value": value}, build.Clock.Now().UTC())
 	return pt
 }
 
@@ -135,6 +137,7 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti
 	p = NewPoint("chain.basefee", baseFeeFloat)
 	pl.AddPoint(p)
 
+	totalGasLimit := int64(0)
 	for _, blockheader := range tipset.Blocks() {
 		bs, err := blockheader.Serialize()
 		if err != nil {
@@ -146,7 +149,20 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti
 
 		p = NewPoint("chain.blockheader_size", len(bs))
 		pl.AddPoint(p)
+
+		msgs, err := api.ChainGetBlockMessages(ctx, blockheader.Cid())
+		if err != nil {
+			return xerrors.Errorf("ChainGetBlockMessages failed: %w", msgs)
+		}
+		for _, m := range msgs.BlsMessages {
+			totalGasLimit += m.GasLimit
+		}
+		for _, m := range msgs.SecpkMessages {
+			totalGasLimit += m.Message.GasLimit
+		}
 	}
+	p = NewPoint("chain.gas_limit_total", totalGasLimit)
+	pl.AddPoint(p)
 
 	return nil
 }
@@ -286,8 +302,16 @@ func RecordTipsetMessagesPoints(ctx context.Context, api api.FullNode, pl *Point
 
 	msgn := make(map[msgTag][]cid.Cid)
 
+	totalGasUsed := int64(0)
+	for _, r := range recp {
+		totalGasUsed += r.GasUsed
+	}
+	p := NewPoint("chain.gas_used_total", totalGasUsed)
+	pl.AddPoint(p)
+
 	for i, msg := range msgs {
 		// FIXME: use float so this doesn't overflow
+		// FIXME: this doesn't work as time points get overriden
 		p := NewPoint("chain.message_gaspremium", msg.Message.GasPremium.Int64())
 		pl.AddPoint(p)
 		p = NewPoint("chain.message_gasfeecap", msg.Message.GasFeeCap.Int64())