Merge branch 'next' into ntwk-calibration

chain/messagepool/selection.go

@@ -177,16 +177,16 @@ func (mp *MessagePool) selectMessagesOptimal(curTs, ts *types.TipSet, tq float64
 			for i := len(chainDeps) - 1; i >= 0; i-- {
 				curChain := chainDeps[i]
 				curChain.merged = true
-				// adjust the next chain for the parent, which is being merged
-				if next := curChain.next; next != nil && next.effPerf > 0 {
-					next.effPerf += next.parentOffset
-				}
 				result = append(result, curChain.msgs...)
 			}
 
 			chain.merged = true
+			// adjust the effective pefromance for all subsequent chains
 			if next := chain.next; next != nil && next.effPerf > 0 {
 				next.effPerf += next.parentOffset
+				for next = next.next; next != nil && next.effPerf > 0; next = next.next {
+					next.setEffPerf()
+				}
 			}
 			result = append(result, chain.msgs...)
 			gasLimit -= chainGasLimit

chain/messagepool/selection_test.go

@@ -771,16 +771,26 @@ func TestOptimalMessageSelection2(t *testing.T) {
 		t.Fatalf("expected %d messages, but got %d", expectedMsgs, len(msgs))
 	}
 
-	nextNonce := uint64(0)
+	var nFrom1, nFrom2 int
+	var nextNonce1, nextNonce2 uint64
 	for _, m := range msgs {
-		if m.Message.From != a2 {
+		if m.Message.From == a1 {
-			t.Fatal("expected message from a2")
+			if m.Message.Nonce != nextNonce1 {
+				t.Fatalf("expected nonce %d but got %d", nextNonce1, m.Message.Nonce)
+			}
+			nextNonce1++
+			nFrom1++
+		} else {
+			if m.Message.Nonce != nextNonce2 {
+				t.Fatalf("expected nonce %d but got %d", nextNonce2, m.Message.Nonce)
+			}
+			nextNonce2++
+			nFrom2++
+		}
 	}
 
-		if m.Message.Nonce != nextNonce {
+	if nFrom1 > nFrom2 {
-			t.Fatalf("expected nonce %d but got %d", nextNonce, m.Message.Nonce)
+		t.Fatalf("expected more messages from a2 than a1; nFrom1=%d nFrom2=%d", nFrom1, nFrom2)
-		}
-		nextNonce++
 	}
 }
 

node/config/def.go

@@ -154,8 +154,8 @@ func DefaultStorageMiner() *StorageMiner {
 		},
 
 		Fees: MinerFeeConfig{
-			MaxPreCommitGasFee:  types.FIL(types.FromFil(1)),
+			MaxPreCommitGasFee:  types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05
-			MaxCommitGasFee:     types.FIL(types.FromFil(1)),
+			MaxCommitGasFee:     types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))),
 			MaxWindowPoStGasFee: types.FIL(types.FromFil(50)),
 		},
 

node/impl/full/gas.go

@@ -3,6 +3,7 @@ package full
 import (
 	"context"
 	"math"
+	"math/rand"
 	"sort"
 
 	"github.com/filecoin-project/lotus/build"
@@ -26,7 +27,7 @@ type GasAPI struct {
 	Mpool *messagepool.MessagePool
 }
 
-const MinGasPremium = 10e3
+const MinGasPremium = 100e3
 const MaxSpendOnFeeDenom = 100
 
 func (a *GasAPI) GasEstimateFeeCap(ctx context.Context, msg *types.Message, maxqueueblks int64,
@@ -111,6 +112,7 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64,
 	at := build.BlockGasTarget * int64(blocks) / 2
 	prev := big.Zero()
 
+	premium := big.Zero()
 	for _, price := range prices {
 		at -= price.limit
 		if at > 0 {
@@ -122,19 +124,29 @@ func (a *GasAPI) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64,
 			return types.BigAdd(price.price, big.NewInt(1)), nil
 		}
 
-		return types.BigAdd(big.Div(types.BigAdd(price.price, prev), types.NewInt(2)), big.NewInt(1)), nil
+		premium = types.BigAdd(big.Div(types.BigAdd(price.price, prev), types.NewInt(2)), big.NewInt(1))
 	}
 
+	if types.BigCmp(premium, big.Zero()) == 0 {
 		switch nblocksincl {
 		case 1:
-		return types.NewInt(2 * MinGasPremium), nil
+			premium = types.NewInt(2 * MinGasPremium)
 		case 2:
-		return types.NewInt(1.5 * MinGasPremium), nil
+			premium = types.NewInt(1.5 * MinGasPremium)
 		default:
-		return types.NewInt(MinGasPremium), nil
+			premium = types.NewInt(MinGasPremium)
 		}
 	}
 
+	// add some noise to normalize behaviour of message selection
+	const precision = 32
+	// mean 1, stddev 0.005 => 95% within +-1%
+	noise := 1 + rand.NormFloat64()*0.005
+	premium = types.BigMul(premium, types.NewInt(uint64(noise*(1<<precision))))
+	premium = types.BigDiv(premium, types.NewInt(1<<precision))
+	return premium, nil
+}
+
 func (a *GasAPI) GasEstimateGasLimit(ctx context.Context, msgIn *types.Message, _ types.TipSetKey) (int64, error) {
 
 	msg := *msgIn

paychmgr/paych_test.go

@@ -507,6 +507,15 @@ func TestAddVoucherNextLane(t *testing.T) {
 	require.NoError(t, err)
 	require.EqualValues(t, ci.NextLane, 3)
 
+	// Allocate a lane (should be lane 3)
+	lane, err := mgr.AllocateLane(ch)
+	require.NoError(t, err)
+	require.EqualValues(t, lane, 3)
+
+	ci, err = mgr.GetChannelInfo(ch)
+	require.NoError(t, err)
+	require.EqualValues(t, ci.NextLane, 4)
+
 	// Add a voucher in lane 1
 	voucherLane = uint64(1)
 	sv = testCreateVoucher(t, ch, voucherLane, nonce, voucherAmount, fromKeyPrivate)
@@ -515,17 +524,89 @@ func TestAddVoucherNextLane(t *testing.T) {
 
 	ci, err = mgr.GetChannelInfo(ch)
 	require.NoError(t, err)
-	require.EqualValues(t, ci.NextLane, 3)
+	require.EqualValues(t, ci.NextLane, 4)
 
-	// Add a voucher in lane 5
+	// Add a voucher in lane 7
-	voucherLane = uint64(5)
+	voucherLane = uint64(7)
 	sv = testCreateVoucher(t, ch, voucherLane, nonce, voucherAmount, fromKeyPrivate)
 	_, err = mgr.AddVoucher(ctx, ch, sv, nil, minDelta)
 	require.NoError(t, err)
 
 	ci, err = mgr.GetChannelInfo(ch)
 	require.NoError(t, err)
-	require.EqualValues(t, ci.NextLane, 6)
+	require.EqualValues(t, ci.NextLane, 8)
+}
+
+func TestAllocateLane(t *testing.T) {
+	ctx := context.Background()
+
+	// Set up a manager with a single payment channel
+	mgr, ch, _ := testSetupMgrWithChannel(ctx, t)
+
+	// First lane should be 0
+	lane, err := mgr.AllocateLane(ch)
+	require.NoError(t, err)
+	require.EqualValues(t, lane, 0)
+
+	// Next lane should be 1
+	lane, err = mgr.AllocateLane(ch)
+	require.NoError(t, err)
+	require.EqualValues(t, lane, 1)
+}
+
+func TestAllocateLaneWithExistingLaneState(t *testing.T) {
+	ctx := context.Background()
+
+	_, fromKeyPublic := testGenerateKeyPair(t)
+
+	ch := tutils.NewIDAddr(t, 100)
+	from := tutils.NewSECP256K1Addr(t, string(fromKeyPublic))
+	to := tutils.NewSECP256K1Addr(t, "secpTo")
+	fromAcct := tutils.NewActorAddr(t, "fromAct")
+	toAcct := tutils.NewActorAddr(t, "toAct")
+
+	mock := newMockManagerAPI()
+	mock.setAccountState(fromAcct, account.State{Address: from})
+	mock.setAccountState(toAcct, account.State{Address: to})
+
+	store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore()))
+
+	actorBalance := big.NewInt(10)
+	toSend := big.NewInt(1)
+	laneStates := map[uint64]paych.LaneState{
+		2: {
+			Nonce:    1,
+			Redeemed: big.NewInt(4),
+		},
+	}
+
+	act := &types.Actor{
+		Code:    builtin.AccountActorCodeID,
+		Head:    cid.Cid{},
+		Nonce:   0,
+		Balance: actorBalance,
+	}
+
+	lsCid, err := mock.storeLaneStates(laneStates)
+	require.NoError(t, err)
+	mock.setPaychState(ch, act, paych.State{
+		From:            fromAcct,
+		To:              toAcct,
+		ToSend:          toSend,
+		SettlingAt:      abi.ChainEpoch(0),
+		MinSettleHeight: abi.ChainEpoch(0),
+		LaneStates:      lsCid,
+	})
+
+	mgr, err := newManager(store, mock)
+	require.NoError(t, err)
+
+	err = mgr.TrackInboundChannel(ctx, ch)
+	require.NoError(t, err)
+
+	lane, err := mgr.AllocateLane(ch)
+	require.NoError(t, err)
+	require.EqualValues(t, 3, lane)
 }
 
 func TestAddVoucherProof(t *testing.T) {
@@ -575,23 +656,6 @@ func TestAddVoucherProof(t *testing.T) {
 	require.Len(t, ci.Vouchers[0].Proof, 1)
 }
 
-func TestAllocateLane(t *testing.T) {
-	ctx := context.Background()
-
-	// Set up a manager with a single payment channel
-	mgr, ch, _ := testSetupMgrWithChannel(ctx, t)
-
-	// First lane should be 0
-	lane, err := mgr.AllocateLane(ch)
-	require.NoError(t, err)
-	require.EqualValues(t, lane, 0)
-
-	// Next lane should be 1
-	lane, err = mgr.AllocateLane(ch)
-	require.NoError(t, err)
-	require.EqualValues(t, lane, 1)
-}
-
 func TestNextNonceForLane(t *testing.T) {
 	ctx := context.Background()
 

paychmgr/state.go

@@ -2,7 +2,6 @@ package paychmgr
 
 import (
 	"context"
-	"errors"
 
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
 
@@ -78,18 +77,15 @@ func (ca *stateAccessor) nextLaneFromState(ctx context.Context, st *paych.State)
 		return 0, nil
 	}
 
-	nextID := int64(0)
+	maxID := int64(0)
-	stopErr := errors.New("stop")
 	if err := laneStates.ForEach(nil, func(i int64) error {
-		if nextID < i {
+		if i > maxID {
-			// We've found a hole. Stop here.
+			maxID = i
-			return stopErr
 		}
-		nextID++
 		return nil
-	}); err != nil && err != stopErr {
+	}); err != nil {
 		return 0, err
 	}
 
-	return uint64(nextID), nil
+	return uint64(maxID + 1), nil
 }

tools/stats/metrics.go

@@ -16,6 +16,7 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/builtin"
 	"github.com/filecoin-project/specs-actors/actors/builtin/power"
 	"github.com/filecoin-project/specs-actors/actors/util/adt"
+	"golang.org/x/xerrors"
 
 	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
@@ -104,7 +105,8 @@ func InfluxNewBatch() (client.BatchPoints, error) {
 }
 
 func NewPoint(name string, value interface{}) models.Point {
-	pt, _ := models.NewPoint(name, models.Tags{}, map[string]interface{}{"value": value}, build.Clock.Now())
+	pt, _ := models.NewPoint(name, models.Tags{},
+		map[string]interface{}{"value": value}, build.Clock.Now().UTC())
 	return pt
 }
 
@@ -135,6 +137,7 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti
 	p = NewPoint("chain.basefee", baseFeeFloat)
 	pl.AddPoint(p)
 
+	totalGasLimit := int64(0)
 	for _, blockheader := range tipset.Blocks() {
 		bs, err := blockheader.Serialize()
 		if err != nil {
@@ -146,7 +149,20 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti
 
 		p = NewPoint("chain.blockheader_size", len(bs))
 		pl.AddPoint(p)
+
+		msgs, err := api.ChainGetBlockMessages(ctx, blockheader.Cid())
+		if err != nil {
+			return xerrors.Errorf("ChainGetBlockMessages failed: %w", msgs)
 		}
+		for _, m := range msgs.BlsMessages {
+			totalGasLimit += m.GasLimit
+		}
+		for _, m := range msgs.SecpkMessages {
+			totalGasLimit += m.Message.GasLimit
+		}
+	}
+	p = NewPoint("chain.gas_limit_total", totalGasLimit)
+	pl.AddPoint(p)
 
 	return nil
 }
@@ -286,8 +302,16 @@ func RecordTipsetMessagesPoints(ctx context.Context, api api.FullNode, pl *Point
 
 	msgn := make(map[msgTag][]cid.Cid)
 
+	totalGasUsed := int64(0)
+	for _, r := range recp {
+		totalGasUsed += r.GasUsed
+	}
+	p := NewPoint("chain.gas_used_total", totalGasUsed)
+	pl.AddPoint(p)
+
 	for i, msg := range msgs {
 		// FIXME: use float so this doesn't overflow
+		// FIXME: this doesn't work as time points get overriden
 		p := NewPoint("chain.message_gaspremium", msg.Message.GasPremium.Int64())
 		pl.AddPoint(p)
 		p = NewPoint("chain.message_gasfeecap", msg.Message.GasFeeCap.Int64())