cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge remote-tracking branch 'origin/master' into feat/async-restartable-workers

Browse Source
This commit is contained in:
Łukasz Magiera
2020-09-14 19:11:50 +02:00
parent 1ebca8f732 86607452d7
commit 381a6cdfac
74 changed files with 1122 additions and 450 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/builder.go
+3 -3
View File
@@ -227,7 +227,7 @@ func Online() Option {
Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap),
Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap),
Override(new(dtypes.DrandConfig), modules.BuiltinDrandConfig),
Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig),
Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages),
@@ -273,7 +273,7 @@ func Online() Option {
Override(new(modules.ClientDealFunds), modules.NewClientDealFunds),
Override(new(storagemarket.StorageClient), modules.StorageClient),
Override(new(storagemarket.StorageClientNode), storageadapter.NewClientNodeAdapter),
Override(new(beacon.RandomBeacon), modules.RandomBeacon),
Override(new(beacon.Schedule), modules.RandomSchedule),
Override(new(*paychmgr.Store), paychmgr.NewStore),
Override(new(*paychmgr.Manager), paychmgr.NewManager),
@@ -537,6 +537,6 @@ func Test() Option {
return Options(
Unset(RunPeerMgrKey),
Unset(new(*peermgr.PeerMgr)),
Override(new(beacon.RandomBeacon), testing.RandomBeacon),
Override(new(beacon.Schedule), testing.RandomBeacon),
)
}
node/impl/client/client.go
+3 -1
View File
@@ -6,6 +6,8 @@ import (
"io"
"os"
"github.com/filecoin-project/go-state-types/dline"
datatransfer "github.com/filecoin-project/go-data-transfer"
"github.com/filecoin-project/go-state-types/big"
"golang.org/x/xerrors"
@@ -80,7 +82,7 @@ type API struct {
Host host.Host
}
func calcDealExpiration(minDuration uint64, md *miner.DeadlineInfo, startEpoch abi.ChainEpoch) abi.ChainEpoch {
func calcDealExpiration(minDuration uint64, md *dline.Info, startEpoch abi.ChainEpoch) abi.ChainEpoch {
// Make sure we give some time for the miner to seal
minExp := startEpoch + abi.ChainEpoch(minDuration)
node/impl/full/beacon.go
+4 -3
View File
@@ -13,12 +13,13 @@ import (
type BeaconAPI struct {
fx.In
Beacon beacon.RandomBeacon
Beacon beacon.Schedule
}
func (a *BeaconAPI) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) {
rr := a.Beacon.MaxBeaconRoundForEpoch(epoch, types.BeaconEntry{})
e := a.Beacon.Entry(ctx, rr)
b := a.Beacon.BeaconForEpoch(epoch)
rr := b.MaxBeaconRoundForEpoch(epoch)
e := b.Entry(ctx, rr)
select {
case be, ok := <-e:
node/impl/full/chain.go
+2 -2
View File
@@ -495,7 +495,7 @@ func (a *ChainAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Mess
return cm.VMMessage(), nil
}
func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) {
func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, skipoldmsgs bool, tsk types.TipSetKey) (<-chan []byte, error) {
ts, err := a.Chain.GetTipSetFromKey(tsk)
if err != nil {
return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
@@ -508,7 +508,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk t
bw := bufio.NewWriterSize(w, 1<<20)
defer bw.Flush() //nolint:errcheck // it is a write to a pipe
if err := a.Chain.Export(ctx, ts, nroots, bw); err != nil {
if err := a.Chain.Export(ctx, ts, nroots, skipoldmsgs, bw); err != nil {
log.Errorf("chain export call failed: %s", err)
return
}
node/impl/full/gas.go
+2 -18
View File
@@ -204,30 +204,14 @@ func (a *GasAPI) GasEstimateMessageGas(ctx context.Context, msg *types.Message,
}
if msg.GasFeeCap == types.EmptyInt || types.BigCmp(msg.GasFeeCap, types.NewInt(0)) == 0 {
feeCap, err := a.GasEstimateFeeCap(ctx, msg, 10, types.EmptyTSK)
feeCap, err := a.GasEstimateFeeCap(ctx, msg, 20, types.EmptyTSK)
if err != nil {
return nil, xerrors.Errorf("estimating fee cap: %w", err)
}
msg.GasFeeCap = feeCap
}
capGasFee(msg, spec.Get().MaxFee)
messagepool.CapGasFee(msg, spec.Get().MaxFee)
return msg, nil
}
func capGasFee(msg *types.Message, maxFee abi.TokenAmount) {
if maxFee.Equals(big.Zero()) {
maxFee = types.NewInt(build.FilecoinPrecision / 10)
}
gl := types.NewInt(uint64(msg.GasLimit))
totalFee := types.BigMul(msg.GasFeeCap, gl)
if totalFee.LessThanEqual(maxFee) {
return
}
msg.GasFeeCap = big.Div(maxFee, gl)
msg.GasPremium = big.Min(msg.GasFeeCap, msg.GasPremium) // cap premium at FeeCap
}
node/impl/full/state.go
+4 -2
View File
@@ -7,6 +7,8 @@ import (
"fmt"
"strconv"
"github.com/filecoin-project/go-state-types/dline"
cid "github.com/ipfs/go-cid"
cbor "github.com/ipfs/go-ipld-cbor"
cbg "github.com/whyrusleeping/cbor-gen"
@@ -53,7 +55,7 @@ type StateAPI struct {
ProofVerifier ffiwrapper.Verifier
StateManager *stmgr.StateManager
Chain *store.ChainStore
Beacon beacon.RandomBeacon
Beacon beacon.Schedule
}
func (a *StateAPI) StateNetworkName(ctx context.Context) (dtypes.NetworkName, error) {
@@ -145,7 +147,7 @@ func (a *StateAPI) StateMinerPartitions(ctx context.Context, m address.Address,
}))))))
}
func (a *StateAPI) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*miner.DeadlineInfo, error) {
func (a *StateAPI) StateMinerProvingDeadline(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*dline.Info, error) {
ts, err := a.Chain.GetTipSetFromKey(tsk)
if err != nil {
return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err)
node/modules/chain.go
+1 -1
View File
@@ -163,7 +163,7 @@ func NetworkName(mctx helpers.MetricsCtx, lc fx.Lifecycle, cs *store.ChainStore,
return netName, err
}
func NewSyncer(lc fx.Lifecycle, ds dtypes.MetadataDS, sm *stmgr.StateManager, exchange exchange.Client, h host.Host, beacon beacon.RandomBeacon, verifier ffiwrapper.Verifier) (*chain.Syncer, error) {
func NewSyncer(lc fx.Lifecycle, ds dtypes.MetadataDS, sm *stmgr.StateManager, exchange exchange.Client, h host.Host, beacon beacon.Schedule, verifier ffiwrapper.Verifier) (*chain.Syncer, error) {
syncer, err := chain.NewSyncer(ds, sm, exchange, h.ConnManager(), h.ID(), beacon, verifier)
if err != nil {
return nil, err
node/modules/core.go
+11 -6
View File
@@ -10,6 +10,7 @@ import (
"github.com/gbrlsnchs/jwt/v3"
logging "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p-core/peer"
"github.com/libp2p/go-libp2p-core/peerstore"
record "github.com/libp2p/go-libp2p-record"
"golang.org/x/xerrors"
@@ -93,14 +94,18 @@ func BuiltinBootstrap() (dtypes.BootstrapPeers, error) {
return build.BuiltinBootstrap()
}
func DrandBootstrap(d dtypes.DrandConfig) (dtypes.DrandBootstrap, error) {
func DrandBootstrap(ds dtypes.DrandSchedule) (dtypes.DrandBootstrap, error) {
// TODO: retry resolving, don't fail if at least one resolve succeeds
addrs, err := addrutil.ParseAddresses(context.TODO(), d.Relays)
if err != nil {
log.Errorf("reoslving drand relays addresses: %+v", err)
return nil, nil
res := []peer.AddrInfo{}
for _, d := range ds {
addrs, err := addrutil.ParseAddresses(context.TODO(), d.Config.Relays)
if err != nil {
log.Errorf("reoslving drand relays addresses: %+v", err)
return res, nil
}
res = append(res, addrs...)
}
return addrs, nil
return res, nil
}
func SetupJournal(lr repo.LockedRepo) error {
node/modules/dtypes/beacon.go
+9
View File
@@ -1,5 +1,14 @@
package dtypes
import "github.com/filecoin-project/go-state-types/abi"
type DrandSchedule []DrandPoint
type DrandPoint struct {
Start abi.ChainEpoch
Config DrandConfig
}
type DrandConfig struct {
Servers []string
Relays []string
node/modules/lp2p/pubsub.go
+122 -114
View File
@@ -49,7 +49,7 @@ type GossipIn struct {
Db dtypes.DrandBootstrap
Cfg *config.Pubsub
Sk *dtypes.ScoreKeeper
Dr dtypes.DrandConfig
Dr dtypes.DrandSchedule
}
func getDrandTopic(chainInfoJSON string) (string, error) {
@@ -74,9 +74,126 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
}
isBootstrapNode := in.Cfg.Bootstrapper
drandTopic, err := getDrandTopic(in.Dr.ChainInfoJSON)
if err != nil {
return nil, err
drandTopicParams := &pubsub.TopicScoreParams{
// expected 2 beaconsn/min
TopicWeight: 0.5, // 5x block topic; max cap is 62.5
// 1 tick per second, maxes at 1 after 1 hour
TimeInMeshWeight: 0.00027, // ~1/3600
TimeInMeshQuantum: time.Second,
TimeInMeshCap: 1,
// deliveries decay after 1 hour, cap at 25 beacons
FirstMessageDeliveriesWeight: 5, // max value is 125
FirstMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour),
FirstMessageDeliveriesCap: 25, // the maximum expected in an hour is ~26, including the decay
// Mesh Delivery Failure is currently turned off for beacons
// This is on purpose as
// - the traffic is very low for meaningful distribution of incoming edges.
// - the reaction time needs to be very slow -- in the order of 10 min at least
// so we might as well let opportunistic grafting repair the mesh on its own
// pace.
// - the network is too small, so large asymmetries can be expected between mesh
// edges.
// We should revisit this once the network grows.
// invalid messages decay after 1 hour
InvalidMessageDeliveriesWeight: -1000,
InvalidMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour),
}
topicParams := map[string]*pubsub.TopicScoreParams{
build.BlocksTopic(in.Nn): {
// expected 10 blocks/min
TopicWeight: 0.1, // max cap is 50, max mesh penalty is -10, single invalid message is -100
// 1 tick per second, maxes at 1 after 1 hour
TimeInMeshWeight: 0.00027, // ~1/3600
TimeInMeshQuantum: time.Second,
TimeInMeshCap: 1,
// deliveries decay after 1 hour, cap at 100 blocks
FirstMessageDeliveriesWeight: 5, // max value is 500
FirstMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour),
FirstMessageDeliveriesCap: 100, // 100 blocks in an hour
// Mesh Delivery Failure is currently turned off for blocks
// This is on purpose as
// - the traffic is very low for meaningful distribution of incoming edges.
// - the reaction time needs to be very slow -- in the order of 10 min at least
// so we might as well let opportunistic grafting repair the mesh on its own
// pace.
// - the network is too small, so large asymmetries can be expected between mesh
// edges.
// We should revisit this once the network grows.
//
// // tracks deliveries in the last minute
// // penalty activates at 1 minute and expects ~0.4 blocks
// MeshMessageDeliveriesWeight: -576, // max penalty is -100
// MeshMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Minute),
// MeshMessageDeliveriesCap: 10, // 10 blocks in a minute
// MeshMessageDeliveriesThreshold: 0.41666, // 10/12/2 blocks/min
// MeshMessageDeliveriesWindow: 10 * time.Millisecond,
// MeshMessageDeliveriesActivation: time.Minute,
//
// // decays after 15 min
// MeshFailurePenaltyWeight: -576,
// MeshFailurePenaltyDecay: pubsub.ScoreParameterDecay(15 * time.Minute),
// invalid messages decay after 1 hour
InvalidMessageDeliveriesWeight: -1000,
InvalidMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour),
},
build.MessagesTopic(in.Nn): {
// expected > 1 tx/second
TopicWeight: 0.1, // max cap is 5, single invalid message is -100
// 1 tick per second, maxes at 1 hour
TimeInMeshWeight: 0.0002778, // ~1/3600
TimeInMeshQuantum: time.Second,
TimeInMeshCap: 1,
// deliveries decay after 10min, cap at 100 tx
FirstMessageDeliveriesWeight: 0.5, // max value is 50
FirstMessageDeliveriesDecay: pubsub.ScoreParameterDecay(10 * time.Minute),
FirstMessageDeliveriesCap: 100, // 100 messages in 10 minutes
// Mesh Delivery Failure is currently turned off for messages
// This is on purpose as the network is still too small, which results in
// asymmetries and potential unmeshing from negative scores.
// // tracks deliveries in the last minute
// // penalty activates at 1 min and expects 2.5 txs
// MeshMessageDeliveriesWeight: -16, // max penalty is -100
// MeshMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Minute),
// MeshMessageDeliveriesCap: 100, // 100 txs in a minute
// MeshMessageDeliveriesThreshold: 2.5, // 60/12/2 txs/minute
// MeshMessageDeliveriesWindow: 10 * time.Millisecond,
// MeshMessageDeliveriesActivation: time.Minute,
// // decays after 5min
// MeshFailurePenaltyWeight: -16,
// MeshFailurePenaltyDecay: pubsub.ScoreParameterDecay(5 * time.Minute),
// invalid messages decay after 1 hour
InvalidMessageDeliveriesWeight: -1000,
InvalidMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour),
},
}
pgTopicWeights := map[string]float64{
build.BlocksTopic(in.Nn): 10,
build.MessagesTopic(in.Nn): 1,
}
for _, d := range in.Dr {
topic, err := getDrandTopic(d.Config.ChainInfoJSON)
if err != nil {
return nil, err
}
topicParams[topic] = drandTopicParams
pgTopicWeights[topic] = 5
}
options := []pubsub.Option{
@@ -124,111 +241,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
RetainScore: 6 * time.Hour,
// topic parameters
Topics: map[string]*pubsub.TopicScoreParams{
drandTopic: {
// expected 2 beaconsn/min
TopicWeight: 0.5, // 5x block topic; max cap is 62.5
// 1 tick per second, maxes at 1 after 1 hour
TimeInMeshWeight: 0.00027, // ~1/3600
TimeInMeshQuantum: time.Second,
TimeInMeshCap: 1,
// deliveries decay after 1 hour, cap at 25 beacons
FirstMessageDeliveriesWeight: 5, // max value is 125
FirstMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour),
FirstMessageDeliveriesCap: 25, // the maximum expected in an hour is ~26, including the decay
// Mesh Delivery Failure is currently turned off for beacons
// This is on purpose as
// - the traffic is very low for meaningful distribution of incoming edges.
// - the reaction time needs to be very slow -- in the order of 10 min at least
// so we might as well let opportunistic grafting repair the mesh on its own
// pace.
// - the network is too small, so large asymmetries can be expected between mesh
// edges.
// We should revisit this once the network grows.
// invalid messages decay after 1 hour
InvalidMessageDeliveriesWeight: -1000,
InvalidMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour),
},
build.BlocksTopic(in.Nn): {
// expected 10 blocks/min
TopicWeight: 0.1, // max cap is 50, max mesh penalty is -10, single invalid message is -100
// 1 tick per second, maxes at 1 after 1 hour
TimeInMeshWeight: 0.00027, // ~1/3600
TimeInMeshQuantum: time.Second,
TimeInMeshCap: 1,
// deliveries decay after 1 hour, cap at 100 blocks
FirstMessageDeliveriesWeight: 5, // max value is 500
FirstMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour),
FirstMessageDeliveriesCap: 100, // 100 blocks in an hour
// Mesh Delivery Failure is currently turned off for blocks
// This is on purpose as
// - the traffic is very low for meaningful distribution of incoming edges.
// - the reaction time needs to be very slow -- in the order of 10 min at least
// so we might as well let opportunistic grafting repair the mesh on its own
// pace.
// - the network is too small, so large asymmetries can be expected between mesh
// edges.
// We should revisit this once the network grows.
//
// // tracks deliveries in the last minute
// // penalty activates at 1 minute and expects ~0.4 blocks
// MeshMessageDeliveriesWeight: -576, // max penalty is -100
// MeshMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Minute),
// MeshMessageDeliveriesCap: 10, // 10 blocks in a minute
// MeshMessageDeliveriesThreshold: 0.41666, // 10/12/2 blocks/min
// MeshMessageDeliveriesWindow: 10 * time.Millisecond,
// MeshMessageDeliveriesActivation: time.Minute,
//
// // decays after 15 min
// MeshFailurePenaltyWeight: -576,
// MeshFailurePenaltyDecay: pubsub.ScoreParameterDecay(15 * time.Minute),
// invalid messages decay after 1 hour
InvalidMessageDeliveriesWeight: -1000,
InvalidMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour),
},
build.MessagesTopic(in.Nn): {
// expected > 1 tx/second
TopicWeight: 0.1, // max cap is 5, single invalid message is -100
// 1 tick per second, maxes at 1 hour
TimeInMeshWeight: 0.0002778, // ~1/3600
TimeInMeshQuantum: time.Second,
TimeInMeshCap: 1,
// deliveries decay after 10min, cap at 100 tx
FirstMessageDeliveriesWeight: 0.5, // max value is 50
FirstMessageDeliveriesDecay: pubsub.ScoreParameterDecay(10 * time.Minute),
FirstMessageDeliveriesCap: 100, // 100 messages in 10 minutes
// Mesh Delivery Failure is currently turned off for messages
// This is on purpose as the network is still too small, which results in
// asymmetries and potential unmeshing from negative scores.
// // tracks deliveries in the last minute
// // penalty activates at 1 min and expects 2.5 txs
// MeshMessageDeliveriesWeight: -16, // max penalty is -100
// MeshMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Minute),
// MeshMessageDeliveriesCap: 100, // 100 txs in a minute
// MeshMessageDeliveriesThreshold: 2.5, // 60/12/2 txs/minute
// MeshMessageDeliveriesWindow: 10 * time.Millisecond,
// MeshMessageDeliveriesActivation: time.Minute,
// // decays after 5min
// MeshFailurePenaltyWeight: -16,
// MeshFailurePenaltyDecay: pubsub.ScoreParameterDecay(5 * time.Minute),
// invalid messages decay after 1 hour
InvalidMessageDeliveriesWeight: -1000,
InvalidMessageDeliveriesDecay: pubsub.ScoreParameterDecay(time.Hour),
},
},
Topics: topicParams,
},
&pubsub.PeerScoreThresholds{
GossipThreshold: -500,
@@ -278,11 +291,6 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
}
// validation queue RED
pgTopicWeights := map[string]float64{
drandTopic: 5,
build.BlocksTopic(in.Nn): 10,
build.MessagesTopic(in.Nn): 1,
}
var pgParams *pubsub.PeerGaterParams
if isBootstrapNode {
node/modules/services.go
+14 -6
View File
@@ -126,19 +126,27 @@ type RandomBeaconParams struct {
PubSub *pubsub.PubSub `optional:"true"`
Cs *store.ChainStore
DrandConfig dtypes.DrandConfig
DrandConfig dtypes.DrandSchedule
}
func BuiltinDrandConfig() dtypes.DrandConfig {
return build.DrandConfig()
func BuiltinDrandConfig() dtypes.DrandSchedule {
return build.DrandConfigSchedule()
}
func RandomBeacon(p RandomBeaconParams, _ dtypes.AfterGenesisSet) (beacon.RandomBeacon, error) {
func RandomSchedule(p RandomBeaconParams, _ dtypes.AfterGenesisSet) (beacon.Schedule, error) {
gen, err := p.Cs.GetGenesis()
if err != nil {
return nil, err
}
//return beacon.NewMockBeacon(build.BlockDelaySecs * time.Second)
return drand.NewDrandBeacon(gen.Timestamp, build.BlockDelaySecs, p.PubSub, p.DrandConfig)
shd := beacon.Schedule{}
for _, dc := range p.DrandConfig {
bc, err := drand.NewDrandBeacon(gen.Timestamp, build.BlockDelaySecs, p.PubSub, dc.Config)
if err != nil {
return nil, xerrors.Errorf("creating drand beacon: %w", err)
}
shd = append(shd, beacon.BeaconPoint{Start: dc.Start, Beacon: bc})
}
return shd, nil
}
node/modules/testing/beacon.go
+5 -2
View File
@@ -7,6 +7,9 @@ import (
"github.com/filecoin-project/lotus/chain/beacon"
)
func RandomBeacon() (beacon.RandomBeacon, error) {
return beacon.NewMockBeacon(time.Duration(build.BlockDelaySecs) * time.Second), nil
func RandomBeacon() (beacon.Schedule, error) {
return beacon.Schedule{
{Start: 0,
Beacon: beacon.NewMockBeacon(time.Duration(build.BlockDelaySecs) * time.Second),
}}, nil
}
node/repo/fsrepo.go
+1 -1
View File
@@ -101,7 +101,7 @@ func (fsr *FsRepo) Init(t RepoType) error {
}
log.Infof("Initializing repo at '%s'", fsr.path)
err = os.Mkdir(fsr.path, 0755) //nolint: gosec
err = os.MkdirAll(fsr.path, 0755) //nolint: gosec
if err != nil && !os.IsExist(err) {
return err
}