cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge branch 'master' into nonsense/cli-show-deals

Browse Source
This commit is contained in:
Anton Evangelatov
2021-03-23 13:58:26 +02:00
parent ac576c75d4 f145d4f46b
commit 22217b7cae
150 changed files with 6518 additions and 1452 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/modules/blockstore.go
+74 -9
View File
@@ -3,19 +3,25 @@ package modules
import (
"context"
"io"
"os"
"path/filepath"
bstore "github.com/ipfs/go-ipfs-blockstore"
"go.uber.org/fx"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/blockstore"
badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
"github.com/filecoin-project/lotus/blockstore/splitstore"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/node/repo"
)
// UniversalBlockstore returns a single universal blockstore that stores both
// chain data and state data.
// chain data and state data. It can be backed by a blockstore directly
// (e.g. Badger), or by a Splitstore.
func UniversalBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.UniversalBlockstore, error) {
bs, err := r.Blockstore(helpers.LifecycleCtx(mctx, lc), repo.UniversalBlockstore)
if err != nil {
@@ -31,17 +37,76 @@ func UniversalBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.Locked
return bs, err
}
// StateBlockstore is a hook to overlay caches for state objects, or in the
// future, to segregate the universal blockstore into different physical state
// and chain stores.
func StateBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (dtypes.StateBlockstore, error) {
func BadgerHotBlockstore(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.HotBlockstore, error) {
path, err := r.SplitstorePath()
if err != nil {
return nil, err
}
path = filepath.Join(path, "hot.badger")
if err := os.MkdirAll(path, 0755); err != nil {
return nil, err
}
opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, r.Readonly())
if err != nil {
return nil, err
}
bs, err := badgerbs.Open(opts)
if err != nil {
return nil, err
}
lc.Append(fx.Hook{
OnStop: func(_ context.Context) error {
return bs.Close()
}})
return bs, nil
}
// ChainBlockstore is a hook to overlay caches for state objects, or in the
// future, to segregate the universal blockstore into different physical state
// and chain stores.
func ChainBlockstore(lc fx.Lifecycle, mctx helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (dtypes.ChainBlockstore, error) {
func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.LockedRepo, ds dtypes.MetadataDS, cold dtypes.UniversalBlockstore, hot dtypes.HotBlockstore) (dtypes.SplitBlockstore, error) {
return func(lc fx.Lifecycle, r repo.LockedRepo, ds dtypes.MetadataDS, cold dtypes.UniversalBlockstore, hot dtypes.HotBlockstore) (dtypes.SplitBlockstore, error) {
path, err := r.SplitstorePath()
if err != nil {
return nil, err
}
cfg := &splitstore.Config{
TrackingStoreType: cfg.Splitstore.TrackingStoreType,
MarkSetType: cfg.Splitstore.MarkSetType,
EnableFullCompaction: cfg.Splitstore.EnableFullCompaction,
EnableGC: cfg.Splitstore.EnableGC,
Archival: cfg.Splitstore.Archival,
}
ss, err := splitstore.Open(path, ds, hot, cold, cfg)
if err != nil {
return nil, err
}
lc.Append(fx.Hook{
OnStop: func(context.Context) error {
return ss.Close()
},
})
return ss, err
}
}
func StateFlatBlockstore(_ fx.Lifecycle, _ helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (dtypes.StateBlockstore, error) {
return bs, nil
}
func StateSplitBlockstore(_ fx.Lifecycle, _ helpers.MetricsCtx, bs dtypes.SplitBlockstore) (dtypes.StateBlockstore, error) {
return bs, nil
}
func ChainFlatBlockstore(_ fx.Lifecycle, _ helpers.MetricsCtx, bs dtypes.UniversalBlockstore) (dtypes.ChainBlockstore, error) {
return bs, nil
}
func ChainSplitBlockstore(_ fx.Lifecycle, _ helpers.MetricsCtx, bs dtypes.SplitBlockstore) (dtypes.ChainBlockstore, error) {
return bs, nil
}
node/modules/chain.go
+14 -1
View File
@@ -14,6 +14,7 @@ import (
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/blockstore/splitstore"
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/beacon"
@@ -72,14 +73,26 @@ func MessagePool(lc fx.Lifecycle, sm *stmgr.StateManager, ps *pubsub.PubSub, ds
return mp, nil
}
func ChainStore(lc fx.Lifecycle, cbs dtypes.ChainBlockstore, sbs dtypes.StateBlockstore, ds dtypes.MetadataDS, syscalls vm.SyscallBuilder, j journal.Journal) *store.ChainStore {
func ChainStore(lc fx.Lifecycle, cbs dtypes.ChainBlockstore, sbs dtypes.StateBlockstore, ds dtypes.MetadataDS, basebs dtypes.BaseBlockstore, syscalls vm.SyscallBuilder, j journal.Journal) *store.ChainStore {
chain := store.NewChainStore(cbs, sbs, ds, syscalls, j)
if err := chain.Load(); err != nil {
log.Warnf("loading chain state from disk: %s", err)
}
var startHook func(context.Context) error
if ss, ok := basebs.(*splitstore.SplitStore); ok {
startHook = func(_ context.Context) error {
err := ss.Start(chain)
if err != nil {
err = xerrors.Errorf("error starting splitstore: %w", err)
}
return err
}
}
lc.Append(fx.Hook{
OnStart: startHook,
OnStop: func(_ context.Context) error {
return chain.Close()
},
node/modules/dtypes/storage.go
+9
View File
@@ -27,6 +27,15 @@ type (
// UniversalBlockstore is the cold blockstore.
UniversalBlockstore blockstore.Blockstore
// HotBlockstore is the Hot blockstore abstraction for the splitstore
HotBlockstore blockstore.Blockstore
// SplitBlockstore is the hot/cold blockstore that sits on top of the ColdBlockstore.
SplitBlockstore blockstore.Blockstore
// BaseBlockstore is something, coz DI
BaseBlockstore blockstore.Blockstore
// ChainBlockstore is a blockstore to store chain data (tipsets, blocks,
// messages). It is physically backed by the BareMonolithBlockstore, but it
// has a cache on top that is specially tuned for chain data access
node/modules/lp2p/pubsub.go
+12 -2
View File
@@ -3,6 +3,7 @@ package lp2p
import (
"context"
"encoding/json"
"net"
"time"
host "github.com/libp2p/go-libp2p-core/host"
@@ -198,6 +199,16 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
drandTopics = append(drandTopics, topic)
}
// IP colocation whitelist
var ipcoloWhitelist []*net.IPNet
for _, cidr := range in.Cfg.IPColocationWhitelist {
_, ipnet, err := net.ParseCIDR(cidr)
if err != nil {
return nil, xerrors.Errorf("error parsing IPColocation subnet %s: %w", cidr, err)
}
ipcoloWhitelist = append(ipcoloWhitelist, ipnet)
}
options := []pubsub.Option{
// Gossipsubv1.1 configuration
pubsub.WithFloodPublish(true),
@@ -228,8 +239,7 @@ func GossipSub(in GossipIn) (service *pubsub.PubSub, err error) {
// This sets the IP colocation threshold to 5 peers before we apply penalties
IPColocationFactorThreshold: 5,
IPColocationFactorWeight: -100,
// TODO we want to whitelist IPv6 /64s that belong to datacenters etc
// IPColocationFactorWhitelist: map[string]struct{}{},
IPColocationFactorWhitelist: ipcoloWhitelist,
// P7: behavioural penalties, decay after 1hr
BehaviourPenaltyThreshold: 6,
node/modules/paych.go
+45
View File
@@ -0,0 +1,45 @@
package modules
import (
"context"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/node/impl/full"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/paychmgr"
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
"go.uber.org/fx"
)
func NewManager(mctx helpers.MetricsCtx, lc fx.Lifecycle, sm stmgr.StateManagerAPI, pchstore *paychmgr.Store, api paychmgr.PaychAPI) *paychmgr.Manager {
ctx := helpers.LifecycleCtx(mctx, lc)
ctx, shutdown := context.WithCancel(ctx)
return paychmgr.NewManager(ctx, shutdown, sm, pchstore, api)
}
func NewPaychStore(ds dtypes.MetadataDS) *paychmgr.Store {
ds = namespace.Wrap(ds, datastore.NewKey("/paych/"))
return paychmgr.NewStore(ds)
}
type PaychAPI struct {
fx.In
full.MpoolAPI
full.StateAPI
}
// HandlePaychManager is called by dependency injection to set up hooks
func HandlePaychManager(lc fx.Lifecycle, pm *paychmgr.Manager) {
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
return pm.Start()
},
OnStop: func(context.Context) error {
return pm.Stop()
},
})
}
node/modules/rpcstatemanager.go
-58
View File
@@ -1,58 +0,0 @@
package modules
import (
"context"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/builtin/paych"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/types"
cbor "github.com/ipfs/go-ipld-cbor"
)
type RPCStateManager struct {
gapi api.GatewayAPI
cstore *cbor.BasicIpldStore
}
func NewRPCStateManager(api api.GatewayAPI) *RPCStateManager {
cstore := cbor.NewCborStore(blockstore.NewAPIBlockstore(api))
return &RPCStateManager{gapi: api, cstore: cstore}
}
func (s *RPCStateManager) GetPaychState(ctx context.Context, addr address.Address, ts *types.TipSet) (*types.Actor, paych.State, error) {
act, err := s.gapi.StateGetActor(ctx, addr, ts.Key())
if err != nil {
return nil, nil, err
}
actState, err := paych.Load(adt.WrapStore(ctx, s.cstore), act)
if err != nil {
return nil, nil, err
}
return act, actState, nil
}
func (s *RPCStateManager) LoadActorTsk(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) {
return s.gapi.StateGetActor(ctx, addr, tsk)
}
func (s *RPCStateManager) LookupID(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) {
return s.gapi.StateLookupID(ctx, addr, ts.Key())
}
func (s *RPCStateManager) ResolveToKeyAddress(ctx context.Context, addr address.Address, ts *types.TipSet) (address.Address, error) {
return s.gapi.StateAccountKey(ctx, addr, ts.Key())
}
func (s *RPCStateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) {
return nil, xerrors.Errorf("RPCStateManager does not implement StateManager.Call")
}
var _ stmgr.StateManagerAPI = (*RPCStateManager)(nil)
node/modules/storage.go
+27 -7
View File
@@ -2,8 +2,10 @@ package modules
import (
"context"
"path/filepath"
"go.uber.org/fx"
"golang.org/x/xerrors"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/lib/backupds"
@@ -28,12 +30,30 @@ func KeyStore(lr repo.LockedRepo) (types.KeyStore, error) {
return lr.KeyStore()
}
func Datastore(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.MetadataDS, error) {
ctx := helpers.LifecycleCtx(mctx, lc)
mds, err := r.Datastore(ctx, "/metadata")
if err != nil {
return nil, err
}
func Datastore(disableLog bool) func(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.MetadataDS, error) {
return func(lc fx.Lifecycle, mctx helpers.MetricsCtx, r repo.LockedRepo) (dtypes.MetadataDS, error) {
ctx := helpers.LifecycleCtx(mctx, lc)
mds, err := r.Datastore(ctx, "/metadata")
if err != nil {
return nil, err
}
return backupds.Wrap(mds), nil
var logdir string
if !disableLog {
logdir = filepath.Join(r.Path(), "kvlog/metadata")
}
bds, err := backupds.Wrap(mds, logdir)
if err != nil {
return nil, xerrors.Errorf("opening backupds: %w", err)
}
lc.Append(fx.Hook{
OnStop: func(_ context.Context) error {
return bds.CloseLog()
},
})
return bds, nil
}
}
node/modules/storageminer.go
+12
View File
@@ -157,6 +157,9 @@ func AddressSelector(addrConf *config.MinerAddressConfig) func() (*storage.Addre
return as, nil
}
as.DisableOwnerFallback = addrConf.DisableOwnerFallback
as.DisableWorkerFallback = addrConf.DisableWorkerFallback
for _, s := range addrConf.PreCommitControl {
addr, err := address.NewFromString(s)
if err != nil {
@@ -175,6 +178,15 @@ func AddressSelector(addrConf *config.MinerAddressConfig) func() (*storage.Addre
as.CommitControl = append(as.CommitControl, addr)
}
for _, s := range addrConf.TerminateControl {
addr, err := address.NewFromString(s)
if err != nil {
return nil, xerrors.Errorf("parsing terminate control address: %w", err)
}
as.TerminateControl = append(as.TerminateControl, addr)
}
return as, nil
}
}