cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

refactor: lite-mode - simplify organization of dep injection

Browse Source
This commit is contained in:
Dirk McCormick 2020-10-08 11:24:31 +02:00
parent b2834baa4b
commit d69e4c7cf2
6 changed files with 105 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
api/test/test.go
View File

@ -35,6 +35,7 @@ var PresealGenesis = -1
const GenesisPreseals = 2
// Options for setting up a mock storage miner
type StorageMiner struct {
Full int
Preseal int
@ -42,12 +43,19 @@ type StorageMiner struct {
type OptionGenerator func([]TestNode) node.Option
// Options for setting up a mock full node
type FullNodeOpts struct {
Lite bool // run node in "lite" mode
Opts OptionGenerator // generate dependency injection options
}
// APIBuilder is a function which is invoked in test suite to provide
// test nodes and networks
//
// fullOpts array defines options for each full node
// storage array defines storage nodes, numbers in the array specify full node
// index the storage node 'belongs' to
type APIBuilder func(t *testing.T, full []OptionGenerator, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode)
type APIBuilder func(t *testing.T, full []FullNodeOpts, storage []StorageMiner, opts ...node.Option) ([]TestNode, []TestStorageNode)
type testSuite struct {
makeNodes APIBuilder
}
@ -65,11 +73,13 @@ func TestApis(t *testing.T, b APIBuilder) {
t.Run("testMiningReal", ts.testMiningReal)
}
func DefaultFullOpts(nFull int) []OptionGenerator {
full := make([]OptionGenerator, nFull)
func DefaultFullOpts(nFull int) []FullNodeOpts {
full := make([]FullNodeOpts, nFull)
for i := range full {
full[i] = func(nodes []TestNode) node.Option {
return node.Options()
full[i] = FullNodeOpts{
Opts: func(nodes []TestNode) node.Option {
return node.Options()
},
}
}
return full

4
chain/sync_test.go
View File

@ -222,7 +222,7 @@ func (tu *syncTestUtil) addSourceNode(gen int) {
var out api.FullNode
stop, err := node.New(tu.ctx,
node.FullAPI(&out),
node.FullAPI(&out, false),
node.Online(),
node.Repo(sourceRepo),
node.MockHost(tu.mn),
@ -254,7 +254,7 @@ func (tu *syncTestUtil) addClientNode() int {
var out api.FullNode
stop, err := node.New(tu.ctx,
node.FullAPI(&out),
node.FullAPI(&out, false),
node.Online(),
node.Repo(repo.NewMemory(nil)),
node.MockHost(tu.mn),

48
cmd/lotus-gateway/endtoend_test.go
View File

@ -8,32 +8,23 @@ import (
"testing"
"time"
init0 "github.com/filecoin-project/specs-actors/actors/builtin/init"
"github.com/filecoin-project/specs-actors/actors/builtin/multisig"
init0 "github.com/filecoin-project/specs-actors/actors/builtin/init"
"github.com/stretchr/testify/require"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/node"
"github.com/filecoin-project/lotus/api/client"
"github.com/filecoin-project/go-jsonrpc"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/stretchr/testify/require"
"github.com/filecoin-project/go-state-types/abi"
builder "github.com/filecoin-project/lotus/node/test"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/client"
"github.com/filecoin-project/lotus/api/test"
"github.com/filecoin-project/lotus/chain/actors/policy"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/node"
builder "github.com/filecoin-project/lotus/node/test"
)
func init() {
@ -179,20 +170,23 @@ func startNodes(ctx context.Context, t *testing.T, blocktime time.Duration) (tes
// Full node
test.OneFull,
// Lite node
func(nodes []test.TestNode) node.Option {
fullNode := nodes[0]
test.FullNodeOpts{
Lite: true,
Opts: func(nodes []test.TestNode) node.Option {
fullNode := nodes[0]
// Create a gateway server in front of the full node
_, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode})
require.NoError(t, err)
// Create a gateway server in front of the full node
_, addr, err := builder.CreateRPCServer(&GatewayAPI{api: fullNode})
require.NoError(t, err)
// Create a gateway client API that connects to the gateway server
var gapi api.GatewayAPI
gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil)
require.NoError(t, err)
// Create a gateway client API that connects to the gateway server
var gapi api.GatewayAPI
gapi, closer, err = client.NewGatewayRPC(ctx, addr, nil)
require.NoError(t, err)
// Override this node with lite-mode options
return node.LiteModeOverrides(gapi)
// Provide the gateway API to dependency injection
return node.Override(new(api.GatewayAPI), gapi)
},
},
)
n, sn := builder.RPCMockSbBuilder(t, opts, test.OneMiner)

21
cmd/lotus/daemon.go
View File

@ -136,6 +136,8 @@ var DaemonCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
isLite := cctx.Bool("lite")
err := runmetrics.Enable(runmetrics.RunMetricOptions{
EnableCPU: true,
EnableMemory: true,
@ -195,8 +197,10 @@ var DaemonCmd = &cli.Command{
return xerrors.Errorf("repo init error: %w", err)
}
if err := paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), 0); err != nil {
return xerrors.Errorf("fetching proof parameters: %w", err)
if !isLite {
if err := paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), 0); err != nil {
return xerrors.Errorf("fetching proof parameters: %w", err)
}
}
var genBytes []byte
@ -243,10 +247,9 @@ var DaemonCmd = &cli.Command{
shutdownChan := make(chan struct{})
// If the daemon is started in "lite mode", replace key APIs
// with a thin client to a gateway server
liteMode := node.Options()
isLite := cctx.Bool("lite")
// If the daemon is started in "lite mode", provide a GatewayAPI
// for RPC calls
liteModeDeps := node.Options()
if isLite {
gapi, closer, err := lcli.GetGatewayAPI(cctx)
if err != nil {
@ -254,13 +257,13 @@ var DaemonCmd = &cli.Command{
}
defer closer()
liteMode = node.LiteModeOverrides(gapi)
liteModeDeps = node.Override(new(api.GatewayAPI), gapi)
}
var api api.FullNode
stop, err := node.New(ctx,
node.FullAPI(&api),
node.FullAPI(&api, isLite),
node.Override(new(dtypes.Bootstrapper), isBootstrapper),
node.Override(new(dtypes.ShutdownChan), shutdownChan),
@ -268,7 +271,7 @@ var DaemonCmd = &cli.Command{
node.Repo(r),
genesis,
liteMode,
liteModeDeps,
node.ApplyIf(func(s *node.Settings) bool { return cctx.IsSet("api") },
node.Override(node.SetApiEndpointKey, func(lr repo.LockedRepo) error {

87
node/builder.go
View File

@ -6,6 +6,13 @@ import (
"os"
"time"
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/exchange"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/chain/wallet"
"github.com/filecoin-project/lotus/node/hello"
logging "github.com/ipfs/go-log"
ci "github.com/libp2p/go-libp2p-core/crypto"
"github.com/libp2p/go-libp2p-core/host"
@ -29,9 +36,7 @@ import (
storage2 "github.com/filecoin-project/specs-storage/storage"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/chain"
"github.com/filecoin-project/lotus/chain/beacon"
"github.com/filecoin-project/lotus/chain/exchange"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/chain/gen/slashfilter"
"github.com/filecoin-project/lotus/chain/market"
@ -39,10 +44,7 @@ import (
"github.com/filecoin-project/lotus/chain/messagesigner"
"github.com/filecoin-project/lotus/chain/metrics"
"github.com/filecoin-project/lotus/chain/stmgr"
"github.com/filecoin-project/lotus/chain/store"
"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/vm"
"github.com/filecoin-project/lotus/chain/wallet"
sectorstorage "github.com/filecoin-project/lotus/extern/sector-storage"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/stores"
@ -56,7 +58,6 @@ import (
"github.com/filecoin-project/lotus/markets/storageadapter"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/config"
"github.com/filecoin-project/lotus/node/hello"
"github.com/filecoin-project/lotus/node/impl"
"github.com/filecoin-project/lotus/node/impl/common"
"github.com/filecoin-project/lotus/node/impl/full"
@ -159,7 +160,7 @@ type Settings struct {
Online bool // Online option applied
Config bool // Config option applied
Lite bool // Start node in "lite" mode
}
func defaults() []Option {
@ -233,6 +234,10 @@ func isType(t repo.RepoType) func(s *Settings) bool {
// Online sets up basic libp2p node
func Online() Option {
isFullOrLiteNode := func(s *Settings) bool { return s.nodeType == repo.FullNode }
isFullNode := func(s *Settings) bool { return s.nodeType == repo.FullNode && !s.Lite }
isLiteNode := func(s *Settings) bool { return s.nodeType == repo.FullNode && s.Lite }
return Options(
// make sure that online is applied before Config.
// This is important because Config overrides some of Online units
@ -246,17 +251,14 @@ func Online() Option {
// common
Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter),
// Full node
ApplyIf(isType(repo.FullNode),
// Full node or lite node
ApplyIf(isFullOrLiteNode,
// TODO: Fix offline mode
Override(new(dtypes.BootstrapPeers), modules.BuiltinBootstrap),
Override(new(dtypes.DrandBootstrap), modules.DrandBootstrap),
Override(new(dtypes.DrandSchedule), modules.BuiltinDrandConfig),
Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages),
Override(new(ffiwrapper.Verifier), ffiwrapper.ProofVerifier),
Override(new(vm.SyscallBuilder), vm.Syscalls),
Override(new(*store.ChainStore), modules.ChainStore),
@ -264,15 +266,8 @@ func Online() Option {
Override(new(*stmgr.StateManager), stmgr.NewStateManagerWithUpgradeSchedule),
Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))),
Override(new(*wallet.Wallet), wallet.NewWallet),
Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))),
Override(new(*messagesigner.MessageSigner), messagesigner.NewMessageSigner),
Override(new(full.ChainModuleAPI), From(new(full.ChainModule))),
Override(new(full.GasModuleAPI), From(new(full.GasModule))),
Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))),
Override(new(full.StateModuleAPI), From(new(full.StateModule))),
Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))),
Override(new(dtypes.ChainGCLocker), blockstore.NewGCLocker),
Override(new(dtypes.ChainGCBlockstore), modules.ChainGCBlockstore),
Override(new(dtypes.ChainBitswap), modules.ChainBitswap),
@ -297,12 +292,6 @@ func Online() Option {
Override(new(dtypes.Graphsync), modules.Graphsync),
Override(new(*dtypes.MpoolLocker), new(dtypes.MpoolLocker)),
Override(RunHelloKey, modules.RunHello),
Override(RunChainExchangeKey, modules.RunChainExchange),
Override(RunPeerMgrKey, modules.RunPeerMgr),
Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks),
Override(new(*discoveryimpl.Local), modules.NewLocalDiscovery),
Override(new(discovery.PeerResolver), modules.RetrievalResolver),
@ -321,8 +310,34 @@ func Online() Option {
Override(SettlePaymentChannelsKey, settler.SettlePaymentChannels),
),
// Lite node
ApplyIf(isLiteNode,
Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))),
Override(new(full.ChainModuleAPI), From(new(api.GatewayAPI))),
Override(new(full.GasModuleAPI), From(new(api.GatewayAPI))),
Override(new(full.MpoolModuleAPI), From(new(api.GatewayAPI))),
Override(new(full.StateModuleAPI), From(new(api.GatewayAPI))),
Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager),
),
// Full node
ApplyIf(isFullNode,
Override(new(messagesigner.MpoolNonceAPI), From(new(*messagepool.MessagePool))),
Override(new(full.ChainModuleAPI), From(new(full.ChainModule))),
Override(new(full.GasModuleAPI), From(new(full.GasModule))),
Override(new(full.MpoolModuleAPI), From(new(full.MpoolModule))),
Override(new(full.StateModuleAPI), From(new(full.StateModule))),
Override(new(stmgr.StateManagerAPI), From(new(*stmgr.StateManager))),
Override(RunHelloKey, modules.RunHello),
Override(RunChainExchangeKey, modules.RunChainExchange),
Override(RunPeerMgrKey, modules.RunPeerMgr),
Override(HandleIncomingMessagesKey, modules.HandleIncomingMessages),
Override(HandleIncomingBlocksKey, modules.HandleIncomingBlocks),
),
// miner
ApplyIf(func(s *Settings) bool { return s.nodeType == repo.StorageMiner },
ApplyIf(isType(repo.StorageMiner),
Override(new(api.Common), From(new(common.CommonAPI))),
Override(new(sectorstorage.StorageAuth), modules.StorageAuth),
@ -402,23 +417,6 @@ func StorageMiner(out *api.StorageMiner) Option {
)
}
func LiteModeOverrides(gapi api.GatewayAPI) Option {
return Options(
Override(new(messagesigner.MpoolNonceAPI), From(new(modules.MpoolNonceAPI))),
Override(new(api.GatewayAPI), gapi),
Override(new(full.ChainModuleAPI), From(new(api.GatewayAPI))),
Override(new(full.GasModuleAPI), From(new(api.GatewayAPI))),
Override(new(full.MpoolModuleAPI), From(new(api.GatewayAPI))),
Override(new(full.StateModuleAPI), From(new(api.GatewayAPI))),
Override(new(stmgr.StateManagerAPI), modules.NewRPCStateManager),
Unset(RunHelloKey),
Unset(RunChainExchangeKey),
Unset(RunPeerMgrKey),
Unset(HandleIncomingBlocksKey),
Unset(HandleIncomingMessagesKey),
)
}
// Config sets up constructors based on the provided Config
func ConfigCommon(cfg *config.Common) Option {
return Options(
@ -533,10 +531,11 @@ func Repo(r repo.Repo) Option {
}
}
func FullAPI(out *api.FullNode) Option {
func FullAPI(out *api.FullNode, lite bool) Option {
return Options(
func(s *Settings) error {
s.nodeType = repo.FullNode
s.Lite = lite
return nil
},
func(s *Settings) error {

24
node/test/builder.go
View File

@ -141,23 +141,23 @@ func CreateTestStorageNode(ctx context.Context, t *testing.T, waddr address.Addr
return test.TestStorageNode{StorageMiner: minerapi, MineOne: mineOne}
}
func Builder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) {
func Builder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) {
return mockBuilderOpts(t, fullOpts, storage, opts, false)
}
func MockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) {
func MockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) {
return mockSbBuilderOpts(t, fullOpts, storage, opts, false)
}
func RPCBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) {
func RPCBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts ...node.Option) ([]test.TestNode, []test.TestStorageNode) {
return mockBuilderOpts(t, fullOpts, storage, opts, true)
}
func RPCMockSbBuilder(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) {
func RPCMockSbBuilder(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner) ([]test.TestNode, []test.TestStorageNode) {
return mockSbBuilderOpts(t, fullOpts, storage, []node.Option{}, true)
}
func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) {
func mockBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, opts []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
@ -237,14 +237,15 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []te
}
stop, err := node.New(ctx,
node.FullAPI(&fulls[i].FullNode),
node.FullAPI(&fulls[i].FullNode, fullOpts[i].Lite),
node.Online(),
node.Repo(repo.NewMemory(nil)),
node.MockHost(mn),
node.Test(),
genesis,
fullOpts[i](fulls),
fullOpts[i].Opts(fulls),
node.Options(opts...),
)
if err != nil {
@ -315,7 +316,7 @@ func mockBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []te
return fulls, storers
}
func mockSbBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []test.StorageMiner, options []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) {
func mockSbBuilderOpts(t *testing.T, fullOpts []test.FullNodeOpts, storage []test.StorageMiner, options []node.Option, rpc bool) ([]test.TestNode, []test.TestStorageNode) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
@ -393,10 +394,8 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []
genesis = node.Override(new(modules.Genesis), modules.LoadGenesis(genbuf.Bytes()))
}
var err error
// TODO: Don't ignore stop
stop, err := node.New(ctx,
node.FullAPI(&fulls[i].FullNode),
node.FullAPI(&fulls[i].FullNode, fullOpts[i].Lite),
node.Online(),
node.Repo(repo.NewMemory(nil)),
node.MockHost(mn),
@ -405,7 +404,8 @@ func mockSbBuilderOpts(t *testing.T, fullOpts []test.OptionGenerator, storage []
node.Override(new(ffiwrapper.Verifier), mock.MockVerifier),
genesis,
fullOpts[i](fulls),
fullOpts[i].Opts(fulls),
node.Options(options...),
)
if err != nil {