From 551efea1075fe9ec0db634c84c46bf1999879d34 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Wed, 18 Oct 2023 08:15:33 +0000 Subject: [PATCH] Upgrade go-nitro and add support for remote Nitro node (#258) Part of [Support running go-nitro node out of process in ipld-eth-server](https://www.notion.so/Support-running-go-nitro-node-out-of-process-in-ipld-eth-server-f0a064ff933a435999104dda2aba5657) Co-authored-by: Prathamesh Musale Reviewed-on: https://git.vdb.to/cerc-io/ipld-eth-server/pulls/258 Co-authored-by: Prathamesh Musale Co-committed-by: Prathamesh Musale --- Dockerfile | 1 + cmd/common.go | 24 +++++--- cmd/serve.go | 124 ++++++++++++++++++++++++++------------ environments/example.toml | 22 ++++--- go.mod | 6 +- go.sum | 4 +- pkg/serve/config.go | 56 +++++++++++------ 7 files changed, 159 insertions(+), 78 deletions(-) diff --git a/Dockerfile b/Dockerfile index 022627fd..c0e5e9d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,7 @@ USER $USER COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/ipld-eth-server/$CONFIG_FILE config.toml COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/ipld-eth-server/entrypoint.sh . +RUN mkdir -p nitro-data && chown -R 5000:5000 nitro-data # keep binaries immutable COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-server/ipld-eth-server ipld-eth-server diff --git a/cmd/common.go b/cmd/common.go index 7cb69e80..f01d296a 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -38,6 +38,8 @@ func addDatabaseFlags(command *cobra.Command) { func addNitroFlags(command *cobra.Command) { // nitro flags + command.PersistentFlags().Bool("nitro-run-node-in-process", false, "nitro run node in process") + command.PersistentFlags().String("nitro-pk", "", "nitro pk") command.PersistentFlags().String("nitro-chain-pk", "", "nitro chainPk") command.PersistentFlags().String("nitro-chain-url", "", "nitro chainUrl") @@ -47,13 +49,19 @@ func addNitroFlags(command *cobra.Command) { command.PersistentFlags().Bool("nitro-use-durable-store", false, "nitro useDurableStore") command.PersistentFlags().String("nitro-durable-store-folder", "", "nitro durableStoreFolder") + command.PersistentFlags().String("nitro-endpoint", "", "nitro endpoint") + // nitro flag bindings - viper.BindPFlag("nitro.pk", command.PersistentFlags().Lookup("nitro-pk")) - viper.BindPFlag("nitro.chainPk", command.PersistentFlags().Lookup("nitro-chain-pk")) - viper.BindPFlag("nitro.chainUrl", command.PersistentFlags().Lookup("nitro-chain-url")) - viper.BindPFlag("nitro.naAddress", command.PersistentFlags().Lookup("nitro-na-address")) - viper.BindPFlag("nitro.vpaAddress", command.PersistentFlags().Lookup("nitro-vpa-address")) - viper.BindPFlag("nitro.caAddress", command.PersistentFlags().Lookup("nitro-ca-address")) - viper.BindPFlag("nitro.useDurableStore", command.PersistentFlags().Lookup("nitro-use-durable-store")) - viper.BindPFlag("nitro.durableStoreFolder", command.PersistentFlags().Lookup("nitro-durable-store")) + viper.BindPFlag("nitro.runNodeInProcess", command.PersistentFlags().Lookup("nitro-run-node-in-process")) + + viper.BindPFlag("nitro.inProcesssNode.pk", command.PersistentFlags().Lookup("nitro-pk")) + viper.BindPFlag("nitro.inProcesssNode.chainPk", command.PersistentFlags().Lookup("nitro-chain-pk")) + viper.BindPFlag("nitro.inProcesssNode.chainUrl", command.PersistentFlags().Lookup("nitro-chain-url")) + viper.BindPFlag("nitro.inProcesssNode.naAddress", command.PersistentFlags().Lookup("nitro-na-address")) + viper.BindPFlag("nitro.inProcesssNode.vpaAddress", command.PersistentFlags().Lookup("nitro-vpa-address")) + viper.BindPFlag("nitro.inProcesssNode.caAddress", command.PersistentFlags().Lookup("nitro-ca-address")) + viper.BindPFlag("nitro.inProcesssNode.useDurableStore", command.PersistentFlags().Lookup("nitro-use-durable-store")) + viper.BindPFlag("nitro.inProcesssNode.durableStoreFolder", command.PersistentFlags().Lookup("nitro-durable-store")) + + viper.BindPFlag("nitro.remoteNode.nitroEndpoint", command.PersistentFlags().Lookup("nitro-endpoint")) } diff --git a/cmd/serve.go b/cmd/serve.go index 970529b0..f7482c74 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -16,6 +16,7 @@ package cmd import ( + "crypto/tls" "errors" "fmt" "math/big" @@ -35,10 +36,9 @@ import ( "github.com/spf13/viper" "github.com/statechannels/go-nitro/node/engine" "github.com/statechannels/go-nitro/node/engine/chainservice" - "github.com/statechannels/go-nitro/node/engine/store" + nitroStore "github.com/statechannels/go-nitro/node/engine/store" "github.com/statechannels/go-nitro/paymentsmanager" "github.com/statechannels/go-nitro/rpc/transport" - "github.com/tidwall/buntdb" "golang.org/x/exp/slog" "github.com/cerc-io/ipld-eth-server/v5/pkg/graphql" @@ -69,6 +69,8 @@ func serve() { logWithCommand.Infof("ipld-eth-server version: %s", v.VersionWithMeta) wg := new(sync.WaitGroup) + defer wg.Wait() + serverConfig, err := s.NewConfig() if err != nil { logWithCommand.Fatal(err) @@ -90,20 +92,59 @@ func serve() { server.Serve(wg) - nitroNode, err := initializeNitroNode(serverConfig.Nitro) - if err != nil { - logWithCommand.Fatal(err) + var voucherValidator paymentsmanager.VoucherValidator + + // TODO: Refactor into a function / subcommand + nitroConfig := serverConfig.Nitro + if nitroConfig.RunNodeInProcess { + log.Info("Running an in-process Nitro node") + + nitroNode, err := initNitroNode(&nitroConfig.InProcessNode) + if err != nil { + logWithCommand.Fatal(err) + } + + pm, err := paymentsmanager.NewPaymentsManager(nitroNode) + if err != nil { + logWithCommand.Fatal(err) + } + pm.Start(wg) + defer pm.Stop() + + // TODO: Read from config file + rpcPort := 4005 + tlsCertFilepath := "" + tlsKeyFilepath := "" + + var cert *tls.Certificate + if tlsCertFilepath != "" && tlsKeyFilepath != "" { + *cert, err = tls.LoadX509KeyPair(tlsCertFilepath, tlsKeyFilepath) + if err != nil { + logWithCommand.Fatal(err) + } + } + + nitroRpcServer, err := initNitroRpcServer(nitroNode, pm, cert, rpcPort) + if err != nil { + logWithCommand.Fatal(err) + } + defer nitroRpcServer.Close() + + voucherValidator = paymentsmanager.InProcessVoucherValidator{PaymentsManager: pm} + } else { + log.Info("Connecting to a remote Nitro node") + + // TODO: Read from config file + isSecure := false + nitroRpcClient, err := nitroRpc.NewHttpRpcClient(nitroConfig.RemoteNode.NitroEndpoint, isSecure) + if err != nil { + logWithCommand.Fatal(err) + } + defer nitroRpcClient.Close() + + voucherValidator = nitroRpc.RemoteVoucherValidator{Client: nitroRpcClient} } - pm, err := paymentsmanager.NewPaymentsManager(nitroNode) - if err != nil { - logWithCommand.Fatal(err) - } - - pm.Start(wg) - - voucherValidator := paymentsmanager.InProcessVoucherValidator{PaymentsManager: pm} - // TODO: Read from config file queryRates := map[string]*big.Int{ "eth_getBlockByNumber": big.NewInt(50), @@ -112,13 +153,6 @@ func serve() { "eth_getLogs": big.NewInt(50), } - // TODO: Read from config file - rpcPort := 4005 - nitroRpcServer, err := initializeNitroRpcServer(nitroNode, rpcPort) - if err != nil { - logWithCommand.Fatal(err) - } - if err := startServers(server, serverConfig, voucherValidator, queryRates); err != nil { logWithCommand.Fatal(err) } @@ -146,10 +180,6 @@ func serve() { graphQL.Stop() } server.Stop() - pm.Stop() - nitroRpcServer.Close() - - wg.Wait() } // TODO: Absorb voucherValidator and queryRates args into existing ones @@ -391,19 +421,19 @@ func init() { } // https://github.com/cerc-io/go-nitro/blob/release-v0.1.1-ts-port-0.1.7/internal/node/node.go#L17 -func initializeNitroNode(nitroConfig *s.NitroConfig) (*nitroNode.Node, error) { +func initNitroNode(config *s.InProcessNitroNodeConfig) (*nitroNode.Node, error) { // TODO: Read from config file - pkString := nitroConfig.Pk - useDurableStore := nitroConfig.UseDurableStore - durableStoreFolder := nitroConfig.DurableStoreFolder + pkString := config.Pk + useDurableStore := config.UseDurableStore + durableStoreFolder := config.DurableStoreFolder msgPort := 3005 wsMsgPort := 5005 - chainUrl := nitroConfig.ChainUrl + chainUrl := config.ChainUrl chainStartBlock := uint64(0) - chainPk := nitroConfig.ChainPk - naAddress := nitroConfig.NaAddress - vpaAddress := nitroConfig.VpaAddress - caAddress := nitroConfig.CaAddress + chainPk := config.ChainPk + naAddress := config.NaAddress + vpaAddress := config.VpaAddress + caAddress := config.CaAddress chainAuthToken := "" publicIp := "0.0.0.0" @@ -418,14 +448,28 @@ func initializeNitroNode(nitroConfig *s.NitroConfig) (*nitroNode.Node, error) { CaAddress: common.HexToAddress(caAddress), } - ourStore, err := store.NewStore(common.Hex2Bytes(pkString), useDurableStore, durableStoreFolder, buntdb.Config{}) + storeOpts := nitroStore.StoreOpts{ + PkBytes: common.Hex2Bytes(pkString), + UseDurableStore: useDurableStore, + DurableStoreFolder: durableStoreFolder, + } + + bootPeers := []string{} + messageOpts := nitrop2pms.MessageOpts{ + PkBytes: common.Hex2Bytes(pkString), + TcpPort: msgPort, + WsMsgPort: wsMsgPort, + BootPeers: bootPeers, + PublicIp: publicIp, + } + + ourStore, err := nitroStore.NewStore(storeOpts) if err != nil { return nil, err } - bootPeers := []string{} log.Info("Initializing message service...", " tcp port=", msgPort, " web socket port=", wsMsgPort) - messageService := nitrop2pms.NewMessageService(publicIp, msgPort, wsMsgPort, *ourStore.GetAddress(), common.Hex2Bytes(pkString), bootPeers) + messageService := nitrop2pms.NewMessageService(messageOpts) // Compare chainOpts.ChainStartBlock to lastBlockNum seen in store. The larger of the two // gets passed as an argument when creating NewEthChainService @@ -453,17 +497,17 @@ func initializeNitroNode(nitroConfig *s.NitroConfig) (*nitroNode.Node, error) { return &node, nil } -func initializeNitroRpcServer(node *nitroNode.Node, rpcPort int) (*nitroRpc.RpcServer, error) { +func initNitroRpcServer(node *nitroNode.Node, pm paymentsmanager.PaymentsManager, cert *tls.Certificate, rpcPort int) (*nitroRpc.RpcServer, error) { var transport transport.Responder var err error slog.Info("Initializing Nitro HTTP RPC transport...") - transport, err = nitroHttpTransport.NewHttpTransportAsServer(fmt.Sprint(rpcPort)) + transport, err = nitroHttpTransport.NewHttpTransportAsServer(fmt.Sprint(rpcPort), cert) if err != nil { return nil, err } - rpcServer, err := nitroRpc.NewRpcServer(node, transport) + rpcServer, err := nitroRpc.NewRpcServer(node, pm, transport) if err != nil { return nil, err } diff --git a/environments/example.toml b/environments/example.toml index 9e8982f4..232ade55 100644 --- a/environments/example.toml +++ b/environments/example.toml @@ -33,11 +33,17 @@ networkID = "1" # $ETH_NETWORK_ID [nitro] - pk = "" # NITRO_PK - chainPk = "" # NITRO_CHAIN_PK - chainUrl = "" # NITRO_CHAIN_URL - naAddress = "" # NITRO_NA_ADDRESS - vpaAddress = "" # NITRO_VPA_ADDRESS - caAddress = "" # NITRO_CA_ADDRESS - useDurableStore = false # NITRO_USE_DURABLE_STORE - durableStoreFolder = "" # NITRO_DURABLE_STORE_FOLDER + runNodeInProcess = false # NITRO_RUN_NODE_IN_PROCESS + + [nitro.inProcesssNode] + pk = "" # NITRO_PK + chainPk = "" # NITRO_CHAIN_PK + chainUrl = "ws://127.0.0.1:8545" # NITRO_CHAIN_URL + naAddress = "" # NITRO_NA_ADDRESS + vpaAddress = "" # NITRO_VPA_ADDRESS + caAddress = "" # NITRO_CA_ADDRESS + useDurableStore = true # NITRO_USE_DURABLE_STORE + durableStoreFolder = "./data/nitronode" # NITRO_DURABLE_STORE_FOLDER + + [nitro.remoteNode] + nitroEndpoint = "127.0.0.1:4005/api/v1" # NITRO_ENDPOINT diff --git a/go.mod b/go.mod index 58f18af2..80f9da3e 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,6 @@ require ( github.com/spf13/cobra v1.4.0 github.com/spf13/viper v1.11.0 github.com/statechannels/go-nitro v0.1.1 - github.com/tidwall/buntdb v1.2.10 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 gorm.io/driver/postgres v1.3.7 gorm.io/gorm v1.23.5 @@ -235,6 +234,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/thoas/go-funk v0.9.3 // indirect github.com/tidwall/btree v1.6.0 // indirect + github.com/tidwall/buntdb v1.2.10 // indirect github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -302,4 +302,6 @@ replace ( github.com/cerc-io/ipfs-ethdb/v5 => github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha.0.20231013070931-0b1a36562a28 ) -replace github.com/statechannels/go-nitro v0.1.1 => github.com/cerc-io/go-nitro v0.1.1-ts-port-0.1.7 +// TODO: Use release +// https://github.com/cerc-io/go-nitro/tree/ts-interop +replace github.com/statechannels/go-nitro v0.1.1 => github.com/cerc-io/go-nitro v0.1.2-ts-port-0.1.8.0.20231018070130-e66ac486a763 diff --git a/go.sum b/go.sum index c512ac90..4717645d 100644 --- a/go.sum +++ b/go.sum @@ -114,8 +114,8 @@ github.com/ceramicnetwork/go-dag-jose v0.1.0 h1:yJ/HVlfKpnD3LdYP03AHyTvbm3BpPiz2 github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI= github.com/cerc-io/eth-ipfs-state-validator/v5 v5.1.1-alpha.0.20231013075659-56aa03028c43 h1:pkGCN+VWo5Qmu4iDjA7noGrE6wM8VOVeX1Mn6ucYhPg= github.com/cerc-io/eth-ipfs-state-validator/v5 v5.1.1-alpha.0.20231013075659-56aa03028c43/go.mod h1:snThUFpyCrpZhTuz3HibJRLL2XaS+lKNsM3XAE0gB/4= -github.com/cerc-io/go-nitro v0.1.1-ts-port-0.1.7 h1:moqgKEUH9EtnyBgEQH65JrD8Q94abj+r6zGT6BJsU90= -github.com/cerc-io/go-nitro v0.1.1-ts-port-0.1.7/go.mod h1:gkKL37JcSo54ybLTI6VJRmP75bWEu9i1kc9RYmQLp+I= +github.com/cerc-io/go-nitro v0.1.2-ts-port-0.1.8.0.20231018070130-e66ac486a763 h1:RTUK5AquaGw/aMbdgpDuj0IHhqgAg6R2own9lI14ZOQ= +github.com/cerc-io/go-nitro v0.1.2-ts-port-0.1.8.0.20231018070130-e66ac486a763/go.mod h1:YYQvj9es00ZfLTwxZLM1M0ihUrqz8+lU2c10G06My3A= github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha.0.20231013070931-0b1a36562a28 h1:5FXtMuZXTIXjjzzLdqgyzx9pjD22FB5os2vXayRn+BQ= github.com/cerc-io/ipfs-ethdb/v5 v5.0.1-alpha.0.20231013070931-0b1a36562a28/go.mod h1:W1C6qTXGsPcsK1HKUYPsXmBORjO2ekdm+101sJkpdNI= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= diff --git a/pkg/serve/config.go b/pkg/serve/config.go index 45998b1f..e1d0f732 100644 --- a/pkg/serve/config.go +++ b/pkg/serve/config.go @@ -80,6 +80,7 @@ const ( DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS" DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME" + NITRO_RUN_NODE_IN_PROCESS = "NITRO_RUN_NODE_IN_PROCESS" NITRO_PK = "NITRO_PK" NITRO_CHAIN_PK = "NITRO_CHAIN_PK" NITRO_CHAIN_URL = "NITRO_CHAIN_URL" @@ -88,9 +89,10 @@ const ( NITRO_CA_ADDRESS = "NITRO_CA_ADDRESS" NITRO_USE_DURABLE_STORE = "NITRO_USE_DURABLE_STORE" NITRO_DURABLE_STORE_FOLDER = "NITRO_DURABLE_STORE_FOLDER" + NITRO_ENDPOINT = "NITRO_ENDPOINT" ) -type NitroConfig struct { +type InProcessNitroNodeConfig struct { Pk string ChainPk string ChainUrl string @@ -101,6 +103,16 @@ type NitroConfig struct { DurableStoreFolder string } +type RemoteNitroNodeConfig struct { + NitroEndpoint string +} + +type NitroConfig struct { + RunNodeInProcess bool + InProcessNode InProcessNitroNodeConfig + RemoteNode RemoteNitroNodeConfig +} + // Config struct type Config struct { DB *sqlx.DB @@ -305,25 +317,33 @@ func (c *Config) dbInit() { } func (c *Config) loadNitroConfig() { - c.Nitro = &NitroConfig{} + c.Nitro = &NitroConfig{InProcessNode: InProcessNitroNodeConfig{}, RemoteNode: RemoteNitroNodeConfig{}} - viper.BindEnv("nitro.pk", NITRO_PK) - viper.BindEnv("nitro.chainPk", NITRO_CHAIN_PK) - viper.BindEnv("nitro.chainUrl", NITRO_CHAIN_URL) - viper.BindEnv("nitro.naAddress", NITRO_NA_ADDRESS) - viper.BindEnv("nitro.vpaAddress", NITRO_VPA_ADDRESS) - viper.BindEnv("nitro.caAddress", NITRO_CA_ADDRESS) - viper.BindEnv("nitro.useDurableStore", NITRO_USE_DURABLE_STORE) - viper.BindEnv("nitro.durableStoreFolder", NITRO_DURABLE_STORE_FOLDER) + viper.BindEnv("nitro.runNodeInProcess", NITRO_RUN_NODE_IN_PROCESS) - c.Nitro.Pk = viper.GetString("nitro.pk") - c.Nitro.ChainPk = viper.GetString("nitro.chainPk") - c.Nitro.ChainUrl = viper.GetString("nitro.chainUrl") - c.Nitro.NaAddress = viper.GetString("nitro.naAddress") - c.Nitro.VpaAddress = viper.GetString("nitro.vpaAddress") - c.Nitro.CaAddress = viper.GetString("nitro.caAddress") - c.Nitro.UseDurableStore = viper.GetBool("nitro.useDurableStore") - c.Nitro.DurableStoreFolder = viper.GetString("nitro.durableStoreFolder") + viper.BindEnv("nitro.inProcesssNode.pk", NITRO_PK) + viper.BindEnv("nitro.inProcesssNode.chainPk", NITRO_CHAIN_PK) + viper.BindEnv("nitro.inProcesssNode.chainUrl", NITRO_CHAIN_URL) + viper.BindEnv("nitro.inProcesssNode.naAddress", NITRO_NA_ADDRESS) + viper.BindEnv("nitro.inProcesssNode.vpaAddress", NITRO_VPA_ADDRESS) + viper.BindEnv("nitro.inProcesssNode.caAddress", NITRO_CA_ADDRESS) + viper.BindEnv("nitro.inProcesssNode.useDurableStore", NITRO_USE_DURABLE_STORE) + viper.BindEnv("nitro.inProcesssNode.durableStoreFolder", NITRO_DURABLE_STORE_FOLDER) + + viper.BindEnv("nitro.remoteNode.nitroEndpoint", NITRO_ENDPOINT) + + c.Nitro.RunNodeInProcess = viper.GetBool("nitro.runNodeInProcess") + + c.Nitro.InProcessNode.Pk = viper.GetString("nitro.inProcesssNode.pk") + c.Nitro.InProcessNode.ChainPk = viper.GetString("nitro.inProcesssNode.chainPk") + c.Nitro.InProcessNode.ChainUrl = viper.GetString("nitro.inProcesssNode.chainUrl") + c.Nitro.InProcessNode.NaAddress = viper.GetString("nitro.inProcesssNode.naAddress") + c.Nitro.InProcessNode.VpaAddress = viper.GetString("nitro.inProcesssNode.vpaAddress") + c.Nitro.InProcessNode.CaAddress = viper.GetString("nitro.inProcesssNode.caAddress") + c.Nitro.InProcessNode.UseDurableStore = viper.GetBool("nitro.inProcesssNode.useDurableStore") + c.Nitro.InProcessNode.DurableStoreFolder = viper.GetString("nitro.inProcesssNode.durableStoreFolder") + + c.Nitro.RemoteNode.NitroEndpoint = viper.GetString("nitro.remoteNode.nitroEndpoint") } func (c *Config) loadGroupCacheConfig() {