From 524ab42674d48351ba75322e8948723c5a2fe0fb Mon Sep 17 00:00:00 2001 From: ramil Date: Wed, 14 Apr 2021 18:53:44 +0300 Subject: [PATCH] normilize CLI options, environment variables merge gap-filler library to "serve" commnad --- README.md | 26 ++++++++ cmd/root.go | 11 +--- cmd/serve.go | 156 ++++++++++++++++++++++++++++++++++++-------- pkg/serve/config.go | 119 ++++++++++++++++++++++++++------- 4 files changed, 250 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 2b98b518..83566560 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,32 @@ The currently supported standard endpoints are: TODO: Add the rest of the standard endpoints and unique endpoints (e.g. getSlice) + +### CLI Options and Environment variables + + +| CLI Option | Environment Variable | Default Value | Comment | +| ----------------------------- | ----------------------------- | ---------------- | ----------------------------------- | +| `database-hostname` | `DATABASE_HOSTNAME` | localhost | IPLD database host | +| `database-port` | `DATABASE_PORT` | 5432 | IPLD database port | +| `database-name` | `DATABASE_NAME` | vulcanize_public | IPLD database name | +| `database-user` | `DATABASE_USER` | | IPLD database user | +| `database-password` | `DATABASE_PASSWORD` | | IPLD database password | +| `eth-server-graphql` | `ETH_SERVER_GRAPHQL` | false | If `true` enable Eth GraphQL Server | +| `eth-server-graphql-path` | `ETH_SERVER_GRAPHQLPATH` | | If `eth-server-graphql` set to true, endpoint url for graphql server (host:port) | +| `eth-server-http` | `ETH_SERVER_HTTP` | true | If `true` enable Eth HTTP JSON-RPC Server | +| `eth-server-http-path` | `ETH_SERVER_HTTP_PATH` | | If `eth-server-http` set to `true`, endpoint url for Eth HTTP JSON-RPC server (host:port) | +| `eth-server-ws` | `ETH_SERVER_WS` | false | If `true` enable Eth WS JSON-RPC Server | +| `eth-server-ws-path` | `ETH_SERVER_WS_PATH` | | If `eth-server-ws` set to `true`, endpoint url for Eth WS JSON-RPC server (host:port) | +| `eth-server-ipc` | `ETH_SERVER_IPC` | false | If `true` enable Eth IPC JSON-RPC Server | +| `eth-server-ipc-path` | `ETH_SERVER_IPC_PATH` | | If `eth-server-ws` set to `true`, path for Eth IPC JSON-RPC server | +| `ipld-server-graphql` | `IPLD_SERVER_GRAPHQL` | false | If `true` enable IPLD GraphQL Server | +| `ipld-server-graphql-path` | `IPLD_SERVER_GRAPHQLPATH` | | If `ipld-server-graphql` set to true, endpoint url for graphql server (host:port) | +| `ipld-postgraphile-path` | `IPLD_POSTGRAPHILEPATH` | | If `ipld-server-graphql` set to true, http url for postgraphile server on top of IPLD db | +| `tracing-http-path` | `TRACING_HTTPPATH` | | If `ipld-server-graphql` set to true, http url for tracing server | +| `tracing-postgraphile-path` | `TRACING.POSTGRAPHILEPATH` | | If `ipld-server-graphql` set to true, http url for postgraphile server on top of tracing db | + + ### Testing `make test` will run the unit tests `make test` setups a clean `vulcanize_testing` db diff --git a/cmd/root.go b/cmd/root.go index c4412111..4e8ac521 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -99,11 +99,7 @@ func init() { viper.AutomaticEnv() rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location") - rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name") - rootCmd.PersistentFlags().Int("database-port", 5432, "database port") - rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname") - rootCmd.PersistentFlags().String("database-user", "", "database user") - rootCmd.PersistentFlags().String("database-password", "", "database password") + rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file") rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)") rootCmd.PersistentFlags().String("log-file", "", "file path for logging") @@ -114,11 +110,6 @@ func init() { rootCmd.PersistentFlags().String("prom-http-addr", "127.0.0.1", "http host for prometheus") rootCmd.PersistentFlags().String("prom-http-port", "8090", "http port for prometheus") - viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) - viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) - viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname")) - viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user")) - viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password")) viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level")) viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file")) diff --git a/cmd/serve.go b/cmd/serve.go index a495c126..e4ac67f5 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -16,6 +16,9 @@ package cmd import ( + "github.com/vulcanize/gap-filler/pkg/mux" + "net/http" + "net/url" "os" "os/signal" "sync" @@ -71,7 +74,7 @@ func serve() { if err := startServers(server, serverConfig); err != nil { logWithCommand.Fatal(err) } - graphQL, err := startGraphQL(server) + graphQL, err := startEthGraphQL(server, serverConfig) if err != nil { logWithCommand.Fatal(err) } @@ -87,27 +90,43 @@ func serve() { } func startServers(server s.Server, settings *s.Config) error { - logWithCommand.Info("starting up IPC server") - _, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs()) - if err != nil { - return err + if settings.IPCEnabled { + logWithCommand.Info("starting up IPC server") + _, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs()) + if err != nil { + return err + } + } else { + logWithCommand.Info("IPC server is disabled") } - logWithCommand.Info("starting up WS server") - _, _, err = srpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb"}, nil, true) - if err != nil { - return err + + if settings.WSEnabled { + logWithCommand.Info("starting up WS server") + _, _, err := srpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb", "net"}, nil, true) + if err != nil { + return err + } + } else { + logWithCommand.Info("WS server is disabled") } - logWithCommand.Info("starting up HTTP server") - _, err = srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"eth", "net"}, nil, []string{"*"}, rpc.HTTPTimeouts{}) - return err + + if settings.HTTPEnabled { + logWithCommand.Info("starting up HTTP server") + _, err := srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"eth", "net"}, nil, []string{"*"}, rpc.HTTPTimeouts{}) + if err != nil { + return err + } + } else { + logWithCommand.Info("HTTP server is disabled") + } + + return nil } -func startGraphQL(server s.Server) (graphQLServer *graphql.Service, err error) { - viper.BindEnv("server.graphql", "SERVER_GRAPHQL") - if viper.GetBool("server.graphql") { - logWithCommand.Info("starting up GraphQL server") - viper.BindEnv("server.graphqlEndpoint", "SERVER_GRAPHQL_ENDPOINT") - endPoint := viper.GetString("server.graphqlEndpoint") +func startEthGraphQL(server s.Server, settings *s.Config) (graphQLServer *graphql.Service, err error) { + if settings.EthGraphqlEnabled { + logWithCommand.Info("starting up ETH GraphQL server") + endPoint := settings.EthGraphqlEndpoint if endPoint != "" { graphQLServer, err = graphql.New(server.Backend(), endPoint, nil, []string{"*"}, rpc.HTTPTimeouts{}) if err != nil { @@ -115,19 +134,76 @@ func startGraphQL(server s.Server) (graphQLServer *graphql.Service, err error) { } err = graphQLServer.Start(nil) } + } else { + logWithCommand.Info("ETH GraphQL server is disabled") } + return } +func startIpldGraphQL(settings *s.Config) error { + if settings.IpldGraphqlEnabled { + logWithCommand.Info("starting up IPLD GraphQL server") + + gqlDefaultAddr, err := url.Parse(settings.IpldPostgraphileEndpoint) + if err != nil { + return err + } + + router, err := mux.NewServeMux(&mux.Options{ + BasePath: "/", + EnableGraphiQL: true, + Postgraphile: mux.PostgraphileOptions{ + Default: gqlDefaultAddr, + TracingAPI: gqlTracingAPIAddr, + }, + RPC: mux.RPCOptions{ + DefaultClients: settings.Client, + TracingClients: tracingClients, + }, + }) + if err != nil { + logWithCommand.Fatal(err) + } + + addr := fmt.Sprintf("%s:%s", viper.GetString("http.host"), viper.GetString("http.port")) + if err := http.ListenAndServe(addr, router); err != nil { + logWithCommand.Fatal(err) + } + } else { + logWithCommand.Info("IPLD GraphQL server is disabled") + } + + return nil +} + func init() { rootCmd.AddCommand(serveCmd) + // database credentials + serveCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name") + serveCmd.PersistentFlags().Int("database-port", 5432, "database port") + serveCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname") + serveCmd.PersistentFlags().String("database-user", "", "database user") + serveCmd.PersistentFlags().String("database-password", "", "database password") + // flags for all config variables - serveCmd.PersistentFlags().Bool("server-graphql", false, "turn on the graphql server") - serveCmd.PersistentFlags().String("server-graphql-endpoint", "", "endpoint url for graphql server") - serveCmd.PersistentFlags().String("server-ws-path", "", "vdb server ws path") - serveCmd.PersistentFlags().String("server-http-path", "", "vdb server http path") - serveCmd.PersistentFlags().String("server-ipc-path", "", "vdb server ipc path") + // eth graphql and json-rpc parameters + serveCmd.PersistentFlags().Bool("eth-server-graphql", false, "turn on the eth graphql server") + serveCmd.PersistentFlags().String("eth-server-graphql-path", "", "endpoint url for eth graphql server (host:port)") + serveCmd.PersistentFlags().Bool("eth-server-http", true, "turn on the eth http json-rpc server") + serveCmd.PersistentFlags().String("eth-server-http-path", "", "endpoint url for eth http json-rpc server (host:port)") + serveCmd.PersistentFlags().Bool("eth-server-ws", false, "turn on the eth websocket json-rpc server") + serveCmd.PersistentFlags().String("eth-server-ws-path", "", "endpoint url for eth websocket json-rpc server (host:port)") + serveCmd.PersistentFlags().Bool("eth-server-ipc", false, "turn on the eth ipc json-rpc server") + serveCmd.PersistentFlags().String("eth-server-ipc-path", "", "path for eth ipc json-rpc server") + + // ipld and tracing graphql parameters + serveCmd.PersistentFlags().Bool("ipld-server-graphql", false, "turn on the ipld graphql server") + serveCmd.PersistentFlags().String("ipld-server-graphql-path", "", "endpoint url for ipld graphql server (host:port)") + serveCmd.PersistentFlags().String("ipld-postgraphile-path", "", "http url to postgraphile on top of ipld database") + serveCmd.PersistentFlags().String("tracing-http-path", "", "http url to tracing service") + serveCmd.PersistentFlags().String("tracing-postgraphile-path", "", "http url to postgraphile on top of tracing db") serveCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node") serveCmd.PersistentFlags().String("eth-node-id", "", "eth node id") @@ -141,11 +217,35 @@ func init() { serveCmd.PersistentFlags().Bool("eth-supports-state-diff", false, "whether or not the proxy ethereum client supports statediffing endpoints") // and their bindings - viper.BindPFlag("server.graphql", serveCmd.PersistentFlags().Lookup("server-graphql")) - viper.BindPFlag("server.graphqlEndpoint", serveCmd.PersistentFlags().Lookup("server-graphql-endpoint")) - viper.BindPFlag("server.wsPath", serveCmd.PersistentFlags().Lookup("server-ws-path")) - viper.BindPFlag("server.httpPath", serveCmd.PersistentFlags().Lookup("server-http-path")) - viper.BindPFlag("server.ipcPath", serveCmd.PersistentFlags().Lookup("server-ipc-path")) + // database + viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name")) + viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port")) + viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname")) + viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user")) + viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password")) + + // eth graphql server + viper.BindPFlag("eth.server.graphql", serveCmd.PersistentFlags().Lookup("eth-server-graphql")) + viper.BindPFlag("eth.server.graphqlPath", serveCmd.PersistentFlags().Lookup("eth-server-graphql-path")) + + // eth http json-rpc server + viper.BindPFlag("eth.server.http", serveCmd.PersistentFlags().Lookup("eth-server-http")) + viper.BindPFlag("eth.server.httpPath", serveCmd.PersistentFlags().Lookup("eth-server-http-path")) + + // eth websocket json-rpc server + viper.BindPFlag("eth.server.ws", serveCmd.PersistentFlags().Lookup("eth-server-ws")) + viper.BindPFlag("eth.server.wsPath", serveCmd.PersistentFlags().Lookup("eth-server-ws-path")) + + // eth ipc json-rpc server + viper.BindPFlag("eth.server.ipc", serveCmd.PersistentFlags().Lookup("eth-server-ipc")) + viper.BindPFlag("eth.server.ipcPath", serveCmd.PersistentFlags().Lookup("eth-server-ipc-path")) + + // ipld and tracing graphql parameters + viper.BindPFlag("ipld.server.graphql", serveCmd.PersistentFlags().Lookup("ipld-server-graphql")) + viper.BindPFlag("ipld.server.graphqlPath", serveCmd.PersistentFlags().Lookup("ipld-server-graphql-path")) + viper.BindPFlag("ipld.postgraphilePath", serveCmd.PersistentFlags().Lookup("ipld-postgraphile-path")) + viper.BindPFlag("tracing.httpPath", serveCmd.PersistentFlags().Lookup("tracing-http-path")) + viper.BindPFlag("tracing.postgraphilePath", serveCmd.PersistentFlags().Lookup("tracing-postgraphile-path")) viper.BindPFlag("ethereum.httpPath", serveCmd.PersistentFlags().Lookup("eth-http-path")) viper.BindPFlag("ethereum.nodeID", serveCmd.PersistentFlags().Lookup("eth-node-id")) diff --git a/pkg/serve/config.go b/pkg/serve/config.go index 4578fddf..e69b4ade 100644 --- a/pkg/serve/config.go +++ b/pkg/serve/config.go @@ -17,6 +17,7 @@ package serve import ( + "errors" "fmt" "math/big" "os" @@ -54,11 +55,27 @@ const ( // Config struct type Config struct { - DB *postgres.DB - DBConfig postgres.Config - WSEndpoint string - HTTPEndpoint string - IPCEndpoint string + DB *postgres.DB + DBConfig postgres.Config + + WSEnabled bool + WSEndpoint string + + HTTPEnabled bool + HTTPEndpoint string + + IPCEnabled bool + IPCEndpoint string + + EthGraphqlEnabled bool + EthGraphqlEndpoint string + + IpldGraphqlEnabled bool + IpldGraphqlEndpoint string + IpldPostgraphileEndpoint string + TracingHttpEndpoint string + TracingPostgraphileEndpoint string + ChainConfig *params.ChainConfig DefaultSender *common.Address RPCGasCap *big.Int @@ -71,9 +88,6 @@ type Config struct { func NewConfig() (*Config, error) { c := new(Config) - viper.BindEnv("server.wsPath", SERVER_WS_PATH) - viper.BindEnv("server.ipcPath", SERVER_IPC_PATH) - viper.BindEnv("server.httpPath", SERVER_HTTP_PATH) viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH) viper.BindEnv("ethereum.defaultSender", ETH_DEFAULT_SENDER_ADDR) viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP) @@ -90,25 +104,82 @@ func NewConfig() (*Config, error) { c.Client = cli c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff") - wsPath := viper.GetString("server.wsPath") - if wsPath == "" { - wsPath = "127.0.0.1:8080" - } - c.WSEndpoint = wsPath - ipcPath := viper.GetString("server.ipcPath") - if ipcPath == "" { - home, err := os.UserHomeDir() - if err != nil { - return nil, err + // websocket server + wsEnabled := viper.GetBool("eth.server.ws") + if wsEnabled { + wsPath := viper.GetString("eth.server.wsPath") + if wsPath == "" { + wsPath = "127.0.0.1:8080" } - ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc") + c.WSEndpoint = wsPath } - c.IPCEndpoint = ipcPath - httpPath := viper.GetString("server.httpPath") - if httpPath == "" { - httpPath = "127.0.0.1:8081" + c.WSEnabled = wsEnabled + + // ipc server + ipcEnabled := viper.GetBool("eth.server.ipc") + if ipcEnabled { + ipcPath := viper.GetString("eth.server.ipcPath") + if ipcPath == "" { + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc") + } + c.IPCEndpoint = ipcPath } - c.HTTPEndpoint = httpPath + c.IPCEnabled = ipcEnabled + + // http server + httpEnabled := viper.GetBool("eth.server.http") + if httpEnabled { + httpPath := viper.GetString("eth.server.httpPath") + if httpPath == "" { + httpPath = "127.0.0.1:8081" + } + c.HTTPEndpoint = httpPath + } + c.HTTPEnabled = httpEnabled + + // eth graphql endpoint + ethGraphqlEnabled := viper.GetBool("eth.server.graphql") + if ethGraphqlEnabled { + ethGraphqlPath := viper.GetString("eth.server.graphqlPath") + if ethGraphqlPath == "" { + ethGraphqlPath = "127.0.0.1:8082" + } + c.EthGraphqlEndpoint = ethGraphqlPath + } + c.EthGraphqlEnabled = ethGraphqlEnabled + + // ipld graphql endpoint + ipldGraphqlEnabled := viper.GetBool("ipld.server.graphql") + if ipldGraphqlEnabled { + ipldGraphqlPath := viper.GetString("ipld.server.graphqlPath") + if ipldGraphqlPath == "" { + ipldGraphqlPath = "127.0.0.1:8083" + } + c.IpldGraphqlEndpoint = ipldGraphqlPath + + ipldPostgraphilePath := viper.GetString("ipld.postgraphilePath") + if ipldPostgraphilePath == "" { + return nil, errors.New("ipld-postgraphile-path parameter is empty") + } + c.IpldPostgraphileEndpoint = ipldPostgraphilePath + + tracingHttpEndpoint := viper.GetString("tracing.httpPath") + tracingPostgraphilePath := viper.GetString("tracing.postgraphilePath") + + // these two parameters either can be both empty or both set + if (tracingHttpEndpoint == "" && tracingPostgraphilePath != "") || (tracingHttpEndpoint != "" && tracingPostgraphilePath == "") { + return nil, errors.New("tracing.httpPath and tracing.postgraphilePath parameters either can be both empty or both set") + } + + c.TracingHttpEndpoint = tracingHttpEndpoint + c.TracingPostgraphileEndpoint = tracingPostgraphilePath + } + c.IpldGraphqlEnabled = ipldGraphqlEnabled + overrideDBConnConfig(&c.DBConfig) serveDB := utils.LoadPostgres(c.DBConfig, nodeInfo, false) prom.RegisterDBCollector(c.DBConfig.Name, serveDB.DB)