normilize CLI options, environment variables

merge gap-filler library to "serve" commnad

README.md

@@ -121,6 +121,32 @@ The currently supported standard endpoints are:
 
 TODO: Add the rest of the standard endpoints and unique endpoints (e.g. getSlice)
 
+
+### CLI Options and Environment variables
+
+
+| CLI Option                    | Environment Variable          | Default Value    | Comment                             |
+| ----------------------------- | ----------------------------- | ---------------- | ----------------------------------- |
+| `database-hostname`           | `DATABASE_HOSTNAME`           | localhost        | IPLD database host                  |
+| `database-port`               | `DATABASE_PORT`               | 5432             | IPLD database port                  |
+| `database-name`               | `DATABASE_NAME`               | vulcanize_public | IPLD database name                  |
+| `database-user`               | `DATABASE_USER`               |                  | IPLD database user                  |
+| `database-password`           | `DATABASE_PASSWORD`           |                  | IPLD database password              |
+| `eth-server-graphql`          | `ETH_SERVER_GRAPHQL`          | false            | If `true` enable Eth GraphQL Server |
+| `eth-server-graphql-path` | `ETH_SERVER_GRAPHQLPATH` |                  | If `eth-server-graphql` set to true, endpoint url for graphql server (host:port)                               |
+| `eth-server-http`          | `ETH_SERVER_HTTP`          | true            | If `true` enable Eth HTTP JSON-RPC Server |
+| `eth-server-http-path`          | `ETH_SERVER_HTTP_PATH`          |             | If `eth-server-http` set to `true`, endpoint url for Eth HTTP JSON-RPC server (host:port)  |
+| `eth-server-ws`          | `ETH_SERVER_WS`          | false            | If `true` enable Eth WS JSON-RPC Server |
+| `eth-server-ws-path`          | `ETH_SERVER_WS_PATH`          |             | If `eth-server-ws` set to `true`, endpoint url for Eth WS JSON-RPC server (host:port)  |
+| `eth-server-ipc`          | `ETH_SERVER_IPC`          | false            | If `true` enable Eth IPC JSON-RPC Server |
+| `eth-server-ipc-path`          | `ETH_SERVER_IPC_PATH`          |             | If `eth-server-ws` set to `true`, path for Eth IPC JSON-RPC server |
+| `ipld-server-graphql`          | `IPLD_SERVER_GRAPHQL`          | false            | If `true` enable IPLD GraphQL Server |
+| `ipld-server-graphql-path` | `IPLD_SERVER_GRAPHQLPATH` |                  | If `ipld-server-graphql` set to true, endpoint url for graphql server (host:port)                               |
+| `ipld-postgraphile-path` | `IPLD_POSTGRAPHILEPATH` |                  | If `ipld-server-graphql` set to true, http url for postgraphile server on top of IPLD db                        |
+| `tracing-http-path` | `TRACING_HTTPPATH` |                  | If `ipld-server-graphql` set to true, http url for tracing server                        |
+| `tracing-postgraphile-path` | `TRACING.POSTGRAPHILEPATH` |                  | If `ipld-server-graphql` set to true, http url for postgraphile server on top of tracing db                        |
+
+
 ### Testing
 `make test` will run the unit tests  
 `make test` setups a clean `vulcanize_testing` db

cmd/root.go

@@ -99,11 +99,7 @@ func init() {
 	viper.AutomaticEnv()
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
-	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
+
-	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
-	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
-	rootCmd.PersistentFlags().String("database-user", "", "database user")
-	rootCmd.PersistentFlags().String("database-password", "", "database password")
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
 	rootCmd.PersistentFlags().String("log-file", "", "file path for logging")
@@ -114,11 +110,6 @@ func init() {
 	rootCmd.PersistentFlags().String("prom-http-addr", "127.0.0.1", "http host for prometheus")
 	rootCmd.PersistentFlags().String("prom-http-port", "8090", "http port for prometheus")
 
-	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
-	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
-	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
-	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
-	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
 

cmd/serve.go

@@ -16,6 +16,9 @@
 package cmd
 
 import (
+	"github.com/vulcanize/gap-filler/pkg/mux"
+	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
 	"sync"
@@ -71,7 +74,7 @@ func serve() {
 	if err := startServers(server, serverConfig); err != nil {
 		logWithCommand.Fatal(err)
 	}
-	graphQL, err := startGraphQL(server)
+	graphQL, err := startEthGraphQL(server, serverConfig)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@@ -87,27 +90,43 @@ func serve() {
 }
 
 func startServers(server s.Server, settings *s.Config) error {
+	if settings.IPCEnabled {
 		logWithCommand.Info("starting up IPC server")
 		_, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
 		if err != nil {
 			return err
 		}
+	} else {
+		logWithCommand.Info("IPC server is disabled")
+	}
+
+	if settings.WSEnabled {
 		logWithCommand.Info("starting up WS server")
-	_, _, err = srpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb"}, nil, true)
+		_, _, err := srpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb", "net"}, nil, true)
 		if err != nil {
 			return err
 		}
-	logWithCommand.Info("starting up HTTP server")
+	} else {
-	_, err = srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"eth", "net"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
+		logWithCommand.Info("WS server is disabled")
-	return err
 	}
 
-func startGraphQL(server s.Server) (graphQLServer *graphql.Service, err error) {
+	if settings.HTTPEnabled {
-	viper.BindEnv("server.graphql", "SERVER_GRAPHQL")
+		logWithCommand.Info("starting up HTTP server")
-	if viper.GetBool("server.graphql") {
+		_, err := srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"eth", "net"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
-		logWithCommand.Info("starting up GraphQL server")
+		if err != nil {
-		viper.BindEnv("server.graphqlEndpoint", "SERVER_GRAPHQL_ENDPOINT")
+			return err
-		endPoint := viper.GetString("server.graphqlEndpoint")
+		}
+	} else {
+		logWithCommand.Info("HTTP server is disabled")
+	}
+
+	return nil
+}
+
+func startEthGraphQL(server s.Server, settings *s.Config) (graphQLServer *graphql.Service, err error) {
+	if settings.EthGraphqlEnabled {
+		logWithCommand.Info("starting up ETH GraphQL server")
+		endPoint := settings.EthGraphqlEndpoint
 		if endPoint != "" {
 			graphQLServer, err = graphql.New(server.Backend(), endPoint, nil, []string{"*"}, rpc.HTTPTimeouts{})
 			if err != nil {
@@ -115,19 +134,76 @@ func startGraphQL(server s.Server) (graphQLServer *graphql.Service, err error) {
 			}
 			err = graphQLServer.Start(nil)
 		}
+	} else {
+		logWithCommand.Info("ETH GraphQL server is disabled")
 	}
+
 	return
 }
 
+func startIpldGraphQL(settings *s.Config) error {
+	if settings.IpldGraphqlEnabled {
+		logWithCommand.Info("starting up IPLD GraphQL server")
+
+		gqlDefaultAddr, err := url.Parse(settings.IpldPostgraphileEndpoint)
+		if err != nil {
+			return err
+		}
+
+		router, err := mux.NewServeMux(&mux.Options{
+			BasePath:       "/",
+			EnableGraphiQL: true,
+			Postgraphile: mux.PostgraphileOptions{
+				Default:    gqlDefaultAddr,
+				TracingAPI: gqlTracingAPIAddr,
+			},
+			RPC: mux.RPCOptions{
+				DefaultClients: settings.Client,
+				TracingClients: tracingClients,
+			},
+		})
+		if err != nil {
+			logWithCommand.Fatal(err)
+		}
+
+		addr := fmt.Sprintf("%s:%s", viper.GetString("http.host"), viper.GetString("http.port"))
+		if err := http.ListenAndServe(addr, router); err != nil {
+			logWithCommand.Fatal(err)
+		}
+	} else {
+		logWithCommand.Info("IPLD GraphQL server is disabled")
+	}
+
+	return nil
+}
+
 func init() {
 	rootCmd.AddCommand(serveCmd)
 
+	// database credentials
+	serveCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
+	serveCmd.PersistentFlags().Int("database-port", 5432, "database port")
+	serveCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
+	serveCmd.PersistentFlags().String("database-user", "", "database user")
+	serveCmd.PersistentFlags().String("database-password", "", "database password")
+
 	// flags for all config variables
-	serveCmd.PersistentFlags().Bool("server-graphql", false, "turn on the graphql server")
+	// eth graphql and json-rpc parameters
-	serveCmd.PersistentFlags().String("server-graphql-endpoint", "", "endpoint url for graphql server")
+	serveCmd.PersistentFlags().Bool("eth-server-graphql", false, "turn on the eth graphql server")
-	serveCmd.PersistentFlags().String("server-ws-path", "", "vdb server ws path")
+	serveCmd.PersistentFlags().String("eth-server-graphql-path", "", "endpoint url for eth graphql server (host:port)")
-	serveCmd.PersistentFlags().String("server-http-path", "", "vdb server http path")
+	serveCmd.PersistentFlags().Bool("eth-server-http", true, "turn on the eth http json-rpc server")
-	serveCmd.PersistentFlags().String("server-ipc-path", "", "vdb server ipc path")
+	serveCmd.PersistentFlags().String("eth-server-http-path", "", "endpoint url for eth http json-rpc server (host:port)")
+	serveCmd.PersistentFlags().Bool("eth-server-ws", false, "turn on the eth websocket json-rpc server")
+	serveCmd.PersistentFlags().String("eth-server-ws-path", "", "endpoint url for eth websocket json-rpc server (host:port)")
+	serveCmd.PersistentFlags().Bool("eth-server-ipc", false, "turn on the eth ipc json-rpc server")
+	serveCmd.PersistentFlags().String("eth-server-ipc-path", "", "path for eth ipc json-rpc server")
+
+	// ipld and tracing graphql parameters
+	serveCmd.PersistentFlags().Bool("ipld-server-graphql", false, "turn on the ipld graphql server")
+	serveCmd.PersistentFlags().String("ipld-server-graphql-path", "", "endpoint url for ipld graphql server (host:port)")
+	serveCmd.PersistentFlags().String("ipld-postgraphile-path", "", "http url to postgraphile on top of ipld database")
+	serveCmd.PersistentFlags().String("tracing-http-path", "", "http url to tracing service")
+	serveCmd.PersistentFlags().String("tracing-postgraphile-path", "", "http url to postgraphile on top of tracing db")
 
 	serveCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
 	serveCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
@@ -141,11 +217,35 @@ func init() {
 	serveCmd.PersistentFlags().Bool("eth-supports-state-diff", false, "whether or not the proxy ethereum client supports statediffing endpoints")
 
 	// and their bindings
-	viper.BindPFlag("server.graphql", serveCmd.PersistentFlags().Lookup("server-graphql"))
+	// database
-	viper.BindPFlag("server.graphqlEndpoint", serveCmd.PersistentFlags().Lookup("server-graphql-endpoint"))
+	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
-	viper.BindPFlag("server.wsPath", serveCmd.PersistentFlags().Lookup("server-ws-path"))
+	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
-	viper.BindPFlag("server.httpPath", serveCmd.PersistentFlags().Lookup("server-http-path"))
+	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
-	viper.BindPFlag("server.ipcPath", serveCmd.PersistentFlags().Lookup("server-ipc-path"))
+	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
+	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
+
+	// eth graphql server
+	viper.BindPFlag("eth.server.graphql", serveCmd.PersistentFlags().Lookup("eth-server-graphql"))
+	viper.BindPFlag("eth.server.graphqlPath", serveCmd.PersistentFlags().Lookup("eth-server-graphql-path"))
+
+	// eth http json-rpc server
+	viper.BindPFlag("eth.server.http", serveCmd.PersistentFlags().Lookup("eth-server-http"))
+	viper.BindPFlag("eth.server.httpPath", serveCmd.PersistentFlags().Lookup("eth-server-http-path"))
+
+	// eth websocket json-rpc server
+	viper.BindPFlag("eth.server.ws", serveCmd.PersistentFlags().Lookup("eth-server-ws"))
+	viper.BindPFlag("eth.server.wsPath", serveCmd.PersistentFlags().Lookup("eth-server-ws-path"))
+
+	// eth ipc json-rpc server
+	viper.BindPFlag("eth.server.ipc", serveCmd.PersistentFlags().Lookup("eth-server-ipc"))
+	viper.BindPFlag("eth.server.ipcPath", serveCmd.PersistentFlags().Lookup("eth-server-ipc-path"))
+
+	// ipld and tracing graphql parameters
+	viper.BindPFlag("ipld.server.graphql", serveCmd.PersistentFlags().Lookup("ipld-server-graphql"))
+	viper.BindPFlag("ipld.server.graphqlPath", serveCmd.PersistentFlags().Lookup("ipld-server-graphql-path"))
+	viper.BindPFlag("ipld.postgraphilePath", serveCmd.PersistentFlags().Lookup("ipld-postgraphile-path"))
+	viper.BindPFlag("tracing.httpPath", serveCmd.PersistentFlags().Lookup("tracing-http-path"))
+	viper.BindPFlag("tracing.postgraphilePath", serveCmd.PersistentFlags().Lookup("tracing-postgraphile-path"))
 
 	viper.BindPFlag("ethereum.httpPath", serveCmd.PersistentFlags().Lookup("eth-http-path"))
 	viper.BindPFlag("ethereum.nodeID", serveCmd.PersistentFlags().Lookup("eth-node-id"))

pkg/serve/config.go

@@ -17,6 +17,7 @@
 package serve
 
 import (
+	"errors"
 	"fmt"
 	"math/big"
 	"os"
@@ -56,9 +57,25 @@ const (
 type Config struct {
 	DB       *postgres.DB
 	DBConfig postgres.Config
+
+	WSEnabled  bool
 	WSEndpoint string
+
+	HTTPEnabled  bool
 	HTTPEndpoint string
+
+	IPCEnabled  bool
 	IPCEndpoint string
+
+	EthGraphqlEnabled  bool
+	EthGraphqlEndpoint string
+
+	IpldGraphqlEnabled          bool
+	IpldGraphqlEndpoint         string
+	IpldPostgraphileEndpoint    string
+	TracingHttpEndpoint         string
+	TracingPostgraphileEndpoint string
+
 	ChainConfig      *params.ChainConfig
 	DefaultSender    *common.Address
 	RPCGasCap        *big.Int
@@ -71,9 +88,6 @@ type Config struct {
 func NewConfig() (*Config, error) {
 	c := new(Config)
 
-	viper.BindEnv("server.wsPath", SERVER_WS_PATH)
-	viper.BindEnv("server.ipcPath", SERVER_IPC_PATH)
-	viper.BindEnv("server.httpPath", SERVER_HTTP_PATH)
 	viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
 	viper.BindEnv("ethereum.defaultSender", ETH_DEFAULT_SENDER_ADDR)
 	viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP)
@@ -90,12 +104,21 @@ func NewConfig() (*Config, error) {
 	c.Client = cli
 	c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff")
 
-	wsPath := viper.GetString("server.wsPath")
+	// websocket server
+	wsEnabled := viper.GetBool("eth.server.ws")
+	if wsEnabled {
+		wsPath := viper.GetString("eth.server.wsPath")
 		if wsPath == "" {
 			wsPath = "127.0.0.1:8080"
 		}
 		c.WSEndpoint = wsPath
-	ipcPath := viper.GetString("server.ipcPath")
+	}
+	c.WSEnabled = wsEnabled
+
+	// ipc server
+	ipcEnabled := viper.GetBool("eth.server.ipc")
+	if ipcEnabled {
+		ipcPath := viper.GetString("eth.server.ipcPath")
 		if ipcPath == "" {
 			home, err := os.UserHomeDir()
 			if err != nil {
@@ -104,11 +127,59 @@ func NewConfig() (*Config, error) {
 			ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
 		}
 		c.IPCEndpoint = ipcPath
-	httpPath := viper.GetString("server.httpPath")
+	}
+	c.IPCEnabled = ipcEnabled
+
+	// http server
+	httpEnabled := viper.GetBool("eth.server.http")
+	if httpEnabled {
+		httpPath := viper.GetString("eth.server.httpPath")
 		if httpPath == "" {
 			httpPath = "127.0.0.1:8081"
 		}
 		c.HTTPEndpoint = httpPath
+	}
+	c.HTTPEnabled = httpEnabled
+
+	// eth graphql endpoint
+	ethGraphqlEnabled := viper.GetBool("eth.server.graphql")
+	if ethGraphqlEnabled {
+		ethGraphqlPath := viper.GetString("eth.server.graphqlPath")
+		if ethGraphqlPath == "" {
+			ethGraphqlPath = "127.0.0.1:8082"
+		}
+		c.EthGraphqlEndpoint = ethGraphqlPath
+	}
+	c.EthGraphqlEnabled = ethGraphqlEnabled
+
+	// ipld graphql endpoint
+	ipldGraphqlEnabled := viper.GetBool("ipld.server.graphql")
+	if ipldGraphqlEnabled {
+		ipldGraphqlPath := viper.GetString("ipld.server.graphqlPath")
+		if ipldGraphqlPath == "" {
+			ipldGraphqlPath = "127.0.0.1:8083"
+		}
+		c.IpldGraphqlEndpoint = ipldGraphqlPath
+
+		ipldPostgraphilePath := viper.GetString("ipld.postgraphilePath")
+		if ipldPostgraphilePath == "" {
+			return nil, errors.New("ipld-postgraphile-path parameter is empty")
+		}
+		c.IpldPostgraphileEndpoint = ipldPostgraphilePath
+
+		tracingHttpEndpoint := viper.GetString("tracing.httpPath")
+		tracingPostgraphilePath := viper.GetString("tracing.postgraphilePath")
+
+		// these two parameters either can be both empty or both set
+		if (tracingHttpEndpoint == "" && tracingPostgraphilePath != "") || (tracingHttpEndpoint != "" && tracingPostgraphilePath == "") {
+			return nil, errors.New("tracing.httpPath and tracing.postgraphilePath parameters either can be both empty or both set")
+		}
+
+		c.TracingHttpEndpoint = tracingHttpEndpoint
+		c.TracingPostgraphileEndpoint = tracingPostgraphilePath
+	}
+	c.IpldGraphqlEnabled = ipldGraphqlEnabled
+
 	overrideDBConnConfig(&c.DBConfig)
 	serveDB := utils.LoadPostgres(c.DBConfig, nodeInfo, false)
 	prom.RegisterDBCollector(c.DBConfig.Name, serveDB.DB)