From fa23e1164e0156124f24a2ff7b2ce69633fc14e4 Mon Sep 17 00:00:00 2001 From: i-norden Date: Fri, 31 Dec 2021 12:17:57 -0600 Subject: [PATCH] fin/fix CLI and ENV integration --- .gitignore | 1 + cmd/root.go | 87 +++++++++++++++++++++------------------ cmd/serve.go | 35 +++++++++++----- environments/example.toml | 13 ++++++ pkg/api.go | 12 ++++-- pkg/backend.go | 24 ++++++----- pkg/config.go | 60 +++++++++++++++++++++++---- pkg/env.go | 42 ++++++++++++++++--- pkg/service.go | 7 ++-- 9 files changed, 199 insertions(+), 82 deletions(-) create mode 100644 environments/example.toml diff --git a/.gitignore b/.gitignore index 9f11b75..4469811 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .idea/ +leveldb-ethdb-rpc diff --git a/cmd/root.go b/cmd/root.go index 203d807..02b3359 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,8 +20,10 @@ import ( "fmt" "os" - "github.com/spf13/cobra" + leveldb_ethdb_rpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -29,59 +31,64 @@ var cfgFile string // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ - Use: "leveldb-ethdb-rpc", - Short: "A brief description of your application", - Long: `A longer description that spans multiple lines and likely contains -examples and usage of using your application. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - // Uncomment the following line if your bare application - // has an action associated with it: - // Run: func(cmd *cobra.Command, args []string) { }, + Use: "leveldb-ethdb-rpc", + PersistentPreRun: initFuncs, } // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { - cobra.CheckErr(rootCmd.Execute()) + log.Info("----- Starting IPFS blockchain watcher -----") + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) + } } -func init() { - cobra.OnInitialize(initConfig) +func initFuncs(cmd *cobra.Command, args []string) { + viper.BindEnv(leveldb_ethdb_rpc.TOML_LOGRUS_FILE, leveldb_ethdb_rpc.LOGRUS_FILE) + logfile := viper.GetString(leveldb_ethdb_rpc.TOML_LOGRUS_FILE) + if logfile != "" { + file, err := os.OpenFile(logfile, + os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err == nil { + log.Infof("Directing output to %s", logfile) + log.SetOutput(file) + } else { + log.SetOutput(os.Stdout) + log.Info("Failed to log to file, using default stdout") + } + } else { + log.SetOutput(os.Stdout) + } + if err := logLevel(); err != nil { + log.Fatal("Could not set log level: ", err) + } +} - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.leveldb-ethdb-rpc.yaml)") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +func logLevel() error { + viper.BindEnv(leveldb_ethdb_rpc.TOML_LOGRUS_LEVEL, leveldb_ethdb_rpc.LOGRUS_LEVEL) + lvl, err := log.ParseLevel(viper.GetString(leveldb_ethdb_rpc.TOML_LOGRUS_LEVEL)) + if err != nil { + return err + } + log.SetLevel(lvl) + if lvl > log.InfoLevel { + log.SetReportCaller(true) + } + log.Info("Log level set to ", lvl.String()) + return nil } // initConfig reads in config file and ENV variables if set. func initConfig() { if cfgFile != "" { - // Use config file from the flag. viper.SetConfigFile(cfgFile) + if err := viper.ReadInConfig(); err == nil { + log.Printf("Using config file: %s", viper.ConfigFileUsed()) + } else { + log.Fatal(fmt.Sprintf("Couldn't read config file: %s", err.Error())) + } } else { - // Find home directory. - home, err := os.UserHomeDir() - cobra.CheckErr(err) - - // Search config in home directory with name ".leveldb-ethdb-rpc" (without extension). - viper.AddConfigPath(home) - viper.SetConfigType("yaml") - viper.SetConfigName(".leveldb-ethdb-rpc") - } - - viper.AutomaticEnv() // read in environment variables that match - - // If a config file is found, read it in. - if err := viper.ReadInConfig(); err == nil { - fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed()) + log.Warn("No config file passed with --config flag") } } diff --git a/cmd/serve.go b/cmd/serve.go index 036d674..78ff416 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -24,8 +24,9 @@ import ( "github.com/ethereum/go-ethereum/rpc" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/spf13/viper" - "github.com/vulcanize/leveldb-ethdb-rpc/pkg" + leveldb_ethdb_rpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg" srpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg/rpc" "github.com/vulcanize/leveldb-ethdb-rpc/version" ) @@ -52,13 +53,13 @@ func serve() { wg := new(sync.WaitGroup) logWithCommand.Debug("loading server configuration variables") - serverConfig, err := pkg.NewConfig() + serverConfig, err := leveldb_ethdb_rpc.NewConfig() if err != nil { logWithCommand.Fatal(err) } logWithCommand.Infof("server config: %+v", serverConfig) logWithCommand.Debug("initializing new server service") - server, err := pkg.NewServer(serverConfig) + server, err := leveldb_ethdb_rpc.NewServer(serverConfig) if err != nil { logWithCommand.Fatal(err) } @@ -76,7 +77,7 @@ func serve() { wg.Wait() } -func startServers(server pkg.Server, settings pkg.Config) error { +func startServers(server leveldb_ethdb_rpc.Server, settings *leveldb_ethdb_rpc.Config) error { if settings.IPCEnabled { logWithCommand.Info("starting up IPC server") _, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs()) @@ -103,13 +104,25 @@ func startServers(server pkg.Server, settings pkg.Config) error { func init() { rootCmd.AddCommand(serveCmd) - // Here you will define your flags and configuration settings. + // CLI flags + serveCmd.PersistentFlags().Bool("ipc-enabled", false, "turn on ipc server") + serveCmd.PersistentFlags().String("ipc-path", "", "ipc server endpoint") + serveCmd.PersistentFlags().Bool("http-enabled", true, "turn on http server; on by default") + serveCmd.PersistentFlags().String("http-path", "127.0.0.1:8500", "http server endpoint; default = 127.0.0.1:8545") - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // serveCmd.PersistentFlags().String("foo", "", "A help for foo") + serveCmd.PersistentFlags().String("leveldb-path", "", "leveldb filesystem path") + serveCmd.PersistentFlags().Int("leveldb-cache-size", 0, "leveldb cache size") + serveCmd.PersistentFlags().String("leveldb-ancient-path", "", "filesystem path to freezer") + serveCmd.PersistentFlags().String("leveldb-namespace", "eth/db/chaindata/", "leveldb namespace") - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // serveCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + // toml bindings + viper.BindPFlag(leveldb_ethdb_rpc.TOML_IPC_ENABLED, serveCmd.PersistentFlags().Lookup("ipc-enabled")) + viper.BindPFlag(leveldb_ethdb_rpc.TOML_IPC_ENDPOINT, serveCmd.PersistentFlags().Lookup("ipc-path")) + viper.BindPFlag(leveldb_ethdb_rpc.TOML_HTTP_ENABLED, serveCmd.PersistentFlags().Lookup("http-enabled")) + viper.BindPFlag(leveldb_ethdb_rpc.TOML_HTTP_ENDPOINT, serveCmd.PersistentFlags().Lookup("http-path")) + + viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_PATH, serveCmd.PersistentFlags().Lookup("leveldb-path")) + viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_CACHE_SIZE, serveCmd.PersistentFlags().Lookup("leveldb-cache-size")) + viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_ANCIENT_PATH, serveCmd.PersistentFlags().Lookup("leveldb-ancient-path")) + viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_NAMESPACE, serveCmd.PersistentFlags().Lookup("leveldb-namespace")) } diff --git a/environments/example.toml b/environments/example.toml new file mode 100644 index 0000000..fe3e8b4 --- /dev/null +++ b/environments/example.toml @@ -0,0 +1,13 @@ +[log] + level = "info" # $LOGRUS_LEVEL + file = "" # $LOGRUS_FILE + +[leveldb] + ipcEnabled = false # $IPC_ENABLED + ipcPath = "~/.vulcanize/vulcanize.ipc" # $IPC_PATH + httpEnabled = true # $HTTP_ENABLED + httpPath = "127.0.0.1:8082" # $HTTP_PATH + path = "/Users/user/Library/Ethereum/geth/chaindata" # $LEVELDB_PATH + ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # $LEVELDB_ANCIENT_PATH + cacheSize = 1024 # $LEVELDB_CACHE_SIZE + namespace = "eth/db/chaindata/" # $LEVELDB_NAMESPACE diff --git a/pkg/api.go b/pkg/api.go index 125084f..ae4841f 100644 --- a/pkg/api.go +++ b/pkg/api.go @@ -14,11 +14,13 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package pkg +package leveldb_ethdb_rpc import ( "context" "errors" + + "github.com/ethereum/go-ethereum/ethdb" ) // APIName is the namespace used for the state diffing service API @@ -55,8 +57,12 @@ func (s *PublicLevelDBAPI) Ancient(ctx context.Context, kind string, number uint return s.b.Ancient(kind, number) } -func (s *PublicLevelDBAPI) ReadAncients(ctx context.Context, kind string, start, count, maxBytes uint64) ([][]byte, error) { - return s.b.ReadAncients(kind, start, count, maxBytes) +func (s *PublicLevelDBAPI) AncientRange(ctx context.Context, kind string, start, count, maxBytes uint64) ([][]byte, error) { + return s.b.AncientRange(kind, start, count, maxBytes) +} + +func (s *PublicLevelDBAPI) ReadAncients(fn func(ethdb.AncientReader) error) error { + return s.b.ReadAncients(fn) } func (s *PublicLevelDBAPI) Ancients(ctx context.Context) (uint64, error) { diff --git a/pkg/backend.go b/pkg/backend.go index d08e822..78b872e 100644 --- a/pkg/backend.go +++ b/pkg/backend.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package pkg +package leveldb_ethdb_rpc import ( "github.com/ethereum/go-ethereum/core/rawdb" @@ -25,12 +25,12 @@ import ( var _ ethdb.Database = &LevelDBBackend{} // NewLevelDBBackend creates a new levelDB RPC server backend -func NewLevelDBBackend(conf Config) (*LevelDBBackend, error) { - db, err := leveldb.New(conf.File, conf.Cache, conf.Handles, conf.Namespace, conf.Readonly) +func NewLevelDBBackend(conf *Config) (*LevelDBBackend, error) { + db, err := leveldb.New(conf.FilePath, conf.Cache, conf.Handles, conf.Namespace, true) if err != nil { return nil, err } - frdb, err := rawdb.NewDatabaseWithFreezer(db, conf.Freezer, conf.Namespace, conf.Readonly) + frdb, err := rawdb.NewDatabaseWithFreezer(db, conf.FreezerPath, conf.Namespace, true) if err != nil { db.Close() return nil, err @@ -62,8 +62,12 @@ func (s *LevelDBBackend) Ancient(kind string, number uint64) ([]byte, error) { return s.ethDB.Ancient(kind, number) } -func (s *LevelDBBackend) ReadAncients(kind string, start, count, maxBytes uint64) ([][]byte, error) { - return s.ethDB.ReadAncients(kind, start, count, maxBytes) +func (s *LevelDBBackend) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { + return s.ethDB.AncientRange(kind, start, count, maxBytes) +} + +func (s *LevelDBBackend) ReadAncients(fn func(ethdb.AncientReader) error) error { + return s.ethDB.ReadAncients(fn) } func (s *LevelDBBackend) Ancients() (uint64, error) { @@ -74,11 +78,11 @@ func (s *LevelDBBackend) AncientSize(kind string) (uint64, error) { return s.ethDB.AncientSize(kind) } -func (s *LevelDBBackend) Put(key []byte, value []byte) error { +func (s *LevelDBBackend) Put(_ []byte, _ []byte) error { return errWriteNotAllowed } -func (s *LevelDBBackend) Delete(key []byte) error { +func (s *LevelDBBackend) Delete(_ []byte) error { return errWriteNotAllowed } @@ -98,7 +102,7 @@ func (s *LevelDBBackend) NewBatch() ethdb.Batch { return nil } -func (s *LevelDBBackend) NewIterator(prefix []byte, start []byte) ethdb.Iterator { +func (s *LevelDBBackend) NewIterator(_ []byte, _ []byte) ethdb.Iterator { return nil } @@ -106,7 +110,7 @@ func (s *LevelDBBackend) Stat(property string) (string, error) { return s.ethDB.Stat(property) } -func (s *LevelDBBackend) Compact(start []byte, limit []byte) error { +func (s *LevelDBBackend) Compact(_ []byte, _ []byte) error { return errWriteNotAllowed } diff --git a/pkg/config.go b/pkg/config.go index 1077239..532728f 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -14,7 +14,14 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package pkg +package leveldb_ethdb_rpc + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common/fdlimit" + "github.com/spf13/viper" +) // Config struct holds the configuration parameters for the levelDB RPC service type Config struct { @@ -23,15 +30,52 @@ type Config struct { HTTPEnabled bool HTTPEndpoint string - File string - Cache int - Handles int - Freezer string - Namespace string - Readonly bool + FilePath string + Cache int + Handles int + FreezerPath string + Namespace string } // NewConfig returns a new Config from viper parameters -func NewConfig() (Config, error) { +func NewConfig() (*Config, error) { + viper.BindEnv(TOML_IPC_ENABLED, IPC_ENABLED) + viper.BindEnv(TOML_IPC_ENDPOINT, IPC_ENDPOINT) + viper.BindEnv(TOML_HTTP_ENABLED, HTTP_ENABLED) + viper.BindEnv(TOML_HTTP_ENDPOINT, HTTP_ENDPOINT) + viper.BindEnv(TOML_LEVELDB_PATH, LEVELDB_PATH) + viper.BindEnv(TOML_LEVELDB_CACHE_SIZE, LEVELDB_CACHE_SIZE) + viper.BindEnv(TOML_LEVELDB_ANCIENT_PATH, LEVELDB_ANCIENT_PATH) + viper.BindEnv(TOML_LEVELDB_NAMESPACE, LEVELDB_NAMESPACE) + + numHandles, err := MakeDatabaseHandles() + if err != nil { + return nil, err + } + return &Config{ + IPCEnabled: viper.GetBool(TOML_IPC_ENABLED), + IPCEndpoint: viper.GetString(TOML_IPC_ENDPOINT), + HTTPEnabled: viper.GetBool(TOML_HTTP_ENABLED), + HTTPEndpoint: viper.GetString(TOML_HTTP_ENDPOINT), + FilePath: viper.GetString(TOML_LEVELDB_PATH), + Cache: viper.GetInt(TOML_LEVELDB_CACHE_SIZE), + Handles: numHandles, + FreezerPath: viper.GetString(TOML_LEVELDB_ANCIENT_PATH), + Namespace: viper.GetString(TOML_LEVELDB_NAMESPACE), + }, nil +} + +// MakeDatabaseHandles raises out the number of allowed file handles per process +// for Geth and returns half of the allowance to assign to the database. +func MakeDatabaseHandles() (int, error) { + limit, err := fdlimit.Maximum() + if err != nil { + return 0, fmt.Errorf("failed to retrieve file descriptor allowance: %v", err) + } + raised, err := fdlimit.Raise(uint64(limit)) + if err != nil { + return 0, fmt.Errorf("failed to raise file descriptor allowance: %v", err) + } + return int(raised / 2), nil // Leave half for networking and other stuff } diff --git a/pkg/env.go b/pkg/env.go index 2c58dd4..e3236df 100644 --- a/pkg/env.go +++ b/pkg/env.go @@ -1,15 +1,45 @@ -package pkg +// VulcanizeDB +// Copyright © 2022 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package leveldb_ethdb_rpc const ( + LOGRUS_LEVEL = "LOGRUS_LEVEL" + LOGRUS_FILE = "LOGRUS_FILE" + IPC_ENABLED = "IPC_ENABLED" - IPC_ENDPOINT = "IPC_ENDPOINT" + IPC_ENDPOINT = "IPC_PATH" HTTP_ENABLED = "HTTP_ENABLED" - HTTP_ENDPOINT = "HTTP_ENDPOINT" + HTTP_ENDPOINT = "HTTP_PATH" LEVELDB_PATH = "LEVELDB_PATH" LEVELDB_CACHE_SIZE = "LEVELDB_CACHE_SIZE" - LEVELDB_HANDLER_NUM = "LEVELDB_HANDLER_NUM" - LEVELDB_FREEZER_PATH = "LEVELDB_FREEZER_PATH" + LEVELDB_ANCIENT_PATH = "LEVELDB_ANCIENT_PATH" LEVELDB_NAMESPACE = "LEVELDB_NAMESPACE" - LEVELDB_READ_ONLY = "LEVELDB_READ_ONLY" + + TOML_LOGRUS_LEVEL = "log.level" + TOML_LOGRUS_FILE = "log.file" + + TOML_IPC_ENABLED = "leveldb.ipcEnabled" + TOML_IPC_ENDPOINT = "leveldb.ipcPath" + TOML_HTTP_ENABLED = "leveldb.httpEnabled" + TOML_HTTP_ENDPOINT = "leveldb.httpPath" + + TOML_LEVELDB_PATH = "leveldb.path" + TOML_LEVELDB_CACHE_SIZE = "leveldb.cacheSize" + TOML_LEVELDB_ANCIENT_PATH = "leveldb.ancient" + TOML_LEVELDB_NAMESPACE = "leveldb.namespace" ) diff --git a/pkg/service.go b/pkg/service.go index 65ff156..0f3b61f 100644 --- a/pkg/service.go +++ b/pkg/service.go @@ -14,16 +14,15 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package pkg +package leveldb_ethdb_rpc import ( "sync" - log "github.com/sirupsen/logrus" - ethnode "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rpc" + log "github.com/sirupsen/logrus" ) // Server is the top level interface for exposing a remote RPC wrapper around levelDB ethdb.Database @@ -42,7 +41,7 @@ type Service struct { } // NewServer creates a new Server using an underlying Service struct -func NewServer(conf Config) (Server, error) { +func NewServer(conf *Config) (Server, error) { sap := new(Service) sap.quitChan = make(chan struct{}) var err error -- 2.45.2