Merge pull request #1 from vulcanize/working_branch

fin/fix CLI and ENV integration

.gitignore

@@ -1 +1,2 @@
 .idea/
+leveldb-ethdb-rpc

cmd/root.go

@@ -20,8 +20,10 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/spf13/cobra"
+	leveldb_ethdb_rpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg"
 
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
@@ -29,59 +31,64 @@ var cfgFile string
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
-	Use:   "leveldb-ethdb-rpc",
+	Use:              "leveldb-ethdb-rpc",
-	Short: "A brief description of your application",
+	PersistentPreRun: initFuncs,
-	Long: `A longer description that spans multiple lines and likely contains
-examples and usage of using your application. For example:
-
-Cobra is a CLI library for Go that empowers applications.
-This application is a tool to generate the needed files
-to quickly create a Cobra application.`,
-	// Uncomment the following line if your bare application
-	// has an action associated with it:
-	// Run: func(cmd *cobra.Command, args []string) { },
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
-	cobra.CheckErr(rootCmd.Execute())
+	log.Info("----- Starting IPFS blockchain watcher -----")
+	if err := rootCmd.Execute(); err != nil {
+		log.Fatal(err)
+	}
 }
 
-func init() {
+func initFuncs(cmd *cobra.Command, args []string) {
-	cobra.OnInitialize(initConfig)
+	viper.BindEnv(leveldb_ethdb_rpc.TOML_LOGRUS_FILE, leveldb_ethdb_rpc.LOGRUS_FILE)
+	logfile := viper.GetString(leveldb_ethdb_rpc.TOML_LOGRUS_FILE)
+	if logfile != "" {
+		file, err := os.OpenFile(logfile,
+			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+		if err == nil {
+			log.Infof("Directing output to %s", logfile)
+			log.SetOutput(file)
+		} else {
+			log.SetOutput(os.Stdout)
+			log.Info("Failed to log to file, using default stdout")
+		}
+	} else {
+		log.SetOutput(os.Stdout)
+	}
+	if err := logLevel(); err != nil {
+		log.Fatal("Could not set log level: ", err)
+	}
+}
 
-	// Here you will define your flags and configuration settings.
+func logLevel() error {
-	// Cobra supports persistent flags, which, if defined here,
+	viper.BindEnv(leveldb_ethdb_rpc.TOML_LOGRUS_LEVEL, leveldb_ethdb_rpc.LOGRUS_LEVEL)
-	// will be global for your application.
+	lvl, err := log.ParseLevel(viper.GetString(leveldb_ethdb_rpc.TOML_LOGRUS_LEVEL))
-
+	if err != nil {
-	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.leveldb-ethdb-rpc.yaml)")
+		return err
-
+	}
-	// Cobra also supports local flags, which will only run
+	log.SetLevel(lvl)
-	// when this action is called directly.
+	if lvl > log.InfoLevel {
-	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+		log.SetReportCaller(true)
+	}
+	log.Info("Log level set to ", lvl.String())
+	return nil
 }
 
 // initConfig reads in config file and ENV variables if set.
 func initConfig() {
 	if cfgFile != "" {
-		// Use config file from the flag.
 		viper.SetConfigFile(cfgFile)
+		if err := viper.ReadInConfig(); err == nil {
+			log.Printf("Using config file: %s", viper.ConfigFileUsed())
+		} else {
+			log.Fatal(fmt.Sprintf("Couldn't read config file: %s", err.Error()))
+		}
 	} else {
-		// Find home directory.
+		log.Warn("No config file passed with --config flag")
-		home, err := os.UserHomeDir()
-		cobra.CheckErr(err)
-
-		// Search config in home directory with name ".leveldb-ethdb-rpc" (without extension).
-		viper.AddConfigPath(home)
-		viper.SetConfigType("yaml")
-		viper.SetConfigName(".leveldb-ethdb-rpc")
-	}
-
-	viper.AutomaticEnv() // read in environment variables that match
-
-	// If a config file is found, read it in.
-	if err := viper.ReadInConfig(); err == nil {
-		fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed())
 	}
 }

cmd/serve.go

@@ -24,8 +24,9 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 
-	"github.com/vulcanize/leveldb-ethdb-rpc/pkg"
+	leveldb_ethdb_rpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg"
 	srpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg/rpc"
 	"github.com/vulcanize/leveldb-ethdb-rpc/version"
 )
@@ -52,13 +53,13 @@ func serve() {
 
 	wg := new(sync.WaitGroup)
 	logWithCommand.Debug("loading server configuration variables")
-	serverConfig, err := pkg.NewConfig()
+	serverConfig, err := leveldb_ethdb_rpc.NewConfig()
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
 	logWithCommand.Infof("server config: %+v", serverConfig)
 	logWithCommand.Debug("initializing new server service")
-	server, err := pkg.NewServer(serverConfig)
+	server, err := leveldb_ethdb_rpc.NewServer(serverConfig)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@@ -76,7 +77,7 @@ func serve() {
 	wg.Wait()
 }
 
-func startServers(server pkg.Server, settings pkg.Config) error {
+func startServers(server leveldb_ethdb_rpc.Server, settings *leveldb_ethdb_rpc.Config) error {
 	if settings.IPCEnabled {
 		logWithCommand.Info("starting up IPC server")
 		_, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
@@ -103,13 +104,25 @@ func startServers(server pkg.Server, settings pkg.Config) error {
 func init() {
 	rootCmd.AddCommand(serveCmd)
 
-	// Here you will define your flags and configuration settings.
+	// CLI flags
+	serveCmd.PersistentFlags().Bool("ipc-enabled", false, "turn on ipc server")
+	serveCmd.PersistentFlags().String("ipc-path", "", "ipc server endpoint")
+	serveCmd.PersistentFlags().Bool("http-enabled", true, "turn on http server; on by default")
+	serveCmd.PersistentFlags().String("http-path", "127.0.0.1:8500", "http server endpoint; default = 127.0.0.1:8545")
 
-	// Cobra supports Persistent Flags which will work for this command
+	serveCmd.PersistentFlags().String("leveldb-path", "", "leveldb filesystem path")
-	// and all subcommands, e.g.:
+	serveCmd.PersistentFlags().Int("leveldb-cache-size", 0, "leveldb cache size")
-	// serveCmd.PersistentFlags().String("foo", "", "A help for foo")
+	serveCmd.PersistentFlags().String("leveldb-ancient-path", "", "filesystem path to freezer")
+	serveCmd.PersistentFlags().String("leveldb-namespace", "eth/db/chaindata/", "leveldb namespace")
 
-	// Cobra supports local flags which will only run when this command
+	// toml bindings
-	// is called directly, e.g.:
+	viper.BindPFlag(leveldb_ethdb_rpc.TOML_IPC_ENABLED, serveCmd.PersistentFlags().Lookup("ipc-enabled"))
-	// serveCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+	viper.BindPFlag(leveldb_ethdb_rpc.TOML_IPC_ENDPOINT, serveCmd.PersistentFlags().Lookup("ipc-path"))
+	viper.BindPFlag(leveldb_ethdb_rpc.TOML_HTTP_ENABLED, serveCmd.PersistentFlags().Lookup("http-enabled"))
+	viper.BindPFlag(leveldb_ethdb_rpc.TOML_HTTP_ENDPOINT, serveCmd.PersistentFlags().Lookup("http-path"))
+
+	viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_PATH, serveCmd.PersistentFlags().Lookup("leveldb-path"))
+	viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_CACHE_SIZE, serveCmd.PersistentFlags().Lookup("leveldb-cache-size"))
+	viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_ANCIENT_PATH, serveCmd.PersistentFlags().Lookup("leveldb-ancient-path"))
+	viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_NAMESPACE, serveCmd.PersistentFlags().Lookup("leveldb-namespace"))
 }

environments/example.toml

@@ -0,0 +1,13 @@
+[log]
+    level = "info" # $LOGRUS_LEVEL
+    file = "" # $LOGRUS_FILE
+
+[leveldb]
+    ipcEnabled = false # $IPC_ENABLED
+    ipcPath = "~/.vulcanize/vulcanize.ipc" # $IPC_PATH
+    httpEnabled = true # $HTTP_ENABLED
+    httpPath = "127.0.0.1:8082" # $HTTP_PATH
+    path = "/Users/user/Library/Ethereum/geth/chaindata" # $LEVELDB_PATH
+    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # $LEVELDB_ANCIENT_PATH
+    cacheSize = 1024 # $LEVELDB_CACHE_SIZE
+    namespace = "eth/db/chaindata/" # $LEVELDB_NAMESPACE

pkg/api.go

@@ -14,11 +14,13 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package pkg
+package leveldb_ethdb_rpc
 
 import (
 	"context"
 	"errors"
+
+	"github.com/ethereum/go-ethereum/ethdb"
 )
 
 // APIName is the namespace used for the state diffing service API
@@ -55,8 +57,12 @@ func (s *PublicLevelDBAPI) Ancient(ctx context.Context, kind string, number uint
 	return s.b.Ancient(kind, number)
 }
 
-func (s *PublicLevelDBAPI) ReadAncients(ctx context.Context, kind string, start, count, maxBytes uint64) ([][]byte, error) {
+func (s *PublicLevelDBAPI) AncientRange(ctx context.Context, kind string, start, count, maxBytes uint64) ([][]byte, error) {
-	return s.b.ReadAncients(kind, start, count, maxBytes)
+	return s.b.AncientRange(kind, start, count, maxBytes)
+}
+
+func (s *PublicLevelDBAPI) ReadAncients(fn func(ethdb.AncientReader) error) error {
+	return s.b.ReadAncients(fn)
 }
 
 func (s *PublicLevelDBAPI) Ancients(ctx context.Context) (uint64, error) {

pkg/backend.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package pkg
+package leveldb_ethdb_rpc
 
 import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -25,12 +25,12 @@ import (
 var _ ethdb.Database = &LevelDBBackend{}
 
 // NewLevelDBBackend creates a new levelDB RPC server backend
-func NewLevelDBBackend(conf Config) (*LevelDBBackend, error) {
+func NewLevelDBBackend(conf *Config) (*LevelDBBackend, error) {
-	db, err := leveldb.New(conf.File, conf.Cache, conf.Handles, conf.Namespace, conf.Readonly)
+	db, err := leveldb.New(conf.FilePath, conf.Cache, conf.Handles, conf.Namespace, true)
 	if err != nil {
 		return nil, err
 	}
-	frdb, err := rawdb.NewDatabaseWithFreezer(db, conf.Freezer, conf.Namespace, conf.Readonly)
+	frdb, err := rawdb.NewDatabaseWithFreezer(db, conf.FreezerPath, conf.Namespace, true)
 	if err != nil {
 		db.Close()
 		return nil, err
@@ -62,8 +62,12 @@ func (s *LevelDBBackend) Ancient(kind string, number uint64) ([]byte, error) {
 	return s.ethDB.Ancient(kind, number)
 }
 
-func (s *LevelDBBackend) ReadAncients(kind string, start, count, maxBytes uint64) ([][]byte, error) {
+func (s *LevelDBBackend) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
-	return s.ethDB.ReadAncients(kind, start, count, maxBytes)
+	return s.ethDB.AncientRange(kind, start, count, maxBytes)
+}
+
+func (s *LevelDBBackend) ReadAncients(fn func(ethdb.AncientReader) error) error {
+	return s.ethDB.ReadAncients(fn)
 }
 
 func (s *LevelDBBackend) Ancients() (uint64, error) {
@@ -74,11 +78,11 @@ func (s *LevelDBBackend) AncientSize(kind string) (uint64, error) {
 	return s.ethDB.AncientSize(kind)
 }
 
-func (s *LevelDBBackend) Put(key []byte, value []byte) error {
+func (s *LevelDBBackend) Put(_ []byte, _ []byte) error {
 	return errWriteNotAllowed
 }
 
-func (s *LevelDBBackend) Delete(key []byte) error {
+func (s *LevelDBBackend) Delete(_ []byte) error {
 	return errWriteNotAllowed
 }
 
@@ -98,7 +102,7 @@ func (s *LevelDBBackend) NewBatch() ethdb.Batch {
 	return nil
 }
 
-func (s *LevelDBBackend) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
+func (s *LevelDBBackend) NewIterator(_ []byte, _ []byte) ethdb.Iterator {
 	return nil
 }
 
@@ -106,7 +110,7 @@ func (s *LevelDBBackend) Stat(property string) (string, error) {
 	return s.ethDB.Stat(property)
 }
 
-func (s *LevelDBBackend) Compact(start []byte, limit []byte) error {
+func (s *LevelDBBackend) Compact(_ []byte, _ []byte) error {
 	return errWriteNotAllowed
 }
 

pkg/config.go

@@ -14,7 +14,14 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package pkg
+package leveldb_ethdb_rpc
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common/fdlimit"
+	"github.com/spf13/viper"
+)
 
 // Config struct holds the configuration parameters for the levelDB RPC service
 type Config struct {
@@ -23,15 +30,52 @@ type Config struct {
 	HTTPEnabled  bool
 	HTTPEndpoint string
 
-	File      string
+	FilePath    string
-	Cache     int
+	Cache       int
-	Handles   int
+	Handles     int
-	Freezer   string
+	FreezerPath string
-	Namespace string
+	Namespace   string
-	Readonly  bool
 }
 
 // NewConfig returns a new Config from viper parameters
-func NewConfig() (Config, error) {
+func NewConfig() (*Config, error) {
+	viper.BindEnv(TOML_IPC_ENABLED, IPC_ENABLED)
+	viper.BindEnv(TOML_IPC_ENDPOINT, IPC_ENDPOINT)
+	viper.BindEnv(TOML_HTTP_ENABLED, HTTP_ENABLED)
+	viper.BindEnv(TOML_HTTP_ENDPOINT, HTTP_ENDPOINT)
 
+	viper.BindEnv(TOML_LEVELDB_PATH, LEVELDB_PATH)
+	viper.BindEnv(TOML_LEVELDB_CACHE_SIZE, LEVELDB_CACHE_SIZE)
+	viper.BindEnv(TOML_LEVELDB_ANCIENT_PATH, LEVELDB_ANCIENT_PATH)
+	viper.BindEnv(TOML_LEVELDB_NAMESPACE, LEVELDB_NAMESPACE)
+
+	numHandles, err := MakeDatabaseHandles()
+	if err != nil {
+		return nil, err
+	}
+	return &Config{
+		IPCEnabled:   viper.GetBool(TOML_IPC_ENABLED),
+		IPCEndpoint:  viper.GetString(TOML_IPC_ENDPOINT),
+		HTTPEnabled:  viper.GetBool(TOML_HTTP_ENABLED),
+		HTTPEndpoint: viper.GetString(TOML_HTTP_ENDPOINT),
+		FilePath:     viper.GetString(TOML_LEVELDB_PATH),
+		Cache:        viper.GetInt(TOML_LEVELDB_CACHE_SIZE),
+		Handles:      numHandles,
+		FreezerPath:  viper.GetString(TOML_LEVELDB_ANCIENT_PATH),
+		Namespace:    viper.GetString(TOML_LEVELDB_NAMESPACE),
+	}, nil
+}
+
+// MakeDatabaseHandles raises out the number of allowed file handles per process
+// for Geth and returns half of the allowance to assign to the database.
+func MakeDatabaseHandles() (int, error) {
+	limit, err := fdlimit.Maximum()
+	if err != nil {
+		return 0, fmt.Errorf("failed to retrieve file descriptor allowance: %v", err)
+	}
+	raised, err := fdlimit.Raise(uint64(limit))
+	if err != nil {
+		return 0, fmt.Errorf("failed to raise file descriptor allowance: %v", err)
+	}
+	return int(raised / 2), nil // Leave half for networking and other stuff
 }

pkg/env.go

@@ -1,15 +1,45 @@
-package pkg
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package leveldb_ethdb_rpc
 
 const (
+	LOGRUS_LEVEL = "LOGRUS_LEVEL"
+	LOGRUS_FILE  = "LOGRUS_FILE"
+
 	IPC_ENABLED   = "IPC_ENABLED"
-	IPC_ENDPOINT  = "IPC_ENDPOINT"
+	IPC_ENDPOINT  = "IPC_PATH"
 	HTTP_ENABLED  = "HTTP_ENABLED"
-	HTTP_ENDPOINT = "HTTP_ENDPOINT"
+	HTTP_ENDPOINT = "HTTP_PATH"
 
 	LEVELDB_PATH         = "LEVELDB_PATH"
 	LEVELDB_CACHE_SIZE   = "LEVELDB_CACHE_SIZE"
-	LEVELDB_HANDLER_NUM  = "LEVELDB_HANDLER_NUM"
+	LEVELDB_ANCIENT_PATH = "LEVELDB_ANCIENT_PATH"
-	LEVELDB_FREEZER_PATH = "LEVELDB_FREEZER_PATH"
 	LEVELDB_NAMESPACE    = "LEVELDB_NAMESPACE"
-	LEVELDB_READ_ONLY    = "LEVELDB_READ_ONLY"
+
+	TOML_LOGRUS_LEVEL = "log.level"
+	TOML_LOGRUS_FILE  = "log.file"
+
+	TOML_IPC_ENABLED   = "leveldb.ipcEnabled"
+	TOML_IPC_ENDPOINT  = "leveldb.ipcPath"
+	TOML_HTTP_ENABLED  = "leveldb.httpEnabled"
+	TOML_HTTP_ENDPOINT = "leveldb.httpPath"
+
+	TOML_LEVELDB_PATH         = "leveldb.path"
+	TOML_LEVELDB_CACHE_SIZE   = "leveldb.cacheSize"
+	TOML_LEVELDB_ANCIENT_PATH = "leveldb.ancient"
+	TOML_LEVELDB_NAMESPACE    = "leveldb.namespace"
 )

pkg/service.go

@@ -14,16 +14,15 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package pkg
+package leveldb_ethdb_rpc
 
 import (
 	"sync"
 
-	log "github.com/sirupsen/logrus"
-
 	ethnode "github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rpc"
+	log "github.com/sirupsen/logrus"
 )
 
 // Server is the top level interface for exposing a remote RPC wrapper around levelDB ethdb.Database
@@ -42,7 +41,7 @@ type Service struct {
 }
 
 // NewServer creates a new Server using an underlying Service struct
-func NewServer(conf Config) (Server, error) {
+func NewServer(conf *Config) (Server, error) {
 	sap := new(Service)
 	sap.quitChan = make(chan struct{})
 	var err error