diff --git a/.gitignore b/.gitignore
index 9f11b75..4469811 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
.idea/
+leveldb-ethdb-rpc
diff --git a/cmd/root.go b/cmd/root.go
index 203d807..02b3359 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -20,8 +20,10 @@ import (
"fmt"
"os"
- "github.com/spf13/cobra"
+ leveldb_ethdb_rpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg"
+ log "github.com/sirupsen/logrus"
+ "github.com/spf13/cobra"
"github.com/spf13/viper"
)
@@ -29,59 +31,64 @@ var cfgFile string
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
- Use: "leveldb-ethdb-rpc",
- Short: "A brief description of your application",
- Long: `A longer description that spans multiple lines and likely contains
-examples and usage of using your application. For example:
-
-Cobra is a CLI library for Go that empowers applications.
-This application is a tool to generate the needed files
-to quickly create a Cobra application.`,
- // Uncomment the following line if your bare application
- // has an action associated with it:
- // Run: func(cmd *cobra.Command, args []string) { },
+ Use: "leveldb-ethdb-rpc",
+ PersistentPreRun: initFuncs,
}
// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
- cobra.CheckErr(rootCmd.Execute())
+ log.Info("----- Starting IPFS blockchain watcher -----")
+ if err := rootCmd.Execute(); err != nil {
+ log.Fatal(err)
+ }
}
-func init() {
- cobra.OnInitialize(initConfig)
+func initFuncs(cmd *cobra.Command, args []string) {
+ viper.BindEnv(leveldb_ethdb_rpc.TOML_LOGRUS_FILE, leveldb_ethdb_rpc.LOGRUS_FILE)
+ logfile := viper.GetString(leveldb_ethdb_rpc.TOML_LOGRUS_FILE)
+ if logfile != "" {
+ file, err := os.OpenFile(logfile,
+ os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+ if err == nil {
+ log.Infof("Directing output to %s", logfile)
+ log.SetOutput(file)
+ } else {
+ log.SetOutput(os.Stdout)
+ log.Info("Failed to log to file, using default stdout")
+ }
+ } else {
+ log.SetOutput(os.Stdout)
+ }
+ if err := logLevel(); err != nil {
+ log.Fatal("Could not set log level: ", err)
+ }
+}
- // Here you will define your flags and configuration settings.
- // Cobra supports persistent flags, which, if defined here,
- // will be global for your application.
-
- rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.leveldb-ethdb-rpc.yaml)")
-
- // Cobra also supports local flags, which will only run
- // when this action is called directly.
- rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+func logLevel() error {
+ viper.BindEnv(leveldb_ethdb_rpc.TOML_LOGRUS_LEVEL, leveldb_ethdb_rpc.LOGRUS_LEVEL)
+ lvl, err := log.ParseLevel(viper.GetString(leveldb_ethdb_rpc.TOML_LOGRUS_LEVEL))
+ if err != nil {
+ return err
+ }
+ log.SetLevel(lvl)
+ if lvl > log.InfoLevel {
+ log.SetReportCaller(true)
+ }
+ log.Info("Log level set to ", lvl.String())
+ return nil
}
// initConfig reads in config file and ENV variables if set.
func initConfig() {
if cfgFile != "" {
- // Use config file from the flag.
viper.SetConfigFile(cfgFile)
+ if err := viper.ReadInConfig(); err == nil {
+ log.Printf("Using config file: %s", viper.ConfigFileUsed())
+ } else {
+ log.Fatal(fmt.Sprintf("Couldn't read config file: %s", err.Error()))
+ }
} else {
- // Find home directory.
- home, err := os.UserHomeDir()
- cobra.CheckErr(err)
-
- // Search config in home directory with name ".leveldb-ethdb-rpc" (without extension).
- viper.AddConfigPath(home)
- viper.SetConfigType("yaml")
- viper.SetConfigName(".leveldb-ethdb-rpc")
- }
-
- viper.AutomaticEnv() // read in environment variables that match
-
- // If a config file is found, read it in.
- if err := viper.ReadInConfig(); err == nil {
- fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed())
+ log.Warn("No config file passed with --config flag")
}
}
diff --git a/cmd/serve.go b/cmd/serve.go
index 036d674..78ff416 100644
--- a/cmd/serve.go
+++ b/cmd/serve.go
@@ -24,8 +24,9 @@ import (
"github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
+ "github.com/spf13/viper"
- "github.com/vulcanize/leveldb-ethdb-rpc/pkg"
+ leveldb_ethdb_rpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg"
srpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg/rpc"
"github.com/vulcanize/leveldb-ethdb-rpc/version"
)
@@ -52,13 +53,13 @@ func serve() {
wg := new(sync.WaitGroup)
logWithCommand.Debug("loading server configuration variables")
- serverConfig, err := pkg.NewConfig()
+ serverConfig, err := leveldb_ethdb_rpc.NewConfig()
if err != nil {
logWithCommand.Fatal(err)
}
logWithCommand.Infof("server config: %+v", serverConfig)
logWithCommand.Debug("initializing new server service")
- server, err := pkg.NewServer(serverConfig)
+ server, err := leveldb_ethdb_rpc.NewServer(serverConfig)
if err != nil {
logWithCommand.Fatal(err)
}
@@ -76,7 +77,7 @@ func serve() {
wg.Wait()
}
-func startServers(server pkg.Server, settings pkg.Config) error {
+func startServers(server leveldb_ethdb_rpc.Server, settings *leveldb_ethdb_rpc.Config) error {
if settings.IPCEnabled {
logWithCommand.Info("starting up IPC server")
_, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
@@ -103,13 +104,25 @@ func startServers(server pkg.Server, settings pkg.Config) error {
func init() {
rootCmd.AddCommand(serveCmd)
- // Here you will define your flags and configuration settings.
+ // CLI flags
+ serveCmd.PersistentFlags().Bool("ipc-enabled", false, "turn on ipc server")
+ serveCmd.PersistentFlags().String("ipc-path", "", "ipc server endpoint")
+ serveCmd.PersistentFlags().Bool("http-enabled", true, "turn on http server; on by default")
+ serveCmd.PersistentFlags().String("http-path", "127.0.0.1:8500", "http server endpoint; default = 127.0.0.1:8545")
- // Cobra supports Persistent Flags which will work for this command
- // and all subcommands, e.g.:
- // serveCmd.PersistentFlags().String("foo", "", "A help for foo")
+ serveCmd.PersistentFlags().String("leveldb-path", "", "leveldb filesystem path")
+ serveCmd.PersistentFlags().Int("leveldb-cache-size", 0, "leveldb cache size")
+ serveCmd.PersistentFlags().String("leveldb-ancient-path", "", "filesystem path to freezer")
+ serveCmd.PersistentFlags().String("leveldb-namespace", "eth/db/chaindata/", "leveldb namespace")
- // Cobra supports local flags which will only run when this command
- // is called directly, e.g.:
- // serveCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+ // toml bindings
+ viper.BindPFlag(leveldb_ethdb_rpc.TOML_IPC_ENABLED, serveCmd.PersistentFlags().Lookup("ipc-enabled"))
+ viper.BindPFlag(leveldb_ethdb_rpc.TOML_IPC_ENDPOINT, serveCmd.PersistentFlags().Lookup("ipc-path"))
+ viper.BindPFlag(leveldb_ethdb_rpc.TOML_HTTP_ENABLED, serveCmd.PersistentFlags().Lookup("http-enabled"))
+ viper.BindPFlag(leveldb_ethdb_rpc.TOML_HTTP_ENDPOINT, serveCmd.PersistentFlags().Lookup("http-path"))
+
+ viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_PATH, serveCmd.PersistentFlags().Lookup("leveldb-path"))
+ viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_CACHE_SIZE, serveCmd.PersistentFlags().Lookup("leveldb-cache-size"))
+ viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_ANCIENT_PATH, serveCmd.PersistentFlags().Lookup("leveldb-ancient-path"))
+ viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_NAMESPACE, serveCmd.PersistentFlags().Lookup("leveldb-namespace"))
}
diff --git a/environments/example.toml b/environments/example.toml
new file mode 100644
index 0000000..fe3e8b4
--- /dev/null
+++ b/environments/example.toml
@@ -0,0 +1,13 @@
+[log]
+ level = "info" # $LOGRUS_LEVEL
+ file = "" # $LOGRUS_FILE
+
+[leveldb]
+ ipcEnabled = false # $IPC_ENABLED
+ ipcPath = "~/.vulcanize/vulcanize.ipc" # $IPC_PATH
+ httpEnabled = true # $HTTP_ENABLED
+ httpPath = "127.0.0.1:8082" # $HTTP_PATH
+ path = "/Users/user/Library/Ethereum/geth/chaindata" # $LEVELDB_PATH
+ ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # $LEVELDB_ANCIENT_PATH
+ cacheSize = 1024 # $LEVELDB_CACHE_SIZE
+ namespace = "eth/db/chaindata/" # $LEVELDB_NAMESPACE
diff --git a/pkg/api.go b/pkg/api.go
index 125084f..ae4841f 100644
--- a/pkg/api.go
+++ b/pkg/api.go
@@ -14,11 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-package pkg
+package leveldb_ethdb_rpc
import (
"context"
"errors"
+
+ "github.com/ethereum/go-ethereum/ethdb"
)
// APIName is the namespace used for the state diffing service API
@@ -55,8 +57,12 @@ func (s *PublicLevelDBAPI) Ancient(ctx context.Context, kind string, number uint
return s.b.Ancient(kind, number)
}
-func (s *PublicLevelDBAPI) ReadAncients(ctx context.Context, kind string, start, count, maxBytes uint64) ([][]byte, error) {
- return s.b.ReadAncients(kind, start, count, maxBytes)
+func (s *PublicLevelDBAPI) AncientRange(ctx context.Context, kind string, start, count, maxBytes uint64) ([][]byte, error) {
+ return s.b.AncientRange(kind, start, count, maxBytes)
+}
+
+func (s *PublicLevelDBAPI) ReadAncients(fn func(ethdb.AncientReader) error) error {
+ return s.b.ReadAncients(fn)
}
func (s *PublicLevelDBAPI) Ancients(ctx context.Context) (uint64, error) {
diff --git a/pkg/backend.go b/pkg/backend.go
index d08e822..78b872e 100644
--- a/pkg/backend.go
+++ b/pkg/backend.go
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-package pkg
+package leveldb_ethdb_rpc
import (
"github.com/ethereum/go-ethereum/core/rawdb"
@@ -25,12 +25,12 @@ import (
var _ ethdb.Database = &LevelDBBackend{}
// NewLevelDBBackend creates a new levelDB RPC server backend
-func NewLevelDBBackend(conf Config) (*LevelDBBackend, error) {
- db, err := leveldb.New(conf.File, conf.Cache, conf.Handles, conf.Namespace, conf.Readonly)
+func NewLevelDBBackend(conf *Config) (*LevelDBBackend, error) {
+ db, err := leveldb.New(conf.FilePath, conf.Cache, conf.Handles, conf.Namespace, true)
if err != nil {
return nil, err
}
- frdb, err := rawdb.NewDatabaseWithFreezer(db, conf.Freezer, conf.Namespace, conf.Readonly)
+ frdb, err := rawdb.NewDatabaseWithFreezer(db, conf.FreezerPath, conf.Namespace, true)
if err != nil {
db.Close()
return nil, err
@@ -62,8 +62,12 @@ func (s *LevelDBBackend) Ancient(kind string, number uint64) ([]byte, error) {
return s.ethDB.Ancient(kind, number)
}
-func (s *LevelDBBackend) ReadAncients(kind string, start, count, maxBytes uint64) ([][]byte, error) {
- return s.ethDB.ReadAncients(kind, start, count, maxBytes)
+func (s *LevelDBBackend) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
+ return s.ethDB.AncientRange(kind, start, count, maxBytes)
+}
+
+func (s *LevelDBBackend) ReadAncients(fn func(ethdb.AncientReader) error) error {
+ return s.ethDB.ReadAncients(fn)
}
func (s *LevelDBBackend) Ancients() (uint64, error) {
@@ -74,11 +78,11 @@ func (s *LevelDBBackend) AncientSize(kind string) (uint64, error) {
return s.ethDB.AncientSize(kind)
}
-func (s *LevelDBBackend) Put(key []byte, value []byte) error {
+func (s *LevelDBBackend) Put(_ []byte, _ []byte) error {
return errWriteNotAllowed
}
-func (s *LevelDBBackend) Delete(key []byte) error {
+func (s *LevelDBBackend) Delete(_ []byte) error {
return errWriteNotAllowed
}
@@ -98,7 +102,7 @@ func (s *LevelDBBackend) NewBatch() ethdb.Batch {
return nil
}
-func (s *LevelDBBackend) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
+func (s *LevelDBBackend) NewIterator(_ []byte, _ []byte) ethdb.Iterator {
return nil
}
@@ -106,7 +110,7 @@ func (s *LevelDBBackend) Stat(property string) (string, error) {
return s.ethDB.Stat(property)
}
-func (s *LevelDBBackend) Compact(start []byte, limit []byte) error {
+func (s *LevelDBBackend) Compact(_ []byte, _ []byte) error {
return errWriteNotAllowed
}
diff --git a/pkg/config.go b/pkg/config.go
index 1077239..532728f 100644
--- a/pkg/config.go
+++ b/pkg/config.go
@@ -14,7 +14,14 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-package pkg
+package leveldb_ethdb_rpc
+
+import (
+ "fmt"
+
+ "github.com/ethereum/go-ethereum/common/fdlimit"
+ "github.com/spf13/viper"
+)
// Config struct holds the configuration parameters for the levelDB RPC service
type Config struct {
@@ -23,15 +30,52 @@ type Config struct {
HTTPEnabled bool
HTTPEndpoint string
- File string
- Cache int
- Handles int
- Freezer string
- Namespace string
- Readonly bool
+ FilePath string
+ Cache int
+ Handles int
+ FreezerPath string
+ Namespace string
}
// NewConfig returns a new Config from viper parameters
-func NewConfig() (Config, error) {
+func NewConfig() (*Config, error) {
+ viper.BindEnv(TOML_IPC_ENABLED, IPC_ENABLED)
+ viper.BindEnv(TOML_IPC_ENDPOINT, IPC_ENDPOINT)
+ viper.BindEnv(TOML_HTTP_ENABLED, HTTP_ENABLED)
+ viper.BindEnv(TOML_HTTP_ENDPOINT, HTTP_ENDPOINT)
+ viper.BindEnv(TOML_LEVELDB_PATH, LEVELDB_PATH)
+ viper.BindEnv(TOML_LEVELDB_CACHE_SIZE, LEVELDB_CACHE_SIZE)
+ viper.BindEnv(TOML_LEVELDB_ANCIENT_PATH, LEVELDB_ANCIENT_PATH)
+ viper.BindEnv(TOML_LEVELDB_NAMESPACE, LEVELDB_NAMESPACE)
+
+ numHandles, err := MakeDatabaseHandles()
+ if err != nil {
+ return nil, err
+ }
+ return &Config{
+ IPCEnabled: viper.GetBool(TOML_IPC_ENABLED),
+ IPCEndpoint: viper.GetString(TOML_IPC_ENDPOINT),
+ HTTPEnabled: viper.GetBool(TOML_HTTP_ENABLED),
+ HTTPEndpoint: viper.GetString(TOML_HTTP_ENDPOINT),
+ FilePath: viper.GetString(TOML_LEVELDB_PATH),
+ Cache: viper.GetInt(TOML_LEVELDB_CACHE_SIZE),
+ Handles: numHandles,
+ FreezerPath: viper.GetString(TOML_LEVELDB_ANCIENT_PATH),
+ Namespace: viper.GetString(TOML_LEVELDB_NAMESPACE),
+ }, nil
+}
+
+// MakeDatabaseHandles raises out the number of allowed file handles per process
+// for Geth and returns half of the allowance to assign to the database.
+func MakeDatabaseHandles() (int, error) {
+ limit, err := fdlimit.Maximum()
+ if err != nil {
+ return 0, fmt.Errorf("failed to retrieve file descriptor allowance: %v", err)
+ }
+ raised, err := fdlimit.Raise(uint64(limit))
+ if err != nil {
+ return 0, fmt.Errorf("failed to raise file descriptor allowance: %v", err)
+ }
+ return int(raised / 2), nil // Leave half for networking and other stuff
}
diff --git a/pkg/env.go b/pkg/env.go
index 2c58dd4..e3236df 100644
--- a/pkg/env.go
+++ b/pkg/env.go
@@ -1,15 +1,45 @@
-package pkg
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package leveldb_ethdb_rpc
const (
+ LOGRUS_LEVEL = "LOGRUS_LEVEL"
+ LOGRUS_FILE = "LOGRUS_FILE"
+
IPC_ENABLED = "IPC_ENABLED"
- IPC_ENDPOINT = "IPC_ENDPOINT"
+ IPC_ENDPOINT = "IPC_PATH"
HTTP_ENABLED = "HTTP_ENABLED"
- HTTP_ENDPOINT = "HTTP_ENDPOINT"
+ HTTP_ENDPOINT = "HTTP_PATH"
LEVELDB_PATH = "LEVELDB_PATH"
LEVELDB_CACHE_SIZE = "LEVELDB_CACHE_SIZE"
- LEVELDB_HANDLER_NUM = "LEVELDB_HANDLER_NUM"
- LEVELDB_FREEZER_PATH = "LEVELDB_FREEZER_PATH"
+ LEVELDB_ANCIENT_PATH = "LEVELDB_ANCIENT_PATH"
LEVELDB_NAMESPACE = "LEVELDB_NAMESPACE"
- LEVELDB_READ_ONLY = "LEVELDB_READ_ONLY"
+
+ TOML_LOGRUS_LEVEL = "log.level"
+ TOML_LOGRUS_FILE = "log.file"
+
+ TOML_IPC_ENABLED = "leveldb.ipcEnabled"
+ TOML_IPC_ENDPOINT = "leveldb.ipcPath"
+ TOML_HTTP_ENABLED = "leveldb.httpEnabled"
+ TOML_HTTP_ENDPOINT = "leveldb.httpPath"
+
+ TOML_LEVELDB_PATH = "leveldb.path"
+ TOML_LEVELDB_CACHE_SIZE = "leveldb.cacheSize"
+ TOML_LEVELDB_ANCIENT_PATH = "leveldb.ancient"
+ TOML_LEVELDB_NAMESPACE = "leveldb.namespace"
)
diff --git a/pkg/service.go b/pkg/service.go
index 65ff156..0f3b61f 100644
--- a/pkg/service.go
+++ b/pkg/service.go
@@ -14,16 +14,15 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-package pkg
+package leveldb_ethdb_rpc
import (
"sync"
- log "github.com/sirupsen/logrus"
-
ethnode "github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rpc"
+ log "github.com/sirupsen/logrus"
)
// Server is the top level interface for exposing a remote RPC wrapper around levelDB ethdb.Database
@@ -42,7 +41,7 @@ type Service struct {
}
// NewServer creates a new Server using an underlying Service struct
-func NewServer(conf Config) (Server, error) {
+func NewServer(conf *Config) (Server, error) {
sap := new(Service)
sap.quitChan = make(chan struct{})
var err error