cerc-io/leveldb-ethdb-rpc
16
0
Fork 0
Code Issues 4 Pull Requests Packages Projects Releases 12 Wiki Activity

fin/fix CLI and ENV integration #1

Merged
roysc merged 1 commits from working_branch into main 2021-12-31 18:59:32 +00:00
Conversation 0 Commits 1 Files Changed 9 +199 -82
9 changed files with 199 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit fa23e1164e - Show all commits

1
.gitignore vendored
View File

@ -1 +1,2 @@
.idea/
leveldb-ethdb-rpc

87
cmd/root.go
View File

@ -20,8 +20,10 @@ import (
"fmt"
"os"
"github.com/spf13/cobra"
leveldb_ethdb_rpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
@ -29,59 +31,64 @@ var cfgFile string
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "leveldb-ethdb-rpc",
Short: "A brief description of your application",
Long: `A longer description that spans multiple lines and likely contains
examples and usage of using your application. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
// Uncomment the following line if your bare application
// has an action associated with it:
// Run: func(cmd *cobra.Command, args []string) { },
Use: "leveldb-ethdb-rpc",
PersistentPreRun: initFuncs,
}
// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
cobra.CheckErr(rootCmd.Execute())
log.Info("----- Starting IPFS blockchain watcher -----")
if err := rootCmd.Execute(); err != nil {
log.Fatal(err)
}
}
func init() {
cobra.OnInitialize(initConfig)
func initFuncs(cmd *cobra.Command, args []string) {
viper.BindEnv(leveldb_ethdb_rpc.TOML_LOGRUS_FILE, leveldb_ethdb_rpc.LOGRUS_FILE)
logfile := viper.GetString(leveldb_ethdb_rpc.TOML_LOGRUS_FILE)
if logfile != "" {
file, err := os.OpenFile(logfile,
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err == nil {
log.Infof("Directing output to %s", logfile)
log.SetOutput(file)
} else {
log.SetOutput(os.Stdout)
log.Info("Failed to log to file, using default stdout")
}
} else {
log.SetOutput(os.Stdout)
}
if err := logLevel(); err != nil {
log.Fatal("Could not set log level: ", err)
}
}
// Here you will define your flags and configuration settings.
// Cobra supports persistent flags, which, if defined here,
// will be global for your application.
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.leveldb-ethdb-rpc.yaml)")
// Cobra also supports local flags, which will only run
// when this action is called directly.
rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
func logLevel() error {
viper.BindEnv(leveldb_ethdb_rpc.TOML_LOGRUS_LEVEL, leveldb_ethdb_rpc.LOGRUS_LEVEL)
lvl, err := log.ParseLevel(viper.GetString(leveldb_ethdb_rpc.TOML_LOGRUS_LEVEL))
if err != nil {
return err
}
log.SetLevel(lvl)
if lvl > log.InfoLevel {
log.SetReportCaller(true)
}
log.Info("Log level set to ", lvl.String())
return nil
}
// initConfig reads in config file and ENV variables if set.
func initConfig() {
if cfgFile != "" {
// Use config file from the flag.
viper.SetConfigFile(cfgFile)
if err := viper.ReadInConfig(); err == nil {
log.Printf("Using config file: %s", viper.ConfigFileUsed())
} else {
log.Fatal(fmt.Sprintf("Couldn't read config file: %s", err.Error()))
}
} else {
// Find home directory.
home, err := os.UserHomeDir()
cobra.CheckErr(err)
// Search config in home directory with name ".leveldb-ethdb-rpc" (without extension).
viper.AddConfigPath(home)
viper.SetConfigType("yaml")
viper.SetConfigName(".leveldb-ethdb-rpc")
}
viper.AutomaticEnv() // read in environment variables that match
// If a config file is found, read it in.
if err := viper.ReadInConfig(); err == nil {
fmt.Fprintln(os.Stderr, "Using config file:", viper.ConfigFileUsed())
log.Warn("No config file passed with --config flag")
}
}

35
cmd/serve.go
View File

@ -24,8 +24,9 @@ import (
"github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulcanize/leveldb-ethdb-rpc/pkg"
leveldb_ethdb_rpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg"
srpc "github.com/vulcanize/leveldb-ethdb-rpc/pkg/rpc"
"github.com/vulcanize/leveldb-ethdb-rpc/version"
)
@ -52,13 +53,13 @@ func serve() {
wg := new(sync.WaitGroup)
logWithCommand.Debug("loading server configuration variables")
serverConfig, err := pkg.NewConfig()
serverConfig, err := leveldb_ethdb_rpc.NewConfig()
if err != nil {
logWithCommand.Fatal(err)
}
logWithCommand.Infof("server config: %+v", serverConfig)
logWithCommand.Debug("initializing new server service")
server, err := pkg.NewServer(serverConfig)
server, err := leveldb_ethdb_rpc.NewServer(serverConfig)
if err != nil {
logWithCommand.Fatal(err)
}
@ -76,7 +77,7 @@ func serve() {
wg.Wait()
}
func startServers(server pkg.Server, settings pkg.Config) error {
func startServers(server leveldb_ethdb_rpc.Server, settings *leveldb_ethdb_rpc.Config) error {
if settings.IPCEnabled {
logWithCommand.Info("starting up IPC server")
_, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
@ -103,13 +104,25 @@ func startServers(server pkg.Server, settings pkg.Config) error {
func init() {
rootCmd.AddCommand(serveCmd)
// Here you will define your flags and configuration settings.
// CLI flags
serveCmd.PersistentFlags().Bool("ipc-enabled", false, "turn on ipc server")
serveCmd.PersistentFlags().String("ipc-path", "", "ipc server endpoint")
serveCmd.PersistentFlags().Bool("http-enabled", true, "turn on http server; on by default")
serveCmd.PersistentFlags().String("http-path", "127.0.0.1:8500", "http server endpoint; default = 127.0.0.1:8545")
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// serveCmd.PersistentFlags().String("foo", "", "A help for foo")
serveCmd.PersistentFlags().String("leveldb-path", "", "leveldb filesystem path")
serveCmd.PersistentFlags().Int("leveldb-cache-size", 0, "leveldb cache size")
serveCmd.PersistentFlags().String("leveldb-ancient-path", "", "filesystem path to freezer")
serveCmd.PersistentFlags().String("leveldb-namespace", "eth/db/chaindata/", "leveldb namespace")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// serveCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
// toml bindings
viper.BindPFlag(leveldb_ethdb_rpc.TOML_IPC_ENABLED, serveCmd.PersistentFlags().Lookup("ipc-enabled"))
viper.BindPFlag(leveldb_ethdb_rpc.TOML_IPC_ENDPOINT, serveCmd.PersistentFlags().Lookup("ipc-path"))
viper.BindPFlag(leveldb_ethdb_rpc.TOML_HTTP_ENABLED, serveCmd.PersistentFlags().Lookup("http-enabled"))
viper.BindPFlag(leveldb_ethdb_rpc.TOML_HTTP_ENDPOINT, serveCmd.PersistentFlags().Lookup("http-path"))
viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_PATH, serveCmd.PersistentFlags().Lookup("leveldb-path"))
viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_CACHE_SIZE, serveCmd.PersistentFlags().Lookup("leveldb-cache-size"))
viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_ANCIENT_PATH, serveCmd.PersistentFlags().Lookup("leveldb-ancient-path"))
viper.BindPFlag(leveldb_ethdb_rpc.TOML_LEVELDB_NAMESPACE, serveCmd.PersistentFlags().Lookup("leveldb-namespace"))
}

13
environments/example.toml Normal file
View File

@ -0,0 +1,13 @@
[log]
level = "info" # $LOGRUS_LEVEL
file = "" # $LOGRUS_FILE
[leveldb]
ipcEnabled = false # $IPC_ENABLED
ipcPath = "~/.vulcanize/vulcanize.ipc" # $IPC_PATH
httpEnabled = true # $HTTP_ENABLED
httpPath = "127.0.0.1:8082" # $HTTP_PATH
path = "/Users/user/Library/Ethereum/geth/chaindata" # $LEVELDB_PATH
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # $LEVELDB_ANCIENT_PATH
cacheSize = 1024 # $LEVELDB_CACHE_SIZE
namespace = "eth/db/chaindata/" # $LEVELDB_NAMESPACE

12
pkg/api.go
View File

@ -14,11 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pkg
package leveldb_ethdb_rpc
import (
"context"
"errors"
"github.com/ethereum/go-ethereum/ethdb"
)
// APIName is the namespace used for the state diffing service API
@ -55,8 +57,12 @@ func (s *PublicLevelDBAPI) Ancient(ctx context.Context, kind string, number uint
return s.b.Ancient(kind, number)
}
func (s *PublicLevelDBAPI) ReadAncients(ctx context.Context, kind string, start, count, maxBytes uint64) ([][]byte, error) {
return s.b.ReadAncients(kind, start, count, maxBytes)
func (s *PublicLevelDBAPI) AncientRange(ctx context.Context, kind string, start, count, maxBytes uint64) ([][]byte, error) {
return s.b.AncientRange(kind, start, count, maxBytes)
}
func (s *PublicLevelDBAPI) ReadAncients(fn func(ethdb.AncientReader) error) error {
return s.b.ReadAncients(fn)
}
func (s *PublicLevelDBAPI) Ancients(ctx context.Context) (uint64, error) {

24
pkg/backend.go
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pkg
package leveldb_ethdb_rpc
import (
"github.com/ethereum/go-ethereum/core/rawdb"
@ -25,12 +25,12 @@ import (
var _ ethdb.Database = &LevelDBBackend{}
// NewLevelDBBackend creates a new levelDB RPC server backend
func NewLevelDBBackend(conf Config) (*LevelDBBackend, error) {
db, err := leveldb.New(conf.File, conf.Cache, conf.Handles, conf.Namespace, conf.Readonly)
func NewLevelDBBackend(conf *Config) (*LevelDBBackend, error) {
db, err := leveldb.New(conf.FilePath, conf.Cache, conf.Handles, conf.Namespace, true)
if err != nil {
return nil, err
}
frdb, err := rawdb.NewDatabaseWithFreezer(db, conf.Freezer, conf.Namespace, conf.Readonly)
frdb, err := rawdb.NewDatabaseWithFreezer(db, conf.FreezerPath, conf.Namespace, true)
if err != nil {
db.Close()
return nil, err
@ -62,8 +62,12 @@ func (s *LevelDBBackend) Ancient(kind string, number uint64) ([]byte, error) {
return s.ethDB.Ancient(kind, number)
}
func (s *LevelDBBackend) ReadAncients(kind string, start, count, maxBytes uint64) ([][]byte, error) {
return s.ethDB.ReadAncients(kind, start, count, maxBytes)
func (s *LevelDBBackend) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
return s.ethDB.AncientRange(kind, start, count, maxBytes)
}
func (s *LevelDBBackend) ReadAncients(fn func(ethdb.AncientReader) error) error {
return s.ethDB.ReadAncients(fn)
}
func (s *LevelDBBackend) Ancients() (uint64, error) {
@ -74,11 +78,11 @@ func (s *LevelDBBackend) AncientSize(kind string) (uint64, error) {
return s.ethDB.AncientSize(kind)
}
func (s *LevelDBBackend) Put(key []byte, value []byte) error {
func (s *LevelDBBackend) Put(_ []byte, _ []byte) error {
return errWriteNotAllowed
}
func (s *LevelDBBackend) Delete(key []byte) error {
func (s *LevelDBBackend) Delete(_ []byte) error {
return errWriteNotAllowed
}
@ -98,7 +102,7 @@ func (s *LevelDBBackend) NewBatch() ethdb.Batch {
return nil
}
func (s *LevelDBBackend) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
func (s *LevelDBBackend) NewIterator(_ []byte, _ []byte) ethdb.Iterator {
return nil
}
@ -106,7 +110,7 @@ func (s *LevelDBBackend) Stat(property string) (string, error) {
return s.ethDB.Stat(property)
}
func (s *LevelDBBackend) Compact(start []byte, limit []byte) error {
func (s *LevelDBBackend) Compact(_ []byte, _ []byte) error {
return errWriteNotAllowed
}

60
pkg/config.go
View File

@ -14,7 +14,14 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pkg
package leveldb_ethdb_rpc
import (
"fmt"
"github.com/ethereum/go-ethereum/common/fdlimit"
"github.com/spf13/viper"
)
// Config struct holds the configuration parameters for the levelDB RPC service
type Config struct {
@ -23,15 +30,52 @@ type Config struct {
HTTPEnabled bool
HTTPEndpoint string
File string
Cache int
Handles int
Freezer string
Namespace string
Readonly bool
FilePath string
Cache int
Handles int
FreezerPath string
Namespace string
}
// NewConfig returns a new Config from viper parameters
func NewConfig() (Config, error) {
func NewConfig() (*Config, error) {
viper.BindEnv(TOML_IPC_ENABLED, IPC_ENABLED)
viper.BindEnv(TOML_IPC_ENDPOINT, IPC_ENDPOINT)
viper.BindEnv(TOML_HTTP_ENABLED, HTTP_ENABLED)
viper.BindEnv(TOML_HTTP_ENDPOINT, HTTP_ENDPOINT)
viper.BindEnv(TOML_LEVELDB_PATH, LEVELDB_PATH)
viper.BindEnv(TOML_LEVELDB_CACHE_SIZE, LEVELDB_CACHE_SIZE)
viper.BindEnv(TOML_LEVELDB_ANCIENT_PATH, LEVELDB_ANCIENT_PATH)
viper.BindEnv(TOML_LEVELDB_NAMESPACE, LEVELDB_NAMESPACE)
numHandles, err := MakeDatabaseHandles()
if err != nil {
return nil, err
}
return &Config{
IPCEnabled: viper.GetBool(TOML_IPC_ENABLED),
IPCEndpoint: viper.GetString(TOML_IPC_ENDPOINT),
HTTPEnabled: viper.GetBool(TOML_HTTP_ENABLED),
HTTPEndpoint: viper.GetString(TOML_HTTP_ENDPOINT),
FilePath: viper.GetString(TOML_LEVELDB_PATH),
Cache: viper.GetInt(TOML_LEVELDB_CACHE_SIZE),
Handles: numHandles,
FreezerPath: viper.GetString(TOML_LEVELDB_ANCIENT_PATH),
Namespace: viper.GetString(TOML_LEVELDB_NAMESPACE),
}, nil
}
// MakeDatabaseHandles raises out the number of allowed file handles per process
// for Geth and returns half of the allowance to assign to the database.
func MakeDatabaseHandles() (int, error) {
limit, err := fdlimit.Maximum()
if err != nil {
return 0, fmt.Errorf("failed to retrieve file descriptor allowance: %v", err)
}
raised, err := fdlimit.Raise(uint64(limit))
if err != nil {
return 0, fmt.Errorf("failed to raise file descriptor allowance: %v", err)
}
return int(raised / 2), nil // Leave half for networking and other stuff
}

42
pkg/env.go
View File

@ -1,15 +1,45 @@
package pkg
// VulcanizeDB
// Copyright © 2022 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package leveldb_ethdb_rpc
const (
LOGRUS_LEVEL = "LOGRUS_LEVEL"
LOGRUS_FILE = "LOGRUS_FILE"
IPC_ENABLED = "IPC_ENABLED"
IPC_ENDPOINT = "IPC_ENDPOINT"
IPC_ENDPOINT = "IPC_PATH"
HTTP_ENABLED = "HTTP_ENABLED"
HTTP_ENDPOINT = "HTTP_ENDPOINT"
HTTP_ENDPOINT = "HTTP_PATH"
LEVELDB_PATH = "LEVELDB_PATH"
LEVELDB_CACHE_SIZE = "LEVELDB_CACHE_SIZE"
LEVELDB_HANDLER_NUM = "LEVELDB_HANDLER_NUM"
LEVELDB_FREEZER_PATH = "LEVELDB_FREEZER_PATH"
LEVELDB_ANCIENT_PATH = "LEVELDB_ANCIENT_PATH"
LEVELDB_NAMESPACE = "LEVELDB_NAMESPACE"
LEVELDB_READ_ONLY = "LEVELDB_READ_ONLY"
TOML_LOGRUS_LEVEL = "log.level"
TOML_LOGRUS_FILE = "log.file"
TOML_IPC_ENABLED = "leveldb.ipcEnabled"
TOML_IPC_ENDPOINT = "leveldb.ipcPath"
TOML_HTTP_ENABLED = "leveldb.httpEnabled"
TOML_HTTP_ENDPOINT = "leveldb.httpPath"
TOML_LEVELDB_PATH = "leveldb.path"
TOML_LEVELDB_CACHE_SIZE = "leveldb.cacheSize"
TOML_LEVELDB_ANCIENT_PATH = "leveldb.ancient"
TOML_LEVELDB_NAMESPACE = "leveldb.namespace"
)

7
pkg/service.go
View File

@ -14,16 +14,15 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package pkg
package leveldb_ethdb_rpc
import (
"sync"
log "github.com/sirupsen/logrus"
ethnode "github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rpc"
log "github.com/sirupsen/logrus"
)
// Server is the top level interface for exposing a remote RPC wrapper around levelDB ethdb.Database
@ -42,7 +41,7 @@ type Service struct {
}
// NewServer creates a new Server using an underlying Service struct
func NewServer(conf Config) (Server, error) {
func NewServer(conf *Config) (Server, error) {
sap := new(Service)
sap.quitChan = make(chan struct{})
var err error