add env bindings

cmd/root.go

@@ -21,6 +21,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -46,7 +48,7 @@ func Execute() {
 }
 
 func initFuncs(cmd *cobra.Command, args []string) {
-	logfile := viper.GetString("logfile")
+	logfile := viper.GetString("log.file")
 	if logfile != "" {
 		file, err := os.OpenFile(logfile,
 			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
@@ -84,7 +86,7 @@ func init() {
 	viper.AutomaticEnv()
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
-	rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
+	rootCmd.PersistentFlags().String("log-file", "", "file path for logging")
 	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
 	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
 	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
@@ -92,13 +94,13 @@ func init() {
 	rootCmd.PersistentFlags().String("database-password", "", "database password")
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
 
-	viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
+	viper.BindPFlag(snapshot.LOGRUS_FILE_TOML, rootCmd.PersistentFlags().Lookup("log-file"))
-	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
+	viper.BindPFlag(snapshot.DATABASE_NAME_TOML, rootCmd.PersistentFlags().Lookup("database-name"))
-	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
+	viper.BindPFlag(snapshot.DATABASE_PORT_TOML, rootCmd.PersistentFlags().Lookup("database-port"))
-	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
+	viper.BindPFlag(snapshot.DATABASE_HOSTNAME_TOML, rootCmd.PersistentFlags().Lookup("database-hostname"))
-	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
+	viper.BindPFlag(snapshot.DATABASE_USER_TOML, rootCmd.PersistentFlags().Lookup("database-user"))
-	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
+	viper.BindPFlag(snapshot.DATABASE_PASSWORD_TOML, rootCmd.PersistentFlags().Lookup("database-password"))
-	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
+	viper.BindPFlag(snapshot.LOGRUS_LEVEL_TOML, rootCmd.PersistentFlags().Lookup("log-level"))
 }
 
 func initConfig() {

cmd/stateSnapshot.go

@@ -94,11 +94,11 @@ func init() {
 	stateSnapshotCmd.PersistentFlags().String("snapshot-mode", "postgres", "output mode for snapshot ('file' or 'postgres')")
 	stateSnapshotCmd.PersistentFlags().String("output-dir", "", "directory for writing ouput to while operating in 'file' mode")
 
-	viper.BindPFlag("leveldb.path", stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path"))
+	viper.BindPFlag(snapshot.LVL_DB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path"))
-	viper.BindPFlag("leveldb.ancient", stateSnapshotCmd.PersistentFlags().Lookup("ancient-path"))
+	viper.BindPFlag(snapshot.ANCIENT_DB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup("ancient-path"))
-	viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height"))
+	viper.BindPFlag(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML, stateSnapshotCmd.PersistentFlags().Lookup("block-height"))
-	viper.BindPFlag("snapshot.workers", stateSnapshotCmd.PersistentFlags().Lookup("workers"))
+	viper.BindPFlag(snapshot.SNAPSHOT_WORKERS_TOML, stateSnapshotCmd.PersistentFlags().Lookup("workers"))
-	viper.BindPFlag("snapshot.recoveryFile", stateSnapshotCmd.PersistentFlags().Lookup("recovery-file"))
+	viper.BindPFlag(snapshot.SNAPSHOT_RECOVERY_FILE_TOML, stateSnapshotCmd.PersistentFlags().Lookup("recovery-file"))
-	viper.BindPFlag("snapshot.mode", stateSnapshotCmd.PersistentFlags().Lookup("snapshot-mode"))
+	viper.BindPFlag(snapshot.SNAPSHOT_MODE_TOML, stateSnapshotCmd.PersistentFlags().Lookup("snapshot-mode"))
-	viper.BindPFlag("file.outputDir", stateSnapshotCmd.PersistentFlags().Lookup("output-dir"))
+	viper.BindPFlag(snapshot.FILE_OUTPUT_DIR_TOML, stateSnapshotCmd.PersistentFlags().Lookup("output-dir"))
 }

environments/example.toml

@@ -5,8 +5,12 @@
     user     = "postgres"
 
 [leveldb]
-    path = "/Users/user/Library/Ethereum/geth/chaindata"
+    path = "/Users/iannorden/Library/Ethereum/geth/chaindata"
-    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
+    ancient = "/Users/iannorden/Library/Ethereum/geth/chaindata/ancient"
+
+[log]
+    level = "info"
+    file = "log_file"
 
 [snapshot]
     mode = "file"

pkg/snapshot/config.go

@@ -26,15 +26,6 @@ import (
 	"github.com/spf13/viper"
 )
 
-const (
-	ANCIENT_DB_PATH   = "ANCIENT_DB_PATH"
-	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
-	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
-	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
-	ETH_NODE_ID       = "ETH_NODE_ID"
-	LVL_DB_PATH       = "LVL_DB_PATH"
-)
-
 // SnapshotMode specifies the snapshot data output method
 type SnapshotMode string
 
@@ -80,24 +71,25 @@ func NewConfig(mode SnapshotMode) (*Config, error) {
 
 // Init Initialises config
 func (c *Config) Init(mode SnapshotMode) error {
-	viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
+	viper.BindEnv(ETH_NODE_ID_TOML, ETH_NODE_ID)
-	viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
+	viper.BindEnv(ETH_CLIENT_NAME_TOML, ETH_CLIENT_NAME)
-	viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
+	viper.BindEnv(ETH_GENESIS_BLOCK_TOML, ETH_GENESIS_BLOCK)
-	viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
+	viper.BindEnv(ETH_NETWORK_ID_TOML, ETH_NETWORK_ID)
+	viper.BindEnv(ETH_CHAIN_ID_TOML, ETH_CHAIN_ID)
 
 	c.Eth.NodeInfo = ethNode.Info{
-		ID:           viper.GetString("ethereum.nodeID"),
+		ID:           viper.GetString(ETH_NODE_ID_TOML),
-		ClientName:   viper.GetString("ethereum.clientName"),
+		ClientName:   viper.GetString(ETH_CLIENT_NAME_TOML),
-		GenesisBlock: viper.GetString("ethereum.genesisBlock"),
+		GenesisBlock: viper.GetString(ETH_GENESIS_BLOCK_TOML),
-		NetworkID:    viper.GetString("ethereum.networkID"),
+		NetworkID:    viper.GetString(ETH_NETWORK_ID_TOML),
-		ChainID:      viper.GetUint64("ethereum.chainID"),
+		ChainID:      viper.GetUint64(ETH_CHAIN_ID_TOML),
 	}
 
-	viper.BindEnv("leveldb.ancient", ANCIENT_DB_PATH)
+	viper.BindEnv(ANCIENT_DB_PATH_TOML, ANCIENT_DB_PATH)
-	viper.BindEnv("leveldb.path", LVL_DB_PATH)
+	viper.BindEnv(LVL_DB_PATH_TOML, LVL_DB_PATH)
 
-	c.Eth.AncientDBPath = viper.GetString("leveldb.ancient")
+	c.Eth.AncientDBPath = viper.GetString(ANCIENT_DB_PATH_TOML)
-	c.Eth.LevelDBPath = viper.GetString("leveldb.path")
+	c.Eth.LevelDBPath = viper.GetString(LVL_DB_PATH_TOML)
 
 	switch mode {
 	case FileSnapshot:
@@ -111,34 +103,34 @@ func (c *Config) Init(mode SnapshotMode) error {
 }
 
 func (c *DBConfig) Init() {
-	viper.BindEnv("database.name", "DATABASE_NAME")
+	viper.BindEnv(DATABASE_NAME_TOML, DATABASE_NAME)
-	viper.BindEnv("database.hostname", "DATABASE_HOSTNAME")
+	viper.BindEnv(DATABASE_HOSTNAME_TOML, DATABASE_HOSTNAME)
-	viper.BindEnv("database.port", "DATABASE_PORT")
+	viper.BindEnv(DATABASE_PORT_TOML, DATABASE_PORT)
-	viper.BindEnv("database.user", "DATABASE_USER")
+	viper.BindEnv(DATABASE_USER_TOML, DATABASE_USER)
-	viper.BindEnv("database.password", "DATABASE_PASSWORD")
+	viper.BindEnv(DATABASE_PASSWORD_TOML, DATABASE_PASSWORD)
-	viper.BindEnv("database.maxIdle", "DATABASE_MAX_IDLE_CONNECTIONS")
+	viper.BindEnv(DATABASE_MAX_IDLE_CONNECTIONS_TOML, DATABASE_MAX_IDLE_CONNECTIONS)
-	viper.BindEnv("database.maxOpen", "DATABASE_MAX_OPEN_CONNECTIONS")
+	viper.BindEnv(DATABASE_MAX_OPEN_CONNECTIONS_TOML, DATABASE_MAX_OPEN_CONNECTIONS)
-	viper.BindEnv("database.maxLifetime", "DATABASE_MAX_CONN_LIFETIME")
+	viper.BindEnv(DATABASE_MAX_CONN_LIFETIME_TOML, DATABASE_MAX_CONN_LIFETIME)
 
 	dbParams := postgres.Config{}
 	// DB params
-	dbParams.DatabaseName = viper.GetString("database.name")
+	dbParams.DatabaseName = viper.GetString(DATABASE_NAME_TOML)
-	dbParams.Hostname = viper.GetString("database.hostname")
+	dbParams.Hostname = viper.GetString(DATABASE_HOSTNAME_TOML)
-	dbParams.Port = viper.GetInt("database.port")
+	dbParams.Port = viper.GetInt(DATABASE_PORT_TOML)
-	dbParams.Username = viper.GetString("database.user")
+	dbParams.Username = viper.GetString(DATABASE_USER_TOML)
-	dbParams.Password = viper.GetString("database.password")
+	dbParams.Password = viper.GetString(DATABASE_PASSWORD_TOML)
 	// Connection config
-	dbParams.MaxIdle = viper.GetInt("database.maxIdle")
+	dbParams.MaxIdle = viper.GetInt(DATABASE_MAX_IDLE_CONNECTIONS_TOML)
-	dbParams.MaxConns = viper.GetInt("database.maxOpen")
+	dbParams.MaxConns = viper.GetInt(DATABASE_MAX_OPEN_CONNECTIONS_TOML)
-	dbParams.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime")) * time.Second
+	dbParams.MaxConnLifetime = time.Duration(viper.GetInt(DATABASE_MAX_CONN_LIFETIME_TOML)) * time.Second
 
 	c.ConnConfig = dbParams
 	c.URI = dbParams.DbConnectionString()
 }
 
 func (c *FileConfig) Init() error {
-	viper.BindEnv("file.outputDir", "FILE_OUTPUT_DIR")
+	viper.BindEnv(FILE_OUTPUT_DIR_TOML, FILE_OUTPUT_DIR)
-	c.OutputDir = viper.GetString("file.outputDir")
+	c.OutputDir = viper.GetString(FILE_OUTPUT_DIR_TOML)
 	if c.OutputDir == "" {
 		logrus.Infof("no output directory set, using default: %s", defaultOutputDir)
 		c.OutputDir = defaultOutputDir

pkg/snapshot/env.go

@@ -0,0 +1,82 @@
+// Copyright © 2020 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package snapshot
+
+/*
+	viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height"))
+	viper.BindPFlag("snapshot.workers", stateSnapshotCmd.PersistentFlags().Lookup("workers"))
+	viper.BindPFlag("snapshot.recoveryFile", stateSnapshotCmd.PersistentFlags().Lookup("recovery-file"))
+	viper.BindPFlag("snapshot.mode", stateSnapshotCmd.PersistentFlags().Lookup("snapshot-mode"))
+*/
+const (
+	SNAPSHOT_BLOCK_HEIGHT  = "SNAPSHOT_BLOCK_HEIGHT"
+	SNAPSHOT_WORKERS       = "SNAPSHOT_WORKERS"
+	SNAPSHOT_RECOVERY_FILE = "SNAPSHOT_RECOVERY_FILE"
+	SNAPSHOT_MODE          = "SNAPSHOT_MODE"
+
+	LOGRUS_LEVEL = "LOGRUS_LEVEL"
+	LOGRUS_FILE  = "LOGRUS_FILE"
+
+	FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR"
+
+	ANCIENT_DB_PATH = "ANCIENT_DB_PATH"
+	LVL_DB_PATH     = "LVL_DB_PATH"
+
+	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
+	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
+	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
+	ETH_NODE_ID       = "ETH_NODE_ID"
+	ETH_CHAIN_ID      = "ETH_CHAIN_ID"
+
+	DATABASE_NAME                 = "DATABASE_NAME"
+	DATABASE_HOSTNAME             = "DATABASE_HOSTNAME"
+	DATABASE_PORT                 = "DATABASE_PORT"
+	DATABASE_USER                 = "DATABASE_USER"
+	DATABASE_PASSWORD             = "DATABASE_PASSWORD"
+	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
+	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
+	DATABASE_MAX_CONN_LIFETIME    = "DATABASE_MAX_CONN_LIFETIME"
+)
+
+const (
+	SNAPSHOT_BLOCK_HEIGHT_TOML  = "snapshot.blockHeight"
+	SNAPSHOT_WORKERS_TOML       = "snapshot.workers"
+	SNAPSHOT_RECOVERY_FILE_TOML = "snapshot.recoveryFile"
+	SNAPSHOT_MODE_TOML          = "snapshot.mode"
+
+	LOGRUS_LEVEL_TOML = "log.level"
+	LOGRUS_FILE_TOML  = "log.file"
+
+	FILE_OUTPUT_DIR_TOML = "file.outputDir"
+
+	ANCIENT_DB_PATH_TOML = "leveldb.ancient"
+	LVL_DB_PATH_TOML     = "leveldb.path"
+
+	ETH_CLIENT_NAME_TOML   = "ethereum.clientName"
+	ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock"
+	ETH_NETWORK_ID_TOML    = "ethereum.networkID"
+	ETH_NODE_ID_TOML       = "ethereum.nodeID"
+	ETH_CHAIN_ID_TOML      = "ethereum.chainID"
+
+	DATABASE_NAME_TOML                 = "database.name"
+	DATABASE_HOSTNAME_TOML             = "database.hostname"
+	DATABASE_PORT_TOML                 = "database.port"
+	DATABASE_USER_TOML                 = "database.user"
+	DATABASE_PASSWORD_TOML             = "database.password"
+	DATABASE_MAX_IDLE_CONNECTIONS_TOML = "database.maxIdle"
+	DATABASE_MAX_OPEN_CONNECTIONS_TOML = "database.maxOpen"
+	DATABASE_MAX_CONN_LIFETIME_TOML    = "database.maxLifetime"
+)

pkg/snapshot/service.go

@@ -55,8 +55,6 @@ type Service struct {
 }
 
 func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
-	println(con.LevelDBPath)
-	println(con.AncientDBPath)
 	edb, err := rawdb.NewLevelDBDatabaseWithFreezer(
 		con.LevelDBPath, 1024, 256, con.AncientDBPath, "ipld-eth-state-snapshot", true,
 	)

pkg/snapshot/tracker.go

@@ -87,7 +87,7 @@ func (tr *iteratorTracker) tracked(it trie.NodeIterator) (ret *trackedIter) {
 
 // dumps iterator path and bounds to a text file so it can be restored later
 func (tr *iteratorTracker) dump() error {
-	log.Info("Dumping recovery state to:", tr.recoveryFile)
+	log.Info("Dumping recovery state to: ", tr.recoveryFile)
 	var rows [][]string
 	for it, _ := range tr.started {
 		var endPath []byte