cerc-io/ipld-eth-state-snapshot
16
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

add env bindings

Browse Source
This commit is contained in:
i-norden 2022-04-04 14:35:18 -05:00
parent 560199f930
commit 31105a009c
7 changed files with 139 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/root.go
View File

@ -21,6 +21,8 @@ import (
"os"
"strings"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
@ -46,7 +48,7 @@ func Execute() {
}
func initFuncs(cmd *cobra.Command, args []string) {
logfile := viper.GetString("logfile")
logfile := viper.GetString("log.file")
if logfile != "" {
file, err := os.OpenFile(logfile,
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
@ -84,7 +86,7 @@ func init() {
viper.AutomaticEnv()
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
rootCmd.PersistentFlags().String("log-file", "", "file path for logging")
rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
@ -92,13 +94,13 @@ func init() {
rootCmd.PersistentFlags().String("database-password", "", "database password")
rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
viper.BindPFlag(snapshot.LOGRUS_FILE_TOML, rootCmd.PersistentFlags().Lookup("log-file"))
viper.BindPFlag(snapshot.DATABASE_NAME_TOML, rootCmd.PersistentFlags().Lookup("database-name"))
viper.BindPFlag(snapshot.DATABASE_PORT_TOML, rootCmd.PersistentFlags().Lookup("database-port"))
viper.BindPFlag(snapshot.DATABASE_HOSTNAME_TOML, rootCmd.PersistentFlags().Lookup("database-hostname"))
viper.BindPFlag(snapshot.DATABASE_USER_TOML, rootCmd.PersistentFlags().Lookup("database-user"))
viper.BindPFlag(snapshot.DATABASE_PASSWORD_TOML, rootCmd.PersistentFlags().Lookup("database-password"))
viper.BindPFlag(snapshot.LOGRUS_LEVEL_TOML, rootCmd.PersistentFlags().Lookup("log-level"))
}
func initConfig() {

14
cmd/stateSnapshot.go
View File

@ -94,11 +94,11 @@ func init() {
stateSnapshotCmd.PersistentFlags().String("snapshot-mode", "postgres", "output mode for snapshot ('file' or 'postgres')")
stateSnapshotCmd.PersistentFlags().String("output-dir", "", "directory for writing ouput to while operating in 'file' mode")
viper.BindPFlag("leveldb.path", stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path"))
viper.BindPFlag("leveldb.ancient", stateSnapshotCmd.PersistentFlags().Lookup("ancient-path"))
viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height"))
viper.BindPFlag("snapshot.workers", stateSnapshotCmd.PersistentFlags().Lookup("workers"))
viper.BindPFlag("snapshot.recoveryFile", stateSnapshotCmd.PersistentFlags().Lookup("recovery-file"))
viper.BindPFlag("snapshot.mode", stateSnapshotCmd.PersistentFlags().Lookup("snapshot-mode"))
viper.BindPFlag("file.outputDir", stateSnapshotCmd.PersistentFlags().Lookup("output-dir"))
viper.BindPFlag(snapshot.LVL_DB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path"))
viper.BindPFlag(snapshot.ANCIENT_DB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup("ancient-path"))
viper.BindPFlag(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML, stateSnapshotCmd.PersistentFlags().Lookup("block-height"))
viper.BindPFlag(snapshot.SNAPSHOT_WORKERS_TOML, stateSnapshotCmd.PersistentFlags().Lookup("workers"))
viper.BindPFlag(snapshot.SNAPSHOT_RECOVERY_FILE_TOML, stateSnapshotCmd.PersistentFlags().Lookup("recovery-file"))
viper.BindPFlag(snapshot.SNAPSHOT_MODE_TOML, stateSnapshotCmd.PersistentFlags().Lookup("snapshot-mode"))
viper.BindPFlag(snapshot.FILE_OUTPUT_DIR_TOML, stateSnapshotCmd.PersistentFlags().Lookup("output-dir"))
}

8
environments/example.toml
View File

@ -5,8 +5,12 @@
user = "postgres"
[leveldb]
path = "/Users/user/Library/Ethereum/geth/chaindata"
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
path = "/Users/iannorden/Library/Ethereum/geth/chaindata"
ancient = "/Users/iannorden/Library/Ethereum/geth/chaindata/ancient"
[log]
level = "info"
file = "log_file"
[snapshot]
mode = "file"

72
pkg/snapshot/config.go
View File

@ -26,15 +26,6 @@ import (
"github.com/spf13/viper"
)
const (
ANCIENT_DB_PATH = "ANCIENT_DB_PATH"
ETH_CLIENT_NAME = "ETH_CLIENT_NAME"
ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
ETH_NETWORK_ID = "ETH_NETWORK_ID"
ETH_NODE_ID = "ETH_NODE_ID"
LVL_DB_PATH = "LVL_DB_PATH"
)
// SnapshotMode specifies the snapshot data output method
type SnapshotMode string
@ -80,24 +71,25 @@ func NewConfig(mode SnapshotMode) (*Config, error) {
// Init Initialises config
func (c *Config) Init(mode SnapshotMode) error {
viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
viper.BindEnv(ETH_NODE_ID_TOML, ETH_NODE_ID)
viper.BindEnv(ETH_CLIENT_NAME_TOML, ETH_CLIENT_NAME)
viper.BindEnv(ETH_GENESIS_BLOCK_TOML, ETH_GENESIS_BLOCK)
viper.BindEnv(ETH_NETWORK_ID_TOML, ETH_NETWORK_ID)
viper.BindEnv(ETH_CHAIN_ID_TOML, ETH_CHAIN_ID)
c.Eth.NodeInfo = ethNode.Info{
ID: viper.GetString("ethereum.nodeID"),
ClientName: viper.GetString("ethereum.clientName"),
GenesisBlock: viper.GetString("ethereum.genesisBlock"),
NetworkID: viper.GetString("ethereum.networkID"),
ChainID: viper.GetUint64("ethereum.chainID"),
ID: viper.GetString(ETH_NODE_ID_TOML),
ClientName: viper.GetString(ETH_CLIENT_NAME_TOML),
GenesisBlock: viper.GetString(ETH_GENESIS_BLOCK_TOML),
NetworkID: viper.GetString(ETH_NETWORK_ID_TOML),
ChainID: viper.GetUint64(ETH_CHAIN_ID_TOML),
}
viper.BindEnv("leveldb.ancient", ANCIENT_DB_PATH)
viper.BindEnv("leveldb.path", LVL_DB_PATH)
viper.BindEnv(ANCIENT_DB_PATH_TOML, ANCIENT_DB_PATH)
viper.BindEnv(LVL_DB_PATH_TOML, LVL_DB_PATH)
c.Eth.AncientDBPath = viper.GetString("leveldb.ancient")
c.Eth.LevelDBPath = viper.GetString("leveldb.path")
c.Eth.AncientDBPath = viper.GetString(ANCIENT_DB_PATH_TOML)
c.Eth.LevelDBPath = viper.GetString(LVL_DB_PATH_TOML)
switch mode {
case FileSnapshot:
@ -111,34 +103,34 @@ func (c *Config) Init(mode SnapshotMode) error {
}
func (c *DBConfig) Init() {
viper.BindEnv("database.name", "DATABASE_NAME")
viper.BindEnv("database.hostname", "DATABASE_HOSTNAME")
viper.BindEnv("database.port", "DATABASE_PORT")
viper.BindEnv("database.user", "DATABASE_USER")
viper.BindEnv("database.password", "DATABASE_PASSWORD")
viper.BindEnv("database.maxIdle", "DATABASE_MAX_IDLE_CONNECTIONS")
viper.BindEnv("database.maxOpen", "DATABASE_MAX_OPEN_CONNECTIONS")
viper.BindEnv("database.maxLifetime", "DATABASE_MAX_CONN_LIFETIME")
viper.BindEnv(DATABASE_NAME_TOML, DATABASE_NAME)
viper.BindEnv(DATABASE_HOSTNAME_TOML, DATABASE_HOSTNAME)
viper.BindEnv(DATABASE_PORT_TOML, DATABASE_PORT)
viper.BindEnv(DATABASE_USER_TOML, DATABASE_USER)
viper.BindEnv(DATABASE_PASSWORD_TOML, DATABASE_PASSWORD)
viper.BindEnv(DATABASE_MAX_IDLE_CONNECTIONS_TOML, DATABASE_MAX_IDLE_CONNECTIONS)
viper.BindEnv(DATABASE_MAX_OPEN_CONNECTIONS_TOML, DATABASE_MAX_OPEN_CONNECTIONS)
viper.BindEnv(DATABASE_MAX_CONN_LIFETIME_TOML, DATABASE_MAX_CONN_LIFETIME)
dbParams := postgres.Config{}
// DB params
dbParams.DatabaseName = viper.GetString("database.name")
dbParams.Hostname = viper.GetString("database.hostname")
dbParams.Port = viper.GetInt("database.port")
dbParams.Username = viper.GetString("database.user")
dbParams.Password = viper.GetString("database.password")
dbParams.DatabaseName = viper.GetString(DATABASE_NAME_TOML)
dbParams.Hostname = viper.GetString(DATABASE_HOSTNAME_TOML)
dbParams.Port = viper.GetInt(DATABASE_PORT_TOML)
dbParams.Username = viper.GetString(DATABASE_USER_TOML)
dbParams.Password = viper.GetString(DATABASE_PASSWORD_TOML)
// Connection config
dbParams.MaxIdle = viper.GetInt("database.maxIdle")
dbParams.MaxConns = viper.GetInt("database.maxOpen")
dbParams.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime")) * time.Second
dbParams.MaxIdle = viper.GetInt(DATABASE_MAX_IDLE_CONNECTIONS_TOML)
dbParams.MaxConns = viper.GetInt(DATABASE_MAX_OPEN_CONNECTIONS_TOML)
dbParams.MaxConnLifetime = time.Duration(viper.GetInt(DATABASE_MAX_CONN_LIFETIME_TOML)) * time.Second
c.ConnConfig = dbParams
c.URI = dbParams.DbConnectionString()
}
func (c *FileConfig) Init() error {
viper.BindEnv("file.outputDir", "FILE_OUTPUT_DIR")
c.OutputDir = viper.GetString("file.outputDir")
viper.BindEnv(FILE_OUTPUT_DIR_TOML, FILE_OUTPUT_DIR)
c.OutputDir = viper.GetString(FILE_OUTPUT_DIR_TOML)
if c.OutputDir == "" {
logrus.Infof("no output directory set, using default: %s", defaultOutputDir)
c.OutputDir = defaultOutputDir

82
pkg/snapshot/env.go Normal file
View File

@ -0,0 +1,82 @@
// Copyright © 2020 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package snapshot
/*
viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height"))
viper.BindPFlag("snapshot.workers", stateSnapshotCmd.PersistentFlags().Lookup("workers"))
viper.BindPFlag("snapshot.recoveryFile", stateSnapshotCmd.PersistentFlags().Lookup("recovery-file"))
viper.BindPFlag("snapshot.mode", stateSnapshotCmd.PersistentFlags().Lookup("snapshot-mode"))
*/
const (
SNAPSHOT_BLOCK_HEIGHT = "SNAPSHOT_BLOCK_HEIGHT"
SNAPSHOT_WORKERS = "SNAPSHOT_WORKERS"
SNAPSHOT_RECOVERY_FILE = "SNAPSHOT_RECOVERY_FILE"
SNAPSHOT_MODE = "SNAPSHOT_MODE"
LOGRUS_LEVEL = "LOGRUS_LEVEL"
LOGRUS_FILE = "LOGRUS_FILE"
FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR"
ANCIENT_DB_PATH = "ANCIENT_DB_PATH"
LVL_DB_PATH = "LVL_DB_PATH"
ETH_CLIENT_NAME = "ETH_CLIENT_NAME"
ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
ETH_NETWORK_ID = "ETH_NETWORK_ID"
ETH_NODE_ID = "ETH_NODE_ID"
ETH_CHAIN_ID = "ETH_CHAIN_ID"
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME"
)
const (
SNAPSHOT_BLOCK_HEIGHT_TOML = "snapshot.blockHeight"
SNAPSHOT_WORKERS_TOML = "snapshot.workers"
SNAPSHOT_RECOVERY_FILE_TOML = "snapshot.recoveryFile"
SNAPSHOT_MODE_TOML = "snapshot.mode"
LOGRUS_LEVEL_TOML = "log.level"
LOGRUS_FILE_TOML = "log.file"
FILE_OUTPUT_DIR_TOML = "file.outputDir"
ANCIENT_DB_PATH_TOML = "leveldb.ancient"
LVL_DB_PATH_TOML = "leveldb.path"
ETH_CLIENT_NAME_TOML = "ethereum.clientName"
ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock"
ETH_NETWORK_ID_TOML = "ethereum.networkID"
ETH_NODE_ID_TOML = "ethereum.nodeID"
ETH_CHAIN_ID_TOML = "ethereum.chainID"
DATABASE_NAME_TOML = "database.name"
DATABASE_HOSTNAME_TOML = "database.hostname"
DATABASE_PORT_TOML = "database.port"
DATABASE_USER_TOML = "database.user"
DATABASE_PASSWORD_TOML = "database.password"
DATABASE_MAX_IDLE_CONNECTIONS_TOML = "database.maxIdle"
DATABASE_MAX_OPEN_CONNECTIONS_TOML = "database.maxOpen"
DATABASE_MAX_CONN_LIFETIME_TOML = "database.maxLifetime"
)

2
pkg/snapshot/service.go
View File

@ -55,8 +55,6 @@ type Service struct {
}
func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
println(con.LevelDBPath)
println(con.AncientDBPath)
edb, err := rawdb.NewLevelDBDatabaseWithFreezer(
con.LevelDBPath, 1024, 256, con.AncientDBPath, "ipld-eth-state-snapshot", true,
)

2
pkg/snapshot/tracker.go
View File

@ -87,7 +87,7 @@ func (tr *iteratorTracker) tracked(it trie.NodeIterator) (ret *trackedIter) {
// dumps iterator path and bounds to a text file so it can be restored later
func (tr *iteratorTracker) dump() error {
log.Info("Dumping recovery state to:", tr.recoveryFile)
log.Info("Dumping recovery state to: ", tr.recoveryFile)
var rows [][]string
for it, _ := range tr.started {
var endPath []byte