cerc-io/ipld-eth-state-snapshot
16
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #28 from vulcanize/release-v3.0.2

Browse Source 
add env bindings
This commit is contained in:
Ian Norden 2022-05-17 22:36:08 -05:00 committed by GitHub
commit 751e064f8f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 184 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
cmd/root.go
View File

@ -24,6 +24,8 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/vulcanize/ipld-eth-state-snapshot/pkg/snapshot"
)
var (
@ -46,7 +48,7 @@ func Execute() {
}
func initFuncs(cmd *cobra.Command, args []string) {
logfile := viper.GetString("logfile")
logfile := viper.GetString(snapshot.LOGRUS_FILE_TOML)
if logfile != "" {
file, err := os.OpenFile(logfile,
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
@ -66,7 +68,7 @@ func initFuncs(cmd *cobra.Command, args []string) {
}
func logLevel() error {
lvl, err := log.ParseLevel(viper.GetString("log.level"))
lvl, err := log.ParseLevel(viper.GetString(snapshot.LOGRUS_LEVEL_TOML))
if err != nil {
return err
}
@ -84,21 +86,21 @@ func init() {
viper.AutomaticEnv()
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
rootCmd.PersistentFlags().String("database-user", "", "database user")
rootCmd.PersistentFlags().String("database-password", "", "database password")
rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
rootCmd.PersistentFlags().String(snapshot.LOGRUS_FILE_CLI, "", "file path for logging")
rootCmd.PersistentFlags().String(snapshot.DATABASE_NAME_CLI, "vulcanize_public", "database name")
rootCmd.PersistentFlags().Int(snapshot.DATABASE_PORT_CLI, 5432, "database port")
rootCmd.PersistentFlags().String(snapshot.DATABASE_HOSTNAME_CLI, "localhost", "database hostname")
rootCmd.PersistentFlags().String(snapshot.DATABASE_USER_CLI, "", "database user")
rootCmd.PersistentFlags().String(snapshot.DATABASE_PASSWORD_CLI, "", "database password")
rootCmd.PersistentFlags().String(snapshot.LOGRUS_LEVEL_CLI, log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic")
viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
viper.BindPFlag(snapshot.LOGRUS_FILE_TOML, rootCmd.PersistentFlags().Lookup(snapshot.LOGRUS_FILE_CLI))
viper.BindPFlag(snapshot.DATABASE_NAME_TOML, rootCmd.PersistentFlags().Lookup(snapshot.DATABASE_NAME_CLI))
viper.BindPFlag(snapshot.DATABASE_PORT_TOML, rootCmd.PersistentFlags().Lookup(snapshot.DATABASE_PORT_CLI))
viper.BindPFlag(snapshot.DATABASE_HOSTNAME_TOML, rootCmd.PersistentFlags().Lookup(snapshot.DATABASE_HOSTNAME_CLI))
viper.BindPFlag(snapshot.DATABASE_USER_TOML, rootCmd.PersistentFlags().Lookup(snapshot.DATABASE_USER_CLI))
viper.BindPFlag(snapshot.DATABASE_PASSWORD_TOML, rootCmd.PersistentFlags().Lookup(snapshot.DATABASE_PASSWORD_CLI))
viper.BindPFlag(snapshot.LOGRUS_LEVEL_TOML, rootCmd.PersistentFlags().Lookup(snapshot.LOGRUS_LEVEL_CLI))
}
func initConfig() {

36
cmd/stateSnapshot.go
View File

@ -40,7 +40,7 @@ var stateSnapshotCmd = &cobra.Command{
}
func stateSnapshot() {
modeStr := viper.GetString("snapshot.mode")
modeStr := viper.GetString(snapshot.SNAPSHOT_MODE_TOML)
mode := snapshot.SnapshotMode(modeStr)
config, err := snapshot.NewConfig(mode)
if err != nil {
@ -52,8 +52,8 @@ func stateSnapshot() {
if err != nil {
logWithCommand.Fatal(err)
}
height := viper.GetInt64("snapshot.blockHeight")
recoveryFile := viper.GetString("snapshot.recoveryFile")
height := viper.GetInt64(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML)
recoveryFile := viper.GetString(snapshot.SNAPSHOT_RECOVERY_FILE_TOML)
if recoveryFile == "" {
recoveryFile = fmt.Sprintf("./%d_snapshot_recovery", height)
logWithCommand.Infof("no recovery file set, creating default: %s", recoveryFile)
@ -68,7 +68,7 @@ func stateSnapshot() {
if err != nil {
logWithCommand.Fatal(err)
}
workers := viper.GetUint("snapshot.workers")
workers := viper.GetUint(snapshot.SNAPSHOT_WORKERS_TOML)
if height < 0 {
if err := snapshotService.CreateLatestSnapshot(workers); err != nil {
@ -86,19 +86,19 @@ func stateSnapshot() {
func init() {
rootCmd.AddCommand(stateSnapshotCmd)
stateSnapshotCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
stateSnapshotCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
stateSnapshotCmd.PersistentFlags().String("block-height", "", "blockheight to extract state at")
stateSnapshotCmd.PersistentFlags().Int("workers", 1, "number of concurrent workers to use")
stateSnapshotCmd.PersistentFlags().String("recovery-file", "", "file to recover from a previous iteration")
stateSnapshotCmd.PersistentFlags().String("snapshot-mode", "postgres", "output mode for snapshot ('file' or 'postgres')")
stateSnapshotCmd.PersistentFlags().String("output-dir", "", "directory for writing ouput to while operating in 'file' mode")
stateSnapshotCmd.PersistentFlags().String(snapshot.LVL_DB_PATH_CLI, "", "path to primary datastore")
stateSnapshotCmd.PersistentFlags().String(snapshot.ANCIENT_DB_PATH_CLI, "", "path to ancient datastore")
stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI, "", "block height to extract state at")
stateSnapshotCmd.PersistentFlags().Int(snapshot.SNAPSHOT_WORKERS_CLI, 1, "number of concurrent workers to use")
stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_RECOVERY_FILE_CLI, "", "file to recover from a previous iteration")
stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_MODE_CLI, "postgres", "output mode for snapshot ('file' or 'postgres')")
stateSnapshotCmd.PersistentFlags().String(snapshot.FILE_OUTPUT_DIR_CLI, "", "directory for writing ouput to while operating in 'file' mode")
viper.BindPFlag("leveldb.path", stateSnapshotCmd.PersistentFlags().Lookup("leveldb-path"))
viper.BindPFlag("leveldb.ancient", stateSnapshotCmd.PersistentFlags().Lookup("ancient-path"))
viper.BindPFlag("snapshot.blockHeight", stateSnapshotCmd.PersistentFlags().Lookup("block-height"))
viper.BindPFlag("snapshot.workers", stateSnapshotCmd.PersistentFlags().Lookup("workers"))
viper.BindPFlag("snapshot.recoveryFile", stateSnapshotCmd.PersistentFlags().Lookup("recovery-file"))
viper.BindPFlag("snapshot.mode", stateSnapshotCmd.PersistentFlags().Lookup("snapshot-mode"))
viper.BindPFlag("file.outputDir", stateSnapshotCmd.PersistentFlags().Lookup("output-dir"))
viper.BindPFlag(snapshot.LVL_DB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LVL_DB_PATH_CLI))
viper.BindPFlag(snapshot.ANCIENT_DB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.ANCIENT_DB_PATH_CLI))
viper.BindPFlag(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI))
viper.BindPFlag(snapshot.SNAPSHOT_WORKERS_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_WORKERS_CLI))
viper.BindPFlag(snapshot.SNAPSHOT_RECOVERY_FILE_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_RECOVERY_FILE_CLI))
viper.BindPFlag(snapshot.SNAPSHOT_MODE_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_MODE_CLI))
viper.BindPFlag(snapshot.FILE_OUTPUT_DIR_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.FILE_OUTPUT_DIR_CLI))
}

8
environments/example.toml
View File

@ -5,8 +5,12 @@
user = "postgres"
[leveldb]
path = "/Users/user/Library/Ethereum/geth/chaindata"
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
path = "/Users/iannorden/Library/Ethereum/geth/chaindata"
ancient = "/Users/iannorden/Library/Ethereum/geth/chaindata/ancient"
[log]
level = "info"
file = "log_file"
[snapshot]
mode = "file"

72
pkg/snapshot/config.go
View File

@ -26,15 +26,6 @@ import (
"github.com/spf13/viper"
)
const (
ANCIENT_DB_PATH = "ANCIENT_DB_PATH"
ETH_CLIENT_NAME = "ETH_CLIENT_NAME"
ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
ETH_NETWORK_ID = "ETH_NETWORK_ID"
ETH_NODE_ID = "ETH_NODE_ID"
LVL_DB_PATH = "LVL_DB_PATH"
)
// SnapshotMode specifies the snapshot data output method
type SnapshotMode string
@ -80,24 +71,25 @@ func NewConfig(mode SnapshotMode) (*Config, error) {
// Init Initialises config
func (c *Config) Init(mode SnapshotMode) error {
viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
viper.BindEnv(ETH_NODE_ID_TOML, ETH_NODE_ID)
viper.BindEnv(ETH_CLIENT_NAME_TOML, ETH_CLIENT_NAME)
viper.BindEnv(ETH_GENESIS_BLOCK_TOML, ETH_GENESIS_BLOCK)
viper.BindEnv(ETH_NETWORK_ID_TOML, ETH_NETWORK_ID)
viper.BindEnv(ETH_CHAIN_ID_TOML, ETH_CHAIN_ID)
c.Eth.NodeInfo = ethNode.Info{
ID: viper.GetString("ethereum.nodeID"),
ClientName: viper.GetString("ethereum.clientName"),
GenesisBlock: viper.GetString("ethereum.genesisBlock"),
NetworkID: viper.GetString("ethereum.networkID"),
ChainID: viper.GetUint64("ethereum.chainID"),
ID: viper.GetString(ETH_NODE_ID_TOML),
ClientName: viper.GetString(ETH_CLIENT_NAME_TOML),
GenesisBlock: viper.GetString(ETH_GENESIS_BLOCK_TOML),
NetworkID: viper.GetString(ETH_NETWORK_ID_TOML),
ChainID: viper.GetUint64(ETH_CHAIN_ID_TOML),
}
viper.BindEnv("leveldb.ancient", ANCIENT_DB_PATH)
viper.BindEnv("leveldb.path", LVL_DB_PATH)
viper.BindEnv(ANCIENT_DB_PATH_TOML, ANCIENT_DB_PATH)
viper.BindEnv(LVL_DB_PATH_TOML, LVL_DB_PATH)
c.Eth.AncientDBPath = viper.GetString("leveldb.ancient")
c.Eth.LevelDBPath = viper.GetString("leveldb.path")
c.Eth.AncientDBPath = viper.GetString(ANCIENT_DB_PATH_TOML)
c.Eth.LevelDBPath = viper.GetString(LVL_DB_PATH_TOML)
switch mode {
case FileSnapshot:
@ -111,34 +103,34 @@ func (c *Config) Init(mode SnapshotMode) error {
}
func (c *DBConfig) Init() {
viper.BindEnv("database.name", "DATABASE_NAME")
viper.BindEnv("database.hostname", "DATABASE_HOSTNAME")
viper.BindEnv("database.port", "DATABASE_PORT")
viper.BindEnv("database.user", "DATABASE_USER")
viper.BindEnv("database.password", "DATABASE_PASSWORD")
viper.BindEnv("database.maxIdle", "DATABASE_MAX_IDLE_CONNECTIONS")
viper.BindEnv("database.maxOpen", "DATABASE_MAX_OPEN_CONNECTIONS")
viper.BindEnv("database.maxLifetime", "DATABASE_MAX_CONN_LIFETIME")
viper.BindEnv(DATABASE_NAME_TOML, DATABASE_NAME)
viper.BindEnv(DATABASE_HOSTNAME_TOML, DATABASE_HOSTNAME)
viper.BindEnv(DATABASE_PORT_TOML, DATABASE_PORT)
viper.BindEnv(DATABASE_USER_TOML, DATABASE_USER)
viper.BindEnv(DATABASE_PASSWORD_TOML, DATABASE_PASSWORD)
viper.BindEnv(DATABASE_MAX_IDLE_CONNECTIONS_TOML, DATABASE_MAX_IDLE_CONNECTIONS)
viper.BindEnv(DATABASE_MAX_OPEN_CONNECTIONS_TOML, DATABASE_MAX_OPEN_CONNECTIONS)
viper.BindEnv(DATABASE_MAX_CONN_LIFETIME_TOML, DATABASE_MAX_CONN_LIFETIME)
dbParams := postgres.Config{}
// DB params
dbParams.DatabaseName = viper.GetString("database.name")
dbParams.Hostname = viper.GetString("database.hostname")
dbParams.Port = viper.GetInt("database.port")
dbParams.Username = viper.GetString("database.user")
dbParams.Password = viper.GetString("database.password")
dbParams.DatabaseName = viper.GetString(DATABASE_NAME_TOML)
dbParams.Hostname = viper.GetString(DATABASE_HOSTNAME_TOML)
dbParams.Port = viper.GetInt(DATABASE_PORT_TOML)
dbParams.Username = viper.GetString(DATABASE_USER_TOML)
dbParams.Password = viper.GetString(DATABASE_PASSWORD_TOML)
// Connection config
dbParams.MaxIdle = viper.GetInt("database.maxIdle")
dbParams.MaxConns = viper.GetInt("database.maxOpen")
dbParams.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime")) * time.Second
dbParams.MaxIdle = viper.GetInt(DATABASE_MAX_IDLE_CONNECTIONS_TOML)
dbParams.MaxConns = viper.GetInt(DATABASE_MAX_OPEN_CONNECTIONS_TOML)
dbParams.MaxConnLifetime = time.Duration(viper.GetInt(DATABASE_MAX_CONN_LIFETIME_TOML)) * time.Second
c.ConnConfig = dbParams
c.URI = dbParams.DbConnectionString()
}
func (c *FileConfig) Init() error {
viper.BindEnv("file.outputDir", "FILE_OUTPUT_DIR")
c.OutputDir = viper.GetString("file.outputDir")
viper.BindEnv(FILE_OUTPUT_DIR_TOML, FILE_OUTPUT_DIR)
c.OutputDir = viper.GetString(FILE_OUTPUT_DIR_TOML)
if c.OutputDir == "" {
logrus.Infof("no output directory set, using default: %s", defaultOutputDir)
c.OutputDir = defaultOutputDir

109
pkg/snapshot/env.go Normal file
View File

@ -0,0 +1,109 @@
// Copyright © 2020 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package snapshot
// ENV variables
const (
SNAPSHOT_BLOCK_HEIGHT = "SNAPSHOT_BLOCK_HEIGHT"
SNAPSHOT_WORKERS = "SNAPSHOT_WORKERS"
SNAPSHOT_RECOVERY_FILE = "SNAPSHOT_RECOVERY_FILE"
SNAPSHOT_MODE = "SNAPSHOT_MODE"
LOGRUS_LEVEL = "LOGRUS_LEVEL"
LOGRUS_FILE = "LOGRUS_FILE"
FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR"
ANCIENT_DB_PATH = "ANCIENT_DB_PATH"
LVL_DB_PATH = "LVL_DB_PATH"
ETH_CLIENT_NAME = "ETH_CLIENT_NAME"
ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
ETH_NETWORK_ID = "ETH_NETWORK_ID"
ETH_NODE_ID = "ETH_NODE_ID"
ETH_CHAIN_ID = "ETH_CHAIN_ID"
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME"
)
// TOML bindings
const (
SNAPSHOT_BLOCK_HEIGHT_TOML = "snapshot.blockHeight"
SNAPSHOT_WORKERS_TOML = "snapshot.workers"
SNAPSHOT_RECOVERY_FILE_TOML = "snapshot.recoveryFile"
SNAPSHOT_MODE_TOML = "snapshot.mode"
LOGRUS_LEVEL_TOML = "log.level"
LOGRUS_FILE_TOML = "log.file"
FILE_OUTPUT_DIR_TOML = "file.outputDir"
ANCIENT_DB_PATH_TOML = "leveldb.ancient"
LVL_DB_PATH_TOML = "leveldb.path"
ETH_CLIENT_NAME_TOML = "ethereum.clientName"
ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock"
ETH_NETWORK_ID_TOML = "ethereum.networkID"
ETH_NODE_ID_TOML = "ethereum.nodeID"
ETH_CHAIN_ID_TOML = "ethereum.chainID"
DATABASE_NAME_TOML = "database.name"
DATABASE_HOSTNAME_TOML = "database.hostname"
DATABASE_PORT_TOML = "database.port"
DATABASE_USER_TOML = "database.user"
DATABASE_PASSWORD_TOML = "database.password"
DATABASE_MAX_IDLE_CONNECTIONS_TOML = "database.maxIdle"
DATABASE_MAX_OPEN_CONNECTIONS_TOML = "database.maxOpen"
DATABASE_MAX_CONN_LIFETIME_TOML = "database.maxLifetime"
)
// CLI flags
const (
SNAPSHOT_BLOCK_HEIGHT_CLI = "block-height"
SNAPSHOT_WORKERS_CLI = "workers"
SNAPSHOT_RECOVERY_FILE_CLI = "recovery-file"
SNAPSHOT_MODE_CLI = "snapshot-mode"
LOGRUS_LEVEL_CLI = "log-level"
LOGRUS_FILE_CLI = "log-file"
FILE_OUTPUT_DIR_CLI = "output-dir"
ANCIENT_DB_PATH_CLI = "ancient-path"
LVL_DB_PATH_CLI = "leveldb-path"
ETH_CLIENT_NAME_CLI = "ethereum-client-name"
ETH_GENESIS_BLOCK_CLI = "ethereum-genesis-block"
ETH_NETWORK_ID_CLI = "ethereum-network-id"
ETH_NODE_ID_CLI = "ethereum-node-id"
ETH_CHAIN_ID_CLI = "ethereum-chain-id"
DATABASE_NAME_CLI = "database-name"
DATABASE_HOSTNAME_CLI = "database-hostname"
DATABASE_PORT_CLI = "database-port"
DATABASE_USER_CLI = "database-user"
DATABASE_PASSWORD_CLI = "database-password"
DATABASE_MAX_IDLE_CONNECTIONS_CLI = "database-max-idle"
DATABASE_MAX_OPEN_CONNECTIONS_CLI = "database-max-open"
DATABASE_MAX_CONN_LIFETIME_CLI = "database-max-lifetime"
)

2
pkg/snapshot/service.go
View File

@ -55,8 +55,6 @@ type Service struct {
}
func NewLevelDB(con *EthConfig) (ethdb.Database, error) {
println(con.LevelDBPath)
println(con.AncientDBPath)
edb, err := rawdb.NewLevelDBDatabaseWithFreezer(
con.LevelDBPath, 1024, 256, con.AncientDBPath, "ipld-eth-state-snapshot", true,
)