cerc-io/ipld-eth-state-snapshot
16
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor to use statediff plugin #1

Merged
roysc merged 27 commits from refactor-use-plugin into v5 2023-09-29 18:43:28 +00:00
Conversation 4 Commits 27 Files Changed 96 +16 -27
5 changed files with 16 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit fc3d1432f4 - Show all commits

4
README.md
View File

@ -26,9 +26,9 @@ Config format:
[leveldb] [leveldb]
# path to geth leveldb # path to geth leveldb
path = "/Users/user/Library/Ethereum/geth/chaindata" # LVL_DB_PATH path = "/Users/user/Library/Ethereum/geth/chaindata" # LEVELDB_PATH
# path to geth ancient database # path to geth ancient database
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # ANCIENT_DB_PATH ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT
[database] [database]
# when operating in 'postgres' output mode # when operating in 'postgres' output mode

8
cmd/stateSnapshot.go
View File

@ -98,8 +98,8 @@ func stateSnapshot() {
func init() { func init() {
rootCmd.AddCommand(stateSnapshotCmd) rootCmd.AddCommand(stateSnapshotCmd)
stateSnapshotCmd.PersistentFlags().String(snapshot.LVL_DB_PATH_CLI, "", "path to primary datastore") stateSnapshotCmd.PersistentFlags().String(snapshot.LEVELDB_PATH_CLI, "", "path to primary datastore")
telackey marked this conversation as resolved Outdated
telackey commented 2023-09-15 21:30:45 +00:00
Outdated
Review
Copy Link

Can you make sure these match eth-statediff-service and any changes are documented? We will need to change all the code that uses these (eg, chain-chunker) to make sure we use the right env.

Can you make sure these match eth-statediff-service and any changes are documented? We will need to change all the code that uses these (eg, chain-chunker) to make sure we use the right env.
roysc commented 2023-09-19 18:58:04 +00:00
Outdated
Review
Copy Link

I mainly changed them to match eth-statediff-service so we're good there, but yes I'll review if anything needs an update including deployments. I'll mention it in Git and the readme, but I'm not actually sure if there's a better changelog-type place to document it.

I mainly changed them to match eth-statediff-service so we're good there, but yes I'll review if anything needs an update including deployments. I'll mention it in Git and the readme, but I'm not actually sure if there's a better changelog-type place to document it.
stateSnapshotCmd.PersistentFlags().String(snapshot.ANCIENT_DB_PATH_CLI, "", "path to ancient datastore") stateSnapshotCmd.PersistentFlags().String(snapshot.LEVELDB_ANCIENT_CLI, "", "path to ancient datastore")
stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI, "", "block height to extract state at") stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI, "", "block height to extract state at")
stateSnapshotCmd.PersistentFlags().Int(snapshot.SNAPSHOT_WORKERS_CLI, 1, "number of concurrent workers to use") stateSnapshotCmd.PersistentFlags().Int(snapshot.SNAPSHOT_WORKERS_CLI, 1, "number of concurrent workers to use")
stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_RECOVERY_FILE_CLI, "", "file to recover from a previous iteration") stateSnapshotCmd.PersistentFlags().String(snapshot.SNAPSHOT_RECOVERY_FILE_CLI, "", "file to recover from a previous iteration")
@ -107,8 +107,8 @@ func init() {
stateSnapshotCmd.PersistentFlags().String(snapshot.FILE_OUTPUT_DIR_CLI, "", "directory for writing ouput to while operating in 'file' mode") stateSnapshotCmd.PersistentFlags().String(snapshot.FILE_OUTPUT_DIR_CLI, "", "directory for writing ouput to while operating in 'file' mode")
stateSnapshotCmd.PersistentFlags().StringArray(snapshot.SNAPSHOT_ACCOUNTS_CLI, nil, "list of account addresses to limit snapshot to") stateSnapshotCmd.PersistentFlags().StringArray(snapshot.SNAPSHOT_ACCOUNTS_CLI, nil, "list of account addresses to limit snapshot to")
viper.BindPFlag(snapshot.LVL_DB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LVL_DB_PATH_CLI)) viper.BindPFlag(snapshot.LEVELDB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LEVELDB_PATH_CLI))
viper.BindPFlag(snapshot.ANCIENT_DB_PATH_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.ANCIENT_DB_PATH_CLI)) viper.BindPFlag(snapshot.LEVELDB_ANCIENT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.LEVELDB_ANCIENT_CLI))
viper.BindPFlag(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI)) viper.BindPFlag(snapshot.SNAPSHOT_BLOCK_HEIGHT_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_BLOCK_HEIGHT_CLI))
viper.BindPFlag(snapshot.SNAPSHOT_WORKERS_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_WORKERS_CLI)) viper.BindPFlag(snapshot.SNAPSHOT_WORKERS_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_WORKERS_CLI))
viper.BindPFlag(snapshot.SNAPSHOT_RECOVERY_FILE_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_RECOVERY_FILE_CLI)) viper.BindPFlag(snapshot.SNAPSHOT_RECOVERY_FILE_TOML, stateSnapshotCmd.PersistentFlags().Lookup(snapshot.SNAPSHOT_RECOVERY_FILE_CLI))

3
environments/example.toml
View File

@ -26,9 +26,6 @@
blockHeight = 32 blockHeight = 32
recoveryFile = "recovery_file" recoveryFile = "recovery_file"
startHeight = 1
endHeight = 12
[file] [file]
outputDir = "output_dir/" outputDir = "output_dir/"

10
pkg/snapshot/config.go
View File

@ -104,11 +104,11 @@ func (c *Config) Init(mode SnapshotMode) error {
ChainID: viper.GetUint64(ETH_CHAIN_ID_TOML), ChainID: viper.GetUint64(ETH_CHAIN_ID_TOML),
} }
viper.BindEnv(ANCIENT_DB_PATH_TOML, ANCIENT_DB_PATH) viper.BindEnv(LEVELDB_ANCIENT_TOML, LEVELDB_ANCIENT)
viper.BindEnv(LVL_DB_PATH_TOML, LVL_DB_PATH) viper.BindEnv(LEVELDB_PATH_TOML, LEVELDB_PATH)
c.Eth.AncientDBPath = viper.GetString(ANCIENT_DB_PATH_TOML) c.Eth.AncientDBPath = viper.GetString(LEVELDB_ANCIENT_TOML)
c.Eth.LevelDBPath = viper.GetString(LVL_DB_PATH_TOML) c.Eth.LevelDBPath = viper.GetString(LEVELDB_PATH_TOML)
switch mode { switch mode {
case FileSnapshot: case FileSnapshot:
@ -159,8 +159,6 @@ func InitFile(c *FileConfig) error {
func (c *ServiceConfig) Init() error { func (c *ServiceConfig) Init() error {
viper.BindEnv(SNAPSHOT_BLOCK_HEIGHT_TOML, SNAPSHOT_BLOCK_HEIGHT) viper.BindEnv(SNAPSHOT_BLOCK_HEIGHT_TOML, SNAPSHOT_BLOCK_HEIGHT)
viper.BindEnv(SNAPSHOT_START_HEIGHT_TOML, SNAPSHOT_START_HEIGHT)
viper.BindEnv(SNAPSHOT_END_HEIGHT_TOML, SNAPSHOT_END_HEIGHT)
viper.BindEnv(SNAPSHOT_MODE_TOML, SNAPSHOT_MODE) viper.BindEnv(SNAPSHOT_MODE_TOML, SNAPSHOT_MODE)
viper.BindEnv(SNAPSHOT_WORKERS_TOML, SNAPSHOT_WORKERS) viper.BindEnv(SNAPSHOT_WORKERS_TOML, SNAPSHOT_WORKERS)
viper.BindEnv(PROM_HTTP_TOML, PROM_HTTP) viper.BindEnv(PROM_HTTP_TOML, PROM_HTTP)

18
pkg/snapshot/env.go
View File

@ -21,8 +21,6 @@ const (
SNAPSHOT_WORKERS = "SNAPSHOT_WORKERS" SNAPSHOT_WORKERS = "SNAPSHOT_WORKERS"
SNAPSHOT_RECOVERY_FILE = "SNAPSHOT_RECOVERY_FILE" SNAPSHOT_RECOVERY_FILE = "SNAPSHOT_RECOVERY_FILE"
SNAPSHOT_MODE = "SNAPSHOT_MODE" SNAPSHOT_MODE = "SNAPSHOT_MODE"
SNAPSHOT_START_HEIGHT = "SNAPSHOT_START_HEIGHT"
SNAPSHOT_END_HEIGHT = "SNAPSHOT_END_HEIGHT"
SNAPSHOT_ACCOUNTS = "SNAPSHOT_ACCOUNTS" SNAPSHOT_ACCOUNTS = "SNAPSHOT_ACCOUNTS"
LOGRUS_LEVEL = "LOGRUS_LEVEL" LOGRUS_LEVEL = "LOGRUS_LEVEL"
@ -36,8 +34,8 @@ const (
FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR" FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR"
ANCIENT_DB_PATH = "ANCIENT_DB_PATH" LEVELDB_ANCIENT = "LEVELDB_ANCIENT"
LVL_DB_PATH = "LVL_DB_PATH" LEVELDB_PATH = "LEVELDB_PATH"
ETH_CLIENT_NAME = "ETH_CLIENT_NAME" ETH_CLIENT_NAME = "ETH_CLIENT_NAME"
ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK" ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
@ -61,8 +59,6 @@ const (
SNAPSHOT_WORKERS_TOML = "snapshot.workers" SNAPSHOT_WORKERS_TOML = "snapshot.workers"
SNAPSHOT_RECOVERY_FILE_TOML = "snapshot.recoveryFile" SNAPSHOT_RECOVERY_FILE_TOML = "snapshot.recoveryFile"
SNAPSHOT_MODE_TOML = "snapshot.mode" SNAPSHOT_MODE_TOML = "snapshot.mode"
SNAPSHOT_START_HEIGHT_TOML = "snapshot.startHeight"
SNAPSHOT_END_HEIGHT_TOML = "snapshot.endHeight"
SNAPSHOT_ACCOUNTS_TOML = "snapshot.accounts" SNAPSHOT_ACCOUNTS_TOML = "snapshot.accounts"
LOGRUS_LEVEL_TOML = "log.level" LOGRUS_LEVEL_TOML = "log.level"
@ -76,8 +72,8 @@ const (
FILE_OUTPUT_DIR_TOML = "file.outputDir" FILE_OUTPUT_DIR_TOML = "file.outputDir"
ANCIENT_DB_PATH_TOML = "leveldb.ancient" LEVELDB_ANCIENT_TOML = "leveldb.ancient"
LVL_DB_PATH_TOML = "leveldb.path" LEVELDB_PATH_TOML = "leveldb.path"
ETH_CLIENT_NAME_TOML = "ethereum.clientName" ETH_CLIENT_NAME_TOML = "ethereum.clientName"
ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock" ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock"
@ -101,8 +97,6 @@ const (
SNAPSHOT_WORKERS_CLI = "workers" SNAPSHOT_WORKERS_CLI = "workers"
SNAPSHOT_RECOVERY_FILE_CLI = "recovery-file" SNAPSHOT_RECOVERY_FILE_CLI = "recovery-file"
SNAPSHOT_MODE_CLI = "snapshot-mode" SNAPSHOT_MODE_CLI = "snapshot-mode"
SNAPSHOT_START_HEIGHT_CLI = "start-height"
SNAPSHOT_END_HEIGHT_CLI = "end-height"
SNAPSHOT_ACCOUNTS_CLI = "snapshot-accounts" SNAPSHOT_ACCOUNTS_CLI = "snapshot-accounts"
LOGRUS_LEVEL_CLI = "log-level" LOGRUS_LEVEL_CLI = "log-level"
@ -116,8 +110,8 @@ const (
FILE_OUTPUT_DIR_CLI = "output-dir" FILE_OUTPUT_DIR_CLI = "output-dir"
ANCIENT_DB_PATH_CLI = "ancient-path" LEVELDB_ANCIENT_CLI = "ancient-path"
LVL_DB_PATH_CLI = "leveldb-path" LEVELDB_PATH_CLI = "leveldb-path"
ETH_CLIENT_NAME_CLI = "ethereum-client-name" ETH_CLIENT_NAME_CLI = "ethereum-client-name"
ETH_GENESIS_BLOCK_CLI = "ethereum-genesis-block" ETH_GENESIS_BLOCK_CLI = "ethereum-genesis-block"