ipld-eth-state-snapshot/pkg/snapshot/env.go

137 lines
4.8 KiB
Go
Raw Permalink Normal View History

2022-04-04 19:35:18 +00:00
// Copyright © 2020 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package snapshot
2022-05-18 03:23:38 +00:00
// ENV variables
2022-04-04 19:35:18 +00:00
const (
SNAPSHOT_BLOCK_HEIGHT = "SNAPSHOT_BLOCK_HEIGHT"
SNAPSHOT_WORKERS = "SNAPSHOT_WORKERS"
SNAPSHOT_RECOVERY_FILE = "SNAPSHOT_RECOVERY_FILE"
SNAPSHOT_MODE = "SNAPSHOT_MODE"
SNAPSHOT_START_HEIGHT = "SNAPSHOT_START_HEIGHT"
SNAPSHOT_END_HEIGHT = "SNAPSHOT_END_HEIGHT"
Account selective snapshot (#46) * snapshotter ignores nodes not along a path along those derived from a list of account addresses if one is provided * config and env updates * cmd update * Encode watched address path bytes to hex for comparison * actually ignore the subtries that are not along the paths of interest * Fixes for account selective snapshot * Use non-concurrent iterator when having a single worker * Only index root node when starting path of an iterator is nil * Upgrade deps * Avoid tracking iterators and skip recovery test * Fix recovery mechanism, use sync Map instead of buffered channels * Add test for account selective snapshot * Continue traversal with concurrent iterators with starting path nil * Use errgroup to simplify error handling with concurrent iterators * Check if all the nodes are indexed in the recovery test * Use concurrency safe sync Map in account selective snapshot test * Only track concurrent iterators and refactor code * Fix node and recovered path comparison * Revert back to using buffered channels for tracking iterators * Add a metric to monitor number of active iterators * Update docs * Update seeked path after node is processed * Return error on context cancellation from subtrie iteration * Add tests for account selective snapshot recovery * Explicity enforce concurrent iterator bounds to avoid duplicate nodes * Update full snapshot test to check nodes being indexed * Refactor code to simplify snapshot logic * Remove unnecessary function argument * Use ctx cancellation for handling signals * Add descriptive comments Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>
2022-08-03 11:35:04 +00:00
SNAPSHOT_ACCOUNTS = "SNAPSHOT_ACCOUNTS"
2022-04-04 19:35:18 +00:00
LOGRUS_LEVEL = "LOGRUS_LEVEL"
LOGRUS_FILE = "LOGRUS_FILE"
PROM_METRICS = "PROM_METRICS"
PROM_HTTP = "PROM_HTTP"
PROM_HTTP_ADDR = "PROM_HTTP_ADDR"
PROM_HTTP_PORT = "PROM_HTTP_PORT"
PROM_DB_STATS = "PROM_DB_STATS"
2022-04-04 19:35:18 +00:00
FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR"
ANCIENT_DB_PATH = "ANCIENT_DB_PATH"
LVL_DB_PATH = "LVL_DB_PATH"
ETH_CLIENT_NAME = "ETH_CLIENT_NAME"
ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
ETH_NETWORK_ID = "ETH_NETWORK_ID"
ETH_NODE_ID = "ETH_NODE_ID"
ETH_CHAIN_ID = "ETH_CHAIN_ID"
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME"
)
2022-05-18 03:23:38 +00:00
// TOML bindings
2022-04-04 19:35:18 +00:00
const (
SNAPSHOT_BLOCK_HEIGHT_TOML = "snapshot.blockHeight"
SNAPSHOT_WORKERS_TOML = "snapshot.workers"
SNAPSHOT_RECOVERY_FILE_TOML = "snapshot.recoveryFile"
SNAPSHOT_MODE_TOML = "snapshot.mode"
SNAPSHOT_START_HEIGHT_TOML = "snapshot.startHeight"
SNAPSHOT_END_HEIGHT_TOML = "snapshot.endHeight"
Account selective snapshot (#46) * snapshotter ignores nodes not along a path along those derived from a list of account addresses if one is provided * config and env updates * cmd update * Encode watched address path bytes to hex for comparison * actually ignore the subtries that are not along the paths of interest * Fixes for account selective snapshot * Use non-concurrent iterator when having a single worker * Only index root node when starting path of an iterator is nil * Upgrade deps * Avoid tracking iterators and skip recovery test * Fix recovery mechanism, use sync Map instead of buffered channels * Add test for account selective snapshot * Continue traversal with concurrent iterators with starting path nil * Use errgroup to simplify error handling with concurrent iterators * Check if all the nodes are indexed in the recovery test * Use concurrency safe sync Map in account selective snapshot test * Only track concurrent iterators and refactor code * Fix node and recovered path comparison * Revert back to using buffered channels for tracking iterators * Add a metric to monitor number of active iterators * Update docs * Update seeked path after node is processed * Return error on context cancellation from subtrie iteration * Add tests for account selective snapshot recovery * Explicity enforce concurrent iterator bounds to avoid duplicate nodes * Update full snapshot test to check nodes being indexed * Refactor code to simplify snapshot logic * Remove unnecessary function argument * Use ctx cancellation for handling signals * Add descriptive comments Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>
2022-08-03 11:35:04 +00:00
SNAPSHOT_ACCOUNTS_TOML = "snapshot.accounts"
2022-04-04 19:35:18 +00:00
LOGRUS_LEVEL_TOML = "log.level"
LOGRUS_FILE_TOML = "log.file"
PROM_METRICS_TOML = "prom.metrics"
PROM_HTTP_TOML = "prom.http"
PROM_HTTP_ADDR_TOML = "prom.httpAddr"
PROM_HTTP_PORT_TOML = "prom.httpPort"
PROM_DB_STATS_TOML = "prom.dbStats"
2022-04-04 19:35:18 +00:00
FILE_OUTPUT_DIR_TOML = "file.outputDir"
ANCIENT_DB_PATH_TOML = "leveldb.ancient"
LVL_DB_PATH_TOML = "leveldb.path"
ETH_CLIENT_NAME_TOML = "ethereum.clientName"
ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock"
ETH_NETWORK_ID_TOML = "ethereum.networkID"
ETH_NODE_ID_TOML = "ethereum.nodeID"
ETH_CHAIN_ID_TOML = "ethereum.chainID"
DATABASE_NAME_TOML = "database.name"
DATABASE_HOSTNAME_TOML = "database.hostname"
DATABASE_PORT_TOML = "database.port"
DATABASE_USER_TOML = "database.user"
DATABASE_PASSWORD_TOML = "database.password"
DATABASE_MAX_IDLE_CONNECTIONS_TOML = "database.maxIdle"
DATABASE_MAX_OPEN_CONNECTIONS_TOML = "database.maxOpen"
DATABASE_MAX_CONN_LIFETIME_TOML = "database.maxLifetime"
)
2022-05-18 03:23:38 +00:00
// CLI flags
const (
SNAPSHOT_BLOCK_HEIGHT_CLI = "block-height"
SNAPSHOT_WORKERS_CLI = "workers"
SNAPSHOT_RECOVERY_FILE_CLI = "recovery-file"
SNAPSHOT_MODE_CLI = "snapshot-mode"
SNAPSHOT_START_HEIGHT_CLI = "start-height"
SNAPSHOT_END_HEIGHT_CLI = "end-height"
Account selective snapshot (#46) * snapshotter ignores nodes not along a path along those derived from a list of account addresses if one is provided * config and env updates * cmd update * Encode watched address path bytes to hex for comparison * actually ignore the subtries that are not along the paths of interest * Fixes for account selective snapshot * Use non-concurrent iterator when having a single worker * Only index root node when starting path of an iterator is nil * Upgrade deps * Avoid tracking iterators and skip recovery test * Fix recovery mechanism, use sync Map instead of buffered channels * Add test for account selective snapshot * Continue traversal with concurrent iterators with starting path nil * Use errgroup to simplify error handling with concurrent iterators * Check if all the nodes are indexed in the recovery test * Use concurrency safe sync Map in account selective snapshot test * Only track concurrent iterators and refactor code * Fix node and recovered path comparison * Revert back to using buffered channels for tracking iterators * Add a metric to monitor number of active iterators * Update docs * Update seeked path after node is processed * Return error on context cancellation from subtrie iteration * Add tests for account selective snapshot recovery * Explicity enforce concurrent iterator bounds to avoid duplicate nodes * Update full snapshot test to check nodes being indexed * Refactor code to simplify snapshot logic * Remove unnecessary function argument * Use ctx cancellation for handling signals * Add descriptive comments Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>
2022-08-03 11:35:04 +00:00
SNAPSHOT_ACCOUNTS_CLI = "snapshot-accounts"
2022-05-18 03:23:38 +00:00
LOGRUS_LEVEL_CLI = "log-level"
LOGRUS_FILE_CLI = "log-file"
PROM_METRICS_CLI = "prom-metrics"
PROM_HTTP_CLI = "prom-http"
PROM_HTTP_ADDR_CLI = "prom-httpAddr"
PROM_HTTP_PORT_CLI = "prom-httpPort"
PROM_DB_STATS_CLI = "prom-dbStats"
2022-05-18 03:23:38 +00:00
FILE_OUTPUT_DIR_CLI = "output-dir"
ANCIENT_DB_PATH_CLI = "ancient-path"
LVL_DB_PATH_CLI = "leveldb-path"
ETH_CLIENT_NAME_CLI = "ethereum-client-name"
ETH_GENESIS_BLOCK_CLI = "ethereum-genesis-block"
ETH_NETWORK_ID_CLI = "ethereum-network-id"
ETH_NODE_ID_CLI = "ethereum-node-id"
ETH_CHAIN_ID_CLI = "ethereum-chain-id"
DATABASE_NAME_CLI = "database-name"
DATABASE_HOSTNAME_CLI = "database-hostname"
DATABASE_PORT_CLI = "database-port"
DATABASE_USER_CLI = "database-user"
DATABASE_PASSWORD_CLI = "database-password"
DATABASE_MAX_IDLE_CONNECTIONS_CLI = "database-max-idle"
DATABASE_MAX_OPEN_CONNECTIONS_CLI = "database-max-open"
DATABASE_MAX_CONN_LIFETIME_CLI = "database-max-lifetime"
)