05aeeab581
* snapshotter ignores nodes not along a path along those derived from a list of account addresses if one is provided * config and env updates * cmd update * Encode watched address path bytes to hex for comparison * actually ignore the subtries that are not along the paths of interest * Fixes for account selective snapshot * Use non-concurrent iterator when having a single worker * Only index root node when starting path of an iterator is nil * Upgrade deps * Avoid tracking iterators and skip recovery test * Fix recovery mechanism, use sync Map instead of buffered channels * Add test for account selective snapshot * Continue traversal with concurrent iterators with starting path nil * Use errgroup to simplify error handling with concurrent iterators * Check if all the nodes are indexed in the recovery test * Use concurrency safe sync Map in account selective snapshot test * Only track concurrent iterators and refactor code * Fix node and recovered path comparison * Revert back to using buffered channels for tracking iterators * Add a metric to monitor number of active iterators * Update docs * Update seeked path after node is processed * Return error on context cancellation from subtrie iteration * Add tests for account selective snapshot recovery * Explicity enforce concurrent iterator bounds to avoid duplicate nodes * Update full snapshot test to check nodes being indexed * Refactor code to simplify snapshot logic * Remove unnecessary function argument * Use ctx cancellation for handling signals * Add descriptive comments Co-authored-by: prathamesh0 <prathamesh.musale0@gmail.com>
137 lines
4.8 KiB
Go
137 lines
4.8 KiB
Go
// Copyright © 2020 Vulcanize, Inc
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package snapshot
|
|
|
|
// ENV variables
|
|
const (
|
|
SNAPSHOT_BLOCK_HEIGHT = "SNAPSHOT_BLOCK_HEIGHT"
|
|
SNAPSHOT_WORKERS = "SNAPSHOT_WORKERS"
|
|
SNAPSHOT_RECOVERY_FILE = "SNAPSHOT_RECOVERY_FILE"
|
|
SNAPSHOT_MODE = "SNAPSHOT_MODE"
|
|
SNAPSHOT_START_HEIGHT = "SNAPSHOT_START_HEIGHT"
|
|
SNAPSHOT_END_HEIGHT = "SNAPSHOT_END_HEIGHT"
|
|
SNAPSHOT_ACCOUNTS = "SNAPSHOT_ACCOUNTS"
|
|
|
|
LOGRUS_LEVEL = "LOGRUS_LEVEL"
|
|
LOGRUS_FILE = "LOGRUS_FILE"
|
|
|
|
PROM_METRICS = "PROM_METRICS"
|
|
PROM_HTTP = "PROM_HTTP"
|
|
PROM_HTTP_ADDR = "PROM_HTTP_ADDR"
|
|
PROM_HTTP_PORT = "PROM_HTTP_PORT"
|
|
PROM_DB_STATS = "PROM_DB_STATS"
|
|
|
|
FILE_OUTPUT_DIR = "FILE_OUTPUT_DIR"
|
|
|
|
ANCIENT_DB_PATH = "ANCIENT_DB_PATH"
|
|
LVL_DB_PATH = "LVL_DB_PATH"
|
|
|
|
ETH_CLIENT_NAME = "ETH_CLIENT_NAME"
|
|
ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
|
|
ETH_NETWORK_ID = "ETH_NETWORK_ID"
|
|
ETH_NODE_ID = "ETH_NODE_ID"
|
|
ETH_CHAIN_ID = "ETH_CHAIN_ID"
|
|
|
|
DATABASE_NAME = "DATABASE_NAME"
|
|
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
|
|
DATABASE_PORT = "DATABASE_PORT"
|
|
DATABASE_USER = "DATABASE_USER"
|
|
DATABASE_PASSWORD = "DATABASE_PASSWORD"
|
|
DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
|
|
DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
|
|
DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME"
|
|
)
|
|
|
|
// TOML bindings
|
|
const (
|
|
SNAPSHOT_BLOCK_HEIGHT_TOML = "snapshot.blockHeight"
|
|
SNAPSHOT_WORKERS_TOML = "snapshot.workers"
|
|
SNAPSHOT_RECOVERY_FILE_TOML = "snapshot.recoveryFile"
|
|
SNAPSHOT_MODE_TOML = "snapshot.mode"
|
|
SNAPSHOT_START_HEIGHT_TOML = "snapshot.startHeight"
|
|
SNAPSHOT_END_HEIGHT_TOML = "snapshot.endHeight"
|
|
SNAPSHOT_ACCOUNTS_TOML = "snapshot.accounts"
|
|
|
|
LOGRUS_LEVEL_TOML = "log.level"
|
|
LOGRUS_FILE_TOML = "log.file"
|
|
|
|
PROM_METRICS_TOML = "prom.metrics"
|
|
PROM_HTTP_TOML = "prom.http"
|
|
PROM_HTTP_ADDR_TOML = "prom.httpAddr"
|
|
PROM_HTTP_PORT_TOML = "prom.httpPort"
|
|
PROM_DB_STATS_TOML = "prom.dbStats"
|
|
|
|
FILE_OUTPUT_DIR_TOML = "file.outputDir"
|
|
|
|
ANCIENT_DB_PATH_TOML = "leveldb.ancient"
|
|
LVL_DB_PATH_TOML = "leveldb.path"
|
|
|
|
ETH_CLIENT_NAME_TOML = "ethereum.clientName"
|
|
ETH_GENESIS_BLOCK_TOML = "ethereum.genesisBlock"
|
|
ETH_NETWORK_ID_TOML = "ethereum.networkID"
|
|
ETH_NODE_ID_TOML = "ethereum.nodeID"
|
|
ETH_CHAIN_ID_TOML = "ethereum.chainID"
|
|
|
|
DATABASE_NAME_TOML = "database.name"
|
|
DATABASE_HOSTNAME_TOML = "database.hostname"
|
|
DATABASE_PORT_TOML = "database.port"
|
|
DATABASE_USER_TOML = "database.user"
|
|
DATABASE_PASSWORD_TOML = "database.password"
|
|
DATABASE_MAX_IDLE_CONNECTIONS_TOML = "database.maxIdle"
|
|
DATABASE_MAX_OPEN_CONNECTIONS_TOML = "database.maxOpen"
|
|
DATABASE_MAX_CONN_LIFETIME_TOML = "database.maxLifetime"
|
|
)
|
|
|
|
// CLI flags
|
|
const (
|
|
SNAPSHOT_BLOCK_HEIGHT_CLI = "block-height"
|
|
SNAPSHOT_WORKERS_CLI = "workers"
|
|
SNAPSHOT_RECOVERY_FILE_CLI = "recovery-file"
|
|
SNAPSHOT_MODE_CLI = "snapshot-mode"
|
|
SNAPSHOT_START_HEIGHT_CLI = "start-height"
|
|
SNAPSHOT_END_HEIGHT_CLI = "end-height"
|
|
SNAPSHOT_ACCOUNTS_CLI = "snapshot-accounts"
|
|
|
|
LOGRUS_LEVEL_CLI = "log-level"
|
|
LOGRUS_FILE_CLI = "log-file"
|
|
|
|
PROM_METRICS_CLI = "prom-metrics"
|
|
PROM_HTTP_CLI = "prom-http"
|
|
PROM_HTTP_ADDR_CLI = "prom-httpAddr"
|
|
PROM_HTTP_PORT_CLI = "prom-httpPort"
|
|
PROM_DB_STATS_CLI = "prom-dbStats"
|
|
|
|
FILE_OUTPUT_DIR_CLI = "output-dir"
|
|
|
|
ANCIENT_DB_PATH_CLI = "ancient-path"
|
|
LVL_DB_PATH_CLI = "leveldb-path"
|
|
|
|
ETH_CLIENT_NAME_CLI = "ethereum-client-name"
|
|
ETH_GENESIS_BLOCK_CLI = "ethereum-genesis-block"
|
|
ETH_NETWORK_ID_CLI = "ethereum-network-id"
|
|
ETH_NODE_ID_CLI = "ethereum-node-id"
|
|
ETH_CHAIN_ID_CLI = "ethereum-chain-id"
|
|
|
|
DATABASE_NAME_CLI = "database-name"
|
|
DATABASE_HOSTNAME_CLI = "database-hostname"
|
|
DATABASE_PORT_CLI = "database-port"
|
|
DATABASE_USER_CLI = "database-user"
|
|
DATABASE_PASSWORD_CLI = "database-password"
|
|
DATABASE_MAX_IDLE_CONNECTIONS_CLI = "database-max-idle"
|
|
DATABASE_MAX_OPEN_CONNECTIONS_CLI = "database-max-open"
|
|
DATABASE_MAX_CONN_LIFETIME_CLI = "database-max-lifetime"
|
|
)
|