diff --git a/README.md b/README.md index 39021b5..363b576 100644 --- a/README.md +++ b/README.md @@ -2,131 +2,21 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/cerc-io/eth-statediff-service)](https://goreportcard.com/report/github.com/cerc-io/eth-statediff-service) ->> standalone statediffing service on top of LevelDB - -Purpose: - -Stand up a statediffing service directly on top of a go-ethereum LevelDB instance. +A standalone statediffing service which runs directly on top of a `go-ethereum` LevelDB instance. This service can serve historical state data over the same rpc interface as [statediffing geth](https://github.com/cerc-io/go-ethereum) without needing to run a full node. ## Setup -Build the binary: +Configure access to the private Git server at `git.vdb.to`, then build the executable: ```bash -make build +go build . ``` ## Configuration -An example config file: - -```toml -[leveldb] - # LevelDB access mode - mode = "local" # LEVELDB_MODE - - # in local mode - # LevelDB paths - path = "/Users/user/Library/Ethereum/geth/chaindata" # LEVELDB_PATH - ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT - - # in remote mode - # URL for leveldb-ethdb-rpc endpoint - url = "http://127.0.0.1:8082/" # LEVELDB_URL - -[server] - ipcPath = ".ipc" # SERVICE_IPC_PATH - httpPath = "127.0.0.1:8545" # SERVICE_HTTP_PATH - -[statediff] - prerun = true # STATEDIFF_PRERUN - serviceWorkers = 1 # STATEDIFF_SERVICE_WORKERS - workerQueueSize = 1024 # STATEDIFF_WORKER_QUEUE_SIZE - trieWorkers = 4 # STATEDIFF_TRIE_WORKERS - -[prerun] - only = false # PRERUN_ONLY - parallel = true # PRERUN_PARALLEL - - # to perform prerun in a specific range (optional) - start = 0 # PRERUN_RANGE_START - stop = 100 # PRERUN_RANGE_STOP - - # to perform prerun over multiple ranges (optional) - ranges = [ - [101, 1000] - ] - - # statediffing params for prerun - [prerun.params] - intermediateStateNodes = true # PRERUN_INTERMEDIATE_STATE_NODES - intermediateStorageNodes = true # PRERUN_INTERMEDIATE_STORAGE_NODES - includeBlock = true # PRERUN_INCLUDE_BLOCK - includeReceipts = true # PRERUN_INCLUDE_RECEIPTS - includeTD = true # PRERUN_INCLUDE_TD - includeCode = true # PRERUN_INCLUDE_CODE - watchedAddresses = [] - -[log] - file = "" # LOG_FILE_PATH - level = "info" # LOG_LEVEL - -[database] - # output type - type = "postgres" - - # with postgres type - # db credentials - name = "vulcanize_test" # DATABASE_NAME - hostname = "localhost" # DATABASE_HOSTNAME - port = 5432 # DATABASE_PORT - user = "vulcanize" # DATABASE_USER - password = "..." # DATABASE_PASSWORD - driver = "sqlx" # DATABASE_DRIVER_TYPE - - # with file type - # file mode - fileMode = "csv" # DATABASE_FILE_MODE - - # with SQL file mode - filePath = "" # DATABASE_FILE_PATH - - # with CSV file mode - fileCsvDir = "output_dir" # DATABASE_FILE_CSV_DIR - - # with dump type - # - dumpDestination = "" # DATABASE_DUMP_DST - -[cache] - database = 1024 # DB_CACHE_SIZE_MB - trie = 1024 # TRIE_CACHE_SIZE_MB - -[prom] - # prometheus metrics - metrics = true # PROM_METRICS - http = true # PROM_HTTP - httpAddr = "localhost" # PROM_HTTP_ADDR - httpPort = "8889" # PROM_HTTP_PORT - dbStats = true # PROM_DB_STATS - -[ethereum] - # node info - nodeID = "" # ETH_NODE_ID - clientName = "eth-statediff-service" # ETH_CLIENT_NAME - networkID = 1 # ETH_NETWORK_ID - chainID = 1 # ETH_CHAIN_ID - genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # ETH_GENESIS_BLOCK - - # path to custom chain config file (optional) - # keep chainID same as that in chain config file - chainConfig = "./chain.json" # ETH_CHAIN_CONFIG - -[debug] - pprof = false # DEBUG_PPROF -``` +See [./environments/example.toml](./environments/example.toml) for an annotated example config file. ### Local Setup @@ -182,14 +72,27 @@ An example config file: Example: ```bash - curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"intermediateStateNodes":true,"intermediateStorageNodes":true,"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT" + curl -X POST -H 'Content-Type: application/json' --data '{ + "jsonrpc": "2.0", + "method": "statediff_writeStateDiffsInRange", + "params": [0, 1, { + "ncludeBlock": true, + "includeReceipts": true, + "includeTD": true, + "includeCode": true + } + ], + "id": 1 + }' "$HOST":"$PORT" ``` * Prerun: - * The process can be configured locally with sets of ranges to process as a "prerun" to processing directed by the server endpoints. - * This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining ranges and params in the -`prerun` section of the config. - * Set the range using `prerun.start` and `prerun.stop`. Use `prerun.ranges` if prerun on more than one range is required. + * The process can be configured locally with sets of ranges to process as a "prerun" to + processing directed by the server endpoints. + * This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining +ranges and params in the `prerun` section of the config. + * Set the range using `prerun.start` and `prerun.stop`. Use `prerun.ranges` if prerun on more + than one range is required. * NOTE: Currently, `params.includeTD` must be set to / passed as `true`. @@ -218,7 +121,8 @@ An example config file: ## Import output data in file mode into a database -* When `eth-statediff-service` is run in file mode (`database.type`) the output is in form of a SQL file or multiple CSV files. +* When `eth-statediff-service` is run in file mode (`database.type`: `file`) the output is in form of a SQL + file or multiple CSV files. ### SQL diff --git a/environments/config.toml b/environments/config.toml deleted file mode 100644 index 3a5dd94..0000000 --- a/environments/config.toml +++ /dev/null @@ -1,58 +0,0 @@ -[leveldb] - mode = "local" - path = "/app/geth-rw/chaindata" - ancient = "/app/geth-rw/chaindata/ancient" - -[server] - ipcPath = "" - httpPath = "0.0.0.0:8545" - -[statediff] - prerun = true - serviceWorkers = 1 - workerQueueSize = 1024 - trieWorkers = 16 - -[prerun] - only = true - ranges = [] - [prerun.params] - intermediateStateNodes = true - intermediateStorageNodes = true - includeBlock = true - includeReceipts = true - includeTD = true - includeCode = true - watchedAddresses = [] - -[log] - file = "" - level = "info" - -[database] - type = "postgres" - name = "" - hostname = "" - port = 5432 - user = "" - password = "" - driver = "sqlx" - -[cache] - database = 1024 - trie = 4096 - -[prom] - metrics = true - http = true - httpAddr = "0.0.0.0" - httpPort = 9100 - dbStats = false - -[ethereum] - nodeID = "" - clientName = "eth-statediff-service" - genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" - networkID = 1 - chainID = 1 - chainConfig = "" diff --git a/environments/example.toml b/environments/example.toml index a0e85e5..0fbbe01 100644 --- a/environments/example.toml +++ b/environments/example.toml @@ -1,63 +1,104 @@ [leveldb] - mode = "local" - path = "/Users/user/Library/Ethereum/geth/chaindata" - ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" - url = "http://127.0.0.1:8082/" + # LevelDB access mode + mode = "local" # LEVELDB_MODE + + # LevelDB paths (local mode) + path = "/Users/user/Library/Ethereum/geth/chaindata" # LEVELDB_PATH + ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT + + # URL for leveldb-ethdb-rpc endpoint (remote mode) + url = "http://127.0.0.1:8082/" # LEVELDB_URL [server] - ipcPath = ".ipc" - httpPath = "127.0.0.1:8545" + ipcPath = ".ipc" # SERVICE_IPC_PATH + httpPath = "127.0.0.1:8545" # SERVICE_HTTP_PATH [statediff] - prerun = true - serviceWorkers = 1 - workerQueueSize = 1024 - trieWorkers = 4 + prerun = true # STATEDIFF_PRERUN + serviceWorkers = 1 # STATEDIFF_SERVICE_WORKERS + workerQueueSize = 1024 # STATEDIFF_WORKER_QUEUE_SIZE + trieWorkers = 4 # STATEDIFF_TRIE_WORKERS [prerun] - only = false + only = false # PRERUN_ONLY + parallel = true # PRERUN_PARALLEL + + # to perform prerun in a specific range (optional) + start = 0 # PRERUN_RANGE_START + stop = 100 # PRERUN_RANGE_STOP + + # to perform prerun over multiple ranges (optional) ranges = [ - [0, 1000] + [101, 1000] ] + + # statediffing params for prerun [prerun.params] - intermediateStateNodes = true - intermediateStorageNodes = true - includeBlock = true - includeReceipts = true - includeTD = true - includeCode = true - watchedAddresses = [] + intermediateStateNodes = true # PRERUN_INTERMEDIATE_STATE_NODES + intermediateStorageNodes = true # PRERUN_INTERMEDIATE_STORAGE_NODES + includeBlock = true # PRERUN_INCLUDE_BLOCK + includeReceipts = true # PRERUN_INCLUDE_RECEIPTS + includeTD = true # PRERUN_INCLUDE_TD + includeCode = true # PRERUN_INCLUDE_CODE + watchedAddresses = [] [log] - file = "" - level = "info" + # Leave empty to output to stdout + file = "" # LOG_FILE_PATH + level = "info" # LOG_LEVEL [database] - name = "vulcanize_test" - hostname = "localhost" - port = 5432 - user = "vulcanize" - password = "..." + # output type type = "postgres" - driver = "sqlx" - dumpDestination = "" - filePath = "" + + # with postgres type + # db credentials + name = "vulcanize_test" # DATABASE_NAME + hostname = "localhost" # DATABASE_HOSTNAME + port = 5432 # DATABASE_PORT + user = "vulcanize" # DATABASE_USER + password = "..." # DATABASE_PASSWORD + # SQL backend to use: + driver = "sqlx" # DATABASE_DRIVER_TYPE + + # with file type + # file mode + fileMode = "csv" # DATABASE_FILE_MODE + + # with SQL file mode + filePath = "" # DATABASE_FILE_PATH + + # with CSV file mode + fileCsvDir = "output_dir" # DATABASE_FILE_CSV_DIR + + # with dump type + # + dumpDestination = "" # DATABASE_DUMP_DST [cache] - database = 1024 - trie = 1024 + # settings for geth internal caches + database = 1024 # DB_CACHE_SIZE_MB + trie = 1024 # TRIE_CACHE_SIZE_MB [prom] - dbStats = false - metrics = true - http = true - httpAddr = "localhost" - httpPort = "8889" + # prometheus metrics + metrics = true # PROM_METRICS + http = true # PROM_HTTP + httpAddr = "localhost" # PROM_HTTP_ADDR + httpPort = "8889" # PROM_HTTP_PORT + dbStats = true # PROM_DB_STATS [ethereum] - chainConfig = "" - nodeID = "" - clientName = "eth-statediff-service" - genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" - networkID = 1 - chainID = 1 + # Identifiers for ethereum node + nodeID = "" # ETH_NODE_ID + clientName = "eth-statediff-service" # ETH_CLIENT_NAME + networkID = 1 # ETH_NETWORK_ID + chainID = 1 # ETH_CHAIN_ID + genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # ETH_GENESIS_BLOCK + + # Path to custom chain config file (optional) + # chainID should match that in this config file + chainConfig = "chain.json" # ETH_CHAIN_CONFIG + +[debug] + pprof = false # DEBUG_PPROF