update docs & config

This commit is contained in:
Roy Crihfield 2023-08-25 19:02:32 +08:00
parent aea8977c7d
commit 53319ba10c
3 changed files with 105 additions and 214 deletions

142
README.md
View File

@ -2,129 +2,21 @@
[![Go Report Card](https://goreportcard.com/badge/github.com/cerc-io/eth-statediff-service)](https://goreportcard.com/report/github.com/cerc-io/eth-statediff-service)
>> standalone statediffing service on top of LevelDB
Purpose:
Stand up a statediffing service directly on top of a go-ethereum LevelDB instance.
A standalone statediffing service which runs directly on top of a `go-ethereum` LevelDB instance.
This service can serve historical state data over the same rpc interface as
[statediffing geth](https://github.com/cerc-io/go-ethereum) without needing to run a full node.
## Setup
Build the binary:
Configure access to the private Git server at `git.vdb.to`, then build the executable:
```bash
make build
go build .
```
## Configuration
An example config file:
```toml
[leveldb]
# LevelDB access mode <local | remote>
mode = "local" # LEVELDB_MODE
# in local mode
# LevelDB paths
path = "/Users/user/Library/Ethereum/geth/chaindata" # LEVELDB_PATH
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT
# in remote mode
# URL for leveldb-ethdb-rpc endpoint
url = "http://127.0.0.1:8082/" # LEVELDB_URL
[server]
ipcPath = ".ipc" # SERVICE_IPC_PATH
httpPath = "127.0.0.1:8545" # SERVICE_HTTP_PATH
[statediff]
prerun = true # STATEDIFF_PRERUN
serviceWorkers = 1 # STATEDIFF_SERVICE_WORKERS
workerQueueSize = 1024 # STATEDIFF_WORKER_QUEUE_SIZE
trieWorkers = 4 # STATEDIFF_TRIE_WORKERS
[prerun]
only = false # PRERUN_ONLY
parallel = true # PRERUN_PARALLEL
# to perform prerun in a specific range (optional)
start = 0 # PRERUN_RANGE_START
stop = 100 # PRERUN_RANGE_STOP
# to perform prerun over multiple ranges (optional)
ranges = [
[101, 1000]
]
# statediffing params for prerun
[prerun.params]
includeBlock = true # PRERUN_INCLUDE_BLOCK
includeReceipts = true # PRERUN_INCLUDE_RECEIPTS
includeTD = true # PRERUN_INCLUDE_TD
includeCode = true # PRERUN_INCLUDE_CODE
watchedAddresses = []
[log]
file = "" # LOG_FILE_PATH
level = "info" # LOG_LEVEL
[database]
# output type <postgres | file | dump>
type = "postgres"
# with postgres type
# db credentials
name = "vulcanize_test" # DATABASE_NAME
hostname = "localhost" # DATABASE_HOSTNAME
port = 5432 # DATABASE_PORT
user = "vulcanize" # DATABASE_USER
password = "..." # DATABASE_PASSWORD
driver = "sqlx" # DATABASE_DRIVER_TYPE <sqlx | pgx>
# with file type
# file mode <sql | csv>
fileMode = "csv" # DATABASE_FILE_MODE
# with SQL file mode
filePath = "" # DATABASE_FILE_PATH
# with CSV file mode
fileCsvDir = "output_dir" # DATABASE_FILE_CSV_DIR
# with dump type
# <stdout | stderr | discard>
dumpDestination = "" # DATABASE_DUMP_DST
[cache]
database = 1024 # DB_CACHE_SIZE_MB
trie = 1024 # TRIE_CACHE_SIZE_MB
[prom]
# prometheus metrics
metrics = true # PROM_METRICS
http = true # PROM_HTTP
httpAddr = "localhost" # PROM_HTTP_ADDR
httpPort = "8889" # PROM_HTTP_PORT
dbStats = true # PROM_DB_STATS
[ethereum]
# node info
nodeID = "" # ETH_NODE_ID
clientName = "eth-statediff-service" # ETH_CLIENT_NAME
networkID = 1 # ETH_NETWORK_ID
chainID = 1 # ETH_CHAIN_ID
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # ETH_GENESIS_BLOCK
# path to custom chain config file (optional)
# keep chainID same as that in chain config file
chainConfig = "./chain.json" # ETH_CHAIN_CONFIG
[debug]
pprof = false # DEBUG_PPROF
```
See [./environments/example.toml](./environments/example.toml) for an annotated example config file.
### Local Setup
@ -180,14 +72,27 @@ An example config file:
Example:
```bash
curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"intermediateStateNodes":true,"intermediateStorageNodes":true,"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT"
curl -X POST -H 'Content-Type: application/json' --data '{
"jsonrpc": "2.0",
"method": "statediff_writeStateDiffsInRange",
"params": [0, 1, {
"ncludeBlock": true,
"includeReceipts": true,
"includeTD": true,
"includeCode": true
}
],
"id": 1
}' "$HOST":"$PORT"
```
* Prerun:
* The process can be configured locally with sets of ranges to process as a "prerun" to processing directed by the server endpoints.
* This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining ranges and params in the
`prerun` section of the config.
* Set the range using `prerun.start` and `prerun.stop`. Use `prerun.ranges` if prerun on more than one range is required.
* The process can be configured locally with sets of ranges to process as a "prerun" to
processing directed by the server endpoints.
* This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining
ranges and params in the `prerun` section of the config.
* Set the range using `prerun.start` and `prerun.stop`. Use `prerun.ranges` if prerun on more
than one range is required.
* NOTE: Currently, `params.includeTD` must be set to / passed as `true`.
@ -216,7 +121,8 @@ An example config file:
## Import output data in file mode into a database
* When `eth-statediff-service` is run in file mode (`database.type`) the output is in form of a SQL file or multiple CSV files.
* When `eth-statediff-service` is run in file mode (`database.type`: `file`) the output is in form of a SQL
file or multiple CSV files.
### SQL

View File

@ -1,56 +0,0 @@
[leveldb]
mode = "local"
path = "/app/geth-rw/chaindata"
ancient = "/app/geth-rw/chaindata/ancient"
[server]
ipcPath = ""
httpPath = "0.0.0.0:8545"
[statediff]
prerun = true
serviceWorkers = 1
workerQueueSize = 1024
trieWorkers = 16
[prerun]
only = true
ranges = []
[prerun.params]
includeBlock = true
includeReceipts = true
includeTD = true
includeCode = true
watchedAddresses = []
[log]
file = ""
level = "info"
[database]
type = "postgres"
name = ""
hostname = ""
port = 5432
user = ""
password = ""
driver = "sqlx"
[cache]
database = 1024
trie = 4096
[prom]
metrics = true
http = true
httpAddr = "0.0.0.0"
httpPort = 9100
dbStats = false
[ethereum]
nodeID = ""
clientName = "eth-statediff-service"
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
networkID = 1
chainID = 1
chainConfig = ""

View File

@ -1,61 +1,102 @@
[leveldb]
mode = "local"
path = "/Users/user/Library/Ethereum/geth/chaindata"
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
url = "http://127.0.0.1:8082/"
# LevelDB access mode <local | remote>
mode = "local" # LEVELDB_MODE
# LevelDB paths (local mode)
path = "/Users/user/Library/Ethereum/geth/chaindata" # LEVELDB_PATH
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT
# URL for leveldb-ethdb-rpc endpoint (remote mode)
url = "http://127.0.0.1:8082/" # LEVELDB_URL
[server]
ipcPath = ".ipc"
httpPath = "127.0.0.1:8545"
ipcPath = ".ipc" # SERVICE_IPC_PATH
httpPath = "127.0.0.1:8545" # SERVICE_HTTP_PATH
[statediff]
prerun = true
serviceWorkers = 1
workerQueueSize = 1024
trieWorkers = 4
prerun = true # STATEDIFF_PRERUN
serviceWorkers = 1 # STATEDIFF_SERVICE_WORKERS
workerQueueSize = 1024 # STATEDIFF_WORKER_QUEUE_SIZE
trieWorkers = 4 # STATEDIFF_TRIE_WORKERS
[prerun]
only = false
only = false # PRERUN_ONLY
parallel = true # PRERUN_PARALLEL
# to perform prerun in a specific range (optional)
start = 0 # PRERUN_RANGE_START
stop = 100 # PRERUN_RANGE_STOP
# to perform prerun over multiple ranges (optional)
ranges = [
[0, 1000]
[101, 1000]
]
# statediffing params for prerun
[prerun.params]
includeBlock = true
includeReceipts = true
includeTD = true
includeCode = true
includeBlock = true # PRERUN_INCLUDE_BLOCK
includeReceipts = true # PRERUN_INCLUDE_RECEIPTS
includeTD = true # PRERUN_INCLUDE_TD
includeCode = true # PRERUN_INCLUDE_CODE
watchedAddresses = []
[log]
file = ""
level = "info"
# Leave empty to output to stdout
file = "" # LOG_FILE_PATH
level = "info" # LOG_LEVEL
[database]
name = "vulcanize_test"
hostname = "localhost"
port = 5432
user = "vulcanize"
password = "..."
# output type <postgres | file | dump>
type = "postgres"
driver = "sqlx"
dumpDestination = ""
filePath = ""
# with postgres type
# db credentials
name = "vulcanize_test" # DATABASE_NAME
hostname = "localhost" # DATABASE_HOSTNAME
port = 5432 # DATABASE_PORT
user = "vulcanize" # DATABASE_USER
password = "..." # DATABASE_PASSWORD
# SQL backend to use: <sqlx | pgx>
driver = "sqlx" # DATABASE_DRIVER_TYPE
# with file type
# file mode <sql | csv>
fileMode = "csv" # DATABASE_FILE_MODE
# with SQL file mode
filePath = "" # DATABASE_FILE_PATH
# with CSV file mode
fileCsvDir = "output_dir" # DATABASE_FILE_CSV_DIR
# with dump type
# <stdout | stderr | discard>
dumpDestination = "" # DATABASE_DUMP_DST
[cache]
database = 1024
trie = 1024
# settings for geth internal caches
database = 1024 # DB_CACHE_SIZE_MB
trie = 1024 # TRIE_CACHE_SIZE_MB
[prom]
dbStats = false
metrics = true
http = true
httpAddr = "localhost"
httpPort = "8889"
# prometheus metrics
metrics = true # PROM_METRICS
http = true # PROM_HTTP
httpAddr = "localhost" # PROM_HTTP_ADDR
httpPort = "8889" # PROM_HTTP_PORT
dbStats = true # PROM_DB_STATS
[ethereum]
chainConfig = ""
nodeID = ""
clientName = "eth-statediff-service"
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
networkID = 1
chainID = 1
# Identifiers for ethereum node
nodeID = "" # ETH_NODE_ID
clientName = "eth-statediff-service" # ETH_CLIENT_NAME
networkID = 1 # ETH_NETWORK_ID
chainID = 1 # ETH_CHAIN_ID
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # ETH_GENESIS_BLOCK
# Path to custom chain config file (optional)
# chainID should match that in this config file
chainConfig = "chain.json" # ETH_CHAIN_CONFIG
[debug]
pprof = false # DEBUG_PPROF