Add instructions to import data output in file mode
This commit is contained in:
parent
1f898f60c3
commit
aa6ee578f4
331
README.md
331
README.md
@ -8,7 +8,7 @@ Purpose:
|
|||||||
|
|
||||||
Stand up a statediffing service directly on top of a go-ethereum LevelDB instance.
|
Stand up a statediffing service directly on top of a go-ethereum LevelDB instance.
|
||||||
This service can serve historical state data over the same rpc interface as
|
This service can serve historical state data over the same rpc interface as
|
||||||
[statediffing geth](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.5) without needing to run a full node
|
[statediffing geth](https://github.com/vulcanize/go-ethereum) without needing to run a full node.
|
||||||
|
|
||||||
## Setup
|
## Setup
|
||||||
|
|
||||||
@ -18,6 +18,111 @@ Build the binary:
|
|||||||
make build
|
make build
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
An example config file:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[leveldb]
|
||||||
|
# LevelDB access mode <local | remote>
|
||||||
|
mode = "local" # LVLDB_MODE
|
||||||
|
|
||||||
|
# in local mode
|
||||||
|
# LevelDB paths
|
||||||
|
path = "/Users/user/Library/Ethereum/geth/chaindata" # LVLDB_PATH
|
||||||
|
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LVLDB_ANCIENT
|
||||||
|
|
||||||
|
# in remote mode
|
||||||
|
# URL for leveldb-ethdb-rpc endpoint
|
||||||
|
url = "http://127.0.0.1:8082/" # LVLDB_URL
|
||||||
|
|
||||||
|
[server]
|
||||||
|
ipcPath = ".ipc" # SERVICE_IPC_PATH
|
||||||
|
httpPath = "127.0.0.1:8545" # SERVICE_HTTP_PATH
|
||||||
|
|
||||||
|
[statediff]
|
||||||
|
prerun = true # STATEDIFF_PRERUN
|
||||||
|
serviceWorkers = 1 # STATEDIFF_SERVICE_WORKERS
|
||||||
|
workerQueueSize = 1024 # STATEDIFF_WORKER_QUEUE_SIZE
|
||||||
|
trieWorkers = 4 # STATEDIFF_TRIE_WORKERS
|
||||||
|
|
||||||
|
[prerun]
|
||||||
|
only = false # PRERUN_ONLY
|
||||||
|
|
||||||
|
# to perform prerun in a specific range (optional)
|
||||||
|
start = 0 # PRERUN_RANGE_START
|
||||||
|
stop = 100 # PRERUN_RANGE_STOP
|
||||||
|
|
||||||
|
# to perform prerun over multiple ranges (optional)
|
||||||
|
ranges = [
|
||||||
|
[101, 1000]
|
||||||
|
]
|
||||||
|
|
||||||
|
# statediffing params for prerun
|
||||||
|
[prerun.params]
|
||||||
|
intermediateStateNodes = true # PRERUN_INTERMEDIATE_STATE_NODES
|
||||||
|
intermediateStorageNodes = true # PRERUN_INTERMEDIATE_STORAGE_NODES
|
||||||
|
includeBlock = true # PRERUN_INCLUDE_BLOCK
|
||||||
|
includeReceipts = true # PRERUN_INCLUDE_RECEIPTS
|
||||||
|
includeTD = true # PRERUN_INCLUDE_TD
|
||||||
|
includeCode = true # PRERUN_INCLUDE_CODE
|
||||||
|
watchedAddresses = []
|
||||||
|
|
||||||
|
[log]
|
||||||
|
file = "" # LOG_FILE_PATH
|
||||||
|
level = "info" # LOG_LEVEL
|
||||||
|
|
||||||
|
[database]
|
||||||
|
# output type <postgres | file | dump>
|
||||||
|
type = "postgres"
|
||||||
|
|
||||||
|
# with postgres type
|
||||||
|
# db credentials
|
||||||
|
name = "vulcanize_test" # DATABASE_NAME
|
||||||
|
hostname = "localhost" # DATABASE_HOSTNAME
|
||||||
|
port = 5432 # DATABASE_PORT
|
||||||
|
user = "vulcanize" # DATABASE_USER
|
||||||
|
password = "..." # DATABASE_PASSWORD
|
||||||
|
driver = "sqlx" # DATABASE_DRIVER_TYPE <sqlx | pgx>
|
||||||
|
|
||||||
|
# with file type
|
||||||
|
# file mode <sql | csv>
|
||||||
|
fileMode = "csv" # DATABASE_FILE_MODE
|
||||||
|
|
||||||
|
# with SQL file mode
|
||||||
|
filePath = "" # DATABASE_FILE_PATH
|
||||||
|
|
||||||
|
# with CSV file mode
|
||||||
|
fileCsvDir = "output_dir" # DATABASE_FILE_CSV_DIR
|
||||||
|
|
||||||
|
# with dump type
|
||||||
|
# <stdout | stderr | discard>
|
||||||
|
dumpDestination = "" # DATABASE_DUMP_DST
|
||||||
|
|
||||||
|
[cache]
|
||||||
|
database = 1024 # DB_CACHE_SIZE_MB
|
||||||
|
trie = 1024 # TRIE_CACHE_SIZE_MB
|
||||||
|
|
||||||
|
[prom]
|
||||||
|
# prometheus metrics
|
||||||
|
metrics = true # PROM_METRICS
|
||||||
|
http = true # PROM_HTTP
|
||||||
|
httpAddr = "localhost" # PROM_HTTP_ADDR
|
||||||
|
httpPort = "8889" # PROM_HTTP_PORT
|
||||||
|
|
||||||
|
[ethereum]
|
||||||
|
# node info
|
||||||
|
nodeID = "" # ETH_NODE_ID
|
||||||
|
clientName = "eth-statediff-service" # ETH_CLIENT_NAME
|
||||||
|
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # ETH_GENESIS_BLOCK
|
||||||
|
networkID = 1 # ETH_NETWORK_ID
|
||||||
|
chainID = 1 # ETH_CHAIN_ID
|
||||||
|
|
||||||
|
# path to custom chain config file (optional)
|
||||||
|
# keep chainID same as that in chain config file
|
||||||
|
chainConfig = "./chain.json" # ETH_CHAIN_CONFIG
|
||||||
|
```
|
||||||
|
|
||||||
### Local Setup
|
### Local Setup
|
||||||
|
|
||||||
* Create a chain config file `chain.json` according to chain config in genesis json file used by local geth.
|
* Create a chain config file `chain.json` according to chain config in genesis json file used by local geth.
|
||||||
@ -42,52 +147,19 @@ make build
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
* Change the following in [config file](./environments/config.toml)
|
Provide the path to the above file in the config.
|
||||||
|
|
||||||
```toml
|
|
||||||
[leveldb]
|
|
||||||
mode = "local"
|
|
||||||
# Path to geth LevelDB data
|
|
||||||
path = "/path-to-local-geth-data/chaindata"
|
|
||||||
ancient = "/path-to-local-geth-data/chaindata/ancient"
|
|
||||||
|
|
||||||
[ethereum]
|
|
||||||
chainConfig = "./chain.json" # Path to custom chain config file
|
|
||||||
chainID = 41337 # Same chain ID as in chain.json
|
|
||||||
|
|
||||||
[database]
|
|
||||||
# Update database config
|
|
||||||
name = "vulcanize_testing"
|
|
||||||
hostname = "localhost"
|
|
||||||
port = 5432
|
|
||||||
user = "postgres"
|
|
||||||
password = "postgres"
|
|
||||||
type = "postgres"
|
|
||||||
```
|
|
||||||
|
|
||||||
* To write statediff for a range of block make changes in [config file](./environments/config.toml)
|
|
||||||
```toml
|
|
||||||
[prerun]
|
|
||||||
only = false
|
|
||||||
ranges = [
|
|
||||||
[8, 15] # Block number range for which to write statediff.
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
* To use remote LevelDB RPC endpoint change the following in [config file](./environments/config.toml)
|
|
||||||
```toml
|
|
||||||
[leveldb]
|
|
||||||
mode = "remote"
|
|
||||||
url = "http://127.0.0.1:8082/" # Remote LevelDB RPC url
|
|
||||||
```
|
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
|
* Create / update the config file (refer to example config above).
|
||||||
|
|
||||||
### `serve`
|
### `serve`
|
||||||
|
|
||||||
To serve state diffs over RPC:
|
* To serve state diffs over RPC:
|
||||||
|
|
||||||
`eth-statediff-service serve --config=<config path>`
|
```bash
|
||||||
|
./eth-statediff-service serve --config=<config path>
|
||||||
|
```
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
|
|
||||||
@ -95,91 +167,108 @@ Example:
|
|||||||
./eth-statediff-service serve --config environments/config.toml
|
./eth-statediff-service serve --config environments/config.toml
|
||||||
```
|
```
|
||||||
|
|
||||||
Available RPC methods are:
|
* Available RPC methods are:
|
||||||
* `statediff_stateTrieAt()`
|
* `statediff_stateTrieAt()`
|
||||||
* `statediff_streamCodeAndCodeHash()`
|
* `statediff_streamCodeAndCodeHash()`
|
||||||
* `statediff_stateDiffAt()`
|
* `statediff_stateDiffAt()`
|
||||||
* `statediff_writeStateDiffAt()`
|
* `statediff_writeStateDiffAt()`
|
||||||
* `statediff_writeStateDiffsInRange()`
|
* `statediff_writeStateDiffsInRange()`
|
||||||
|
|
||||||
e.g. `curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"intermediateStateNodes":true,"intermediateStorageNodes":true,"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT"`
|
Example:
|
||||||
|
|
||||||
The process can be configured locally with sets of ranges to process as a "prerun" to processing directed by the server endpoints.
|
```bash
|
||||||
This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining ranged and params in the
|
curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"intermediateStateNodes":true,"intermediateStorageNodes":true,"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT"
|
||||||
`prerun` section of the config as shown below.
|
```
|
||||||
|
|
||||||
## Configuration
|
* Prerun:
|
||||||
|
* The process can be configured locally with sets of ranges to process as a "prerun" to processing directed by the server endpoints.
|
||||||
An example config file:
|
* This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining ranges and params in the
|
||||||
|
`prerun` section of the config.
|
||||||
```toml
|
* Set the range using `prerun.start` and `prerun.stop`. Use `prerun.ranges` if prerun on more than one range is required.
|
||||||
[leveldb]
|
* Currently, `prerun.params.includeTD` must be set to `true`.
|
||||||
mode = "local"
|
|
||||||
# path and ancient LevelDB paths required in local mode
|
## Import output data in file mode into a database
|
||||||
path = "/Users/user/Library/Ethereum/geth/chaindata"
|
|
||||||
ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
|
* When `eth-statediff-service` is run in file mode (`database.type`) the output is in form of a SQL file or multiple CSV files.
|
||||||
# url for leveldb-ethdb-rpc endpoint required in remote mode
|
|
||||||
url = "http://127.0.0.1:8082/"
|
* Assuming the output files are located in `./output_dir` directory, if the DB is running in docker we need to mount the directory containing the files as a volume in the DB service. Eg:
|
||||||
|
|
||||||
[server]
|
```yaml
|
||||||
ipcPath = ".ipc"
|
# In docker-compose file
|
||||||
httpPath = "127.0.0.1:8545"
|
services:
|
||||||
|
ipld-eth-db:
|
||||||
[statediff]
|
volumes:
|
||||||
prerun = true
|
- ./output_dir:/output_dir
|
||||||
serviceWorkers = 1
|
```
|
||||||
workerQueueSize = 1024
|
|
||||||
trieWorkers = 4
|
* Start `bash` in the DB container to run import commands:
|
||||||
|
|
||||||
[prerun]
|
```bash
|
||||||
only = false
|
docker exec -it <CONTAINER_ID> bash
|
||||||
ranges = [
|
```
|
||||||
[0, 1000]
|
|
||||||
]
|
### SQL
|
||||||
[prerun.params]
|
|
||||||
intermediateStateNodes = true
|
* De-duplicate data:
|
||||||
intermediateStorageNodes = true
|
|
||||||
includeBlock = true
|
```bash
|
||||||
includeReceipts = true
|
sort -u output_dir/statediff.sql -o output_dir/statediff.sql
|
||||||
includeTD = true
|
```
|
||||||
includeCode = true
|
|
||||||
watchedAddresses = []
|
* Run the following to import data:
|
||||||
|
|
||||||
[log]
|
```bash
|
||||||
file = ""
|
psql -U <DATABASE_USER> <DATABASE_NAME> --set ON_ERROR_STOP=on -f /output_dir/statediff.sql
|
||||||
level = "info"
|
```
|
||||||
|
|
||||||
[eth]
|
### CSV
|
||||||
chainID = 1
|
|
||||||
|
- De-duplicate data:
|
||||||
[database]
|
|
||||||
name = "vulcanize_test"
|
```bash
|
||||||
hostname = "localhost"
|
# public.blocks
|
||||||
port = 5432
|
sort -u output_dir/public.blocks.csv -o output_dir/public.blocks.csv
|
||||||
user = "vulcanize"
|
```
|
||||||
password = "..."
|
|
||||||
type = "postgres"
|
* Run `psql`:
|
||||||
driver = "sqlx"
|
|
||||||
dumpDestination = ""
|
```bash
|
||||||
fileMode = "csv"
|
psql -U <DATABASE_USER> <DATABASE_NAME>
|
||||||
fileCsvDir = ""
|
```
|
||||||
filePath = ""
|
|
||||||
|
* Run the following to import data:
|
||||||
[cache]
|
|
||||||
database = 1024
|
```bash
|
||||||
trie = 1024
|
# public.nodes
|
||||||
|
COPY public.nodes FROM '/output_dir/public.nodes.csv' CSV;
|
||||||
[prom]
|
|
||||||
dbStats = false
|
# public.nodes
|
||||||
metrics = true
|
COPY public.blocks FROM '/output_dir/public.blocks.csv' CSV;
|
||||||
http = true
|
|
||||||
httpAddr = "localhost"
|
# eth.header_cids
|
||||||
httpPort = "8889"
|
COPY eth.header_cids FROM '/output_dir/eth.header_cids.csv' CSV;
|
||||||
|
|
||||||
[ethereum]
|
# eth.uncle_cids
|
||||||
nodeID = ""
|
COPY eth.uncle_cids FROM '/output_dir/eth.uncle_cids.csv' CSV;
|
||||||
clientName = "eth-statediff-service"
|
|
||||||
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
|
# eth.transaction_cids
|
||||||
networkID = 1
|
COPY eth.transaction_cids FROM '/output_dir/eth.transaction_cids.csv' CSV FORCE NOT NULL dst;
|
||||||
chainID = 1
|
|
||||||
|
# eth.access_list_elements
|
||||||
|
COPY eth.access_list_elements FROM '/output_dir/eth.access_list_elements.csv' CSV;
|
||||||
|
|
||||||
|
# eth.receipt_cids
|
||||||
|
COPY eth.receipt_cids FROM '/output_dir/eth.receipt_cids.csv' CSV FORCE NOT NULL post_state, contract, contract_hash;
|
||||||
|
|
||||||
|
# eth.log_cids
|
||||||
|
COPY eth.log_cids FROM '/output_dir/eth.log_cids.csv' CSV FORCE NOT NULL topic0, topic1, topic2, topic3;
|
||||||
|
|
||||||
|
# eth.state_cids
|
||||||
|
COPY eth.state_cids FROM '/output_dir/eth.state_cids.csv' CSV FORCE NOT NULL state_leaf_key;
|
||||||
|
|
||||||
|
# eth.storage_cids
|
||||||
|
COPY eth.storage_cids FROM '/output_dir/eth.storage_cids.csv' CSV FORCE NOT NULL storage_leaf_key;
|
||||||
|
|
||||||
|
# eth.state_accounts
|
||||||
|
COPY eth.state_accounts FROM '/output_dir/eth.state_accounts.csv' CSV;
|
||||||
```
|
```
|
||||||
@ -2,7 +2,6 @@
|
|||||||
mode = "local"
|
mode = "local"
|
||||||
path = "/app/geth-rw/chaindata"
|
path = "/app/geth-rw/chaindata"
|
||||||
ancient = "/app/geth-rw/chaindata/ancient"
|
ancient = "/app/geth-rw/chaindata/ancient"
|
||||||
url = "http://127.0.0.1:8082/"
|
|
||||||
|
|
||||||
[server]
|
[server]
|
||||||
ipcPath = ""
|
ipcPath = ""
|
||||||
@ -31,33 +30,28 @@
|
|||||||
level = "info"
|
level = "info"
|
||||||
|
|
||||||
[database]
|
[database]
|
||||||
|
type = "postgres"
|
||||||
name = ""
|
name = ""
|
||||||
hostname = ""
|
hostname = ""
|
||||||
port = 5432
|
port = 5432
|
||||||
user = ""
|
user = ""
|
||||||
password = ""
|
password = ""
|
||||||
type = "postgres"
|
|
||||||
driver = "sqlx"
|
driver = "sqlx"
|
||||||
dumpDestination = ""
|
|
||||||
fileMode = "csv"
|
|
||||||
fileCsvDir = ""
|
|
||||||
filePath = ""
|
|
||||||
|
|
||||||
[cache]
|
[cache]
|
||||||
database = 1024
|
database = 1024
|
||||||
trie = 4096
|
trie = 4096
|
||||||
|
|
||||||
[prom]
|
[prom]
|
||||||
dbStats = false
|
|
||||||
metrics = true
|
metrics = true
|
||||||
http = true
|
http = true
|
||||||
httpAddr = "0.0.0.0"
|
httpAddr = "0.0.0.0"
|
||||||
httpPort = 9100
|
httpPort = 9100
|
||||||
|
|
||||||
[ethereum]
|
[ethereum]
|
||||||
chainConfig = ""
|
|
||||||
nodeID = ""
|
nodeID = ""
|
||||||
clientName = "eth-statediff-service"
|
clientName = "eth-statediff-service"
|
||||||
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
|
genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
|
||||||
networkID = 1
|
networkID = 1
|
||||||
chainID = 1
|
chainID = 1
|
||||||
|
chainConfig = ""
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user