Compare commits
19
Commits
v4.1.2-alpha
..
v4
| Author | SHA1 | Date | |
|---|---|---|---|
 dboreham
|
46cd8b1834 | ||
|
dboreham
|
c0cd87ba6a | ||
 prathamesh0
|
2db235f244 | ||
|
dboreham
|
a827c4a36b | ||
|
dboreham
|
14b9c169bc | ||
|
dboreham
|
a2772762e1 | ||
|
dboreham
|
b1440d9673 | ||
 Michael Shaw
|
67d8bced4f | ||
|
Michael Shaw
|
12f4810ced | ||
|
Michael Shaw
|
7a8d38c955 | ||
|
Michael Shaw
|
d09cd0afe6 | ||
|
Michael
|
bcca82eaa3 | ||
|
Michael
|
dad77b561d | ||
|
Michael Shaw
|
c939822a95 | ||
|
dboreham
|
6d103cb1f1 | ||
|
dboreham
|
99f84b6fe6 | ||
|
dboreham
|
5b7f5feb1b | ||
|
dboreham
|
7f8885f044 | ||
|
dboreham
|
d235f3b84c |
@@ -32,38 +32,42 @@ jobs:
|
||||
needs: run-tests
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Get the version
|
||||
id: vars
|
||||
run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
|
||||
- name: Run docker build
|
||||
run: make docker-build
|
||||
- name: Tag docker image
|
||||
run: docker tag vulcanize/eth-statediff-service docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
|
||||
- name: Docker Login
|
||||
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
|
||||
- name: Docker Push
|
||||
run: docker push docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
|
||||
push_to_registries:
|
||||
name: Push Docker image to Docker Hub
|
||||
runs-on: ubuntu-latest
|
||||
if: |
|
||||
always() &&
|
||||
(needs.build.result == 'success') &&
|
||||
github.event_name == 'release'
|
||||
needs: build
|
||||
steps:
|
||||
- name: Get the version
|
||||
id: vars
|
||||
run: |
|
||||
echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
|
||||
echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
|
||||
- name: Docker Login to Github Registry
|
||||
run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
|
||||
- name: Docker Pull
|
||||
run: docker pull docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
|
||||
- name: Docker Login to Docker Registry
|
||||
run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
|
||||
- name: Run docker build
|
||||
run: make docker-build
|
||||
- name: Tag docker image
|
||||
run: docker tag docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} vulcanize/eth-statediff-service:${{steps.vars.outputs.tag}}
|
||||
- name: Docker Push to Docker Hub
|
||||
run: docker push vulcanize/eth-statediff-service:${{steps.vars.outputs.tag}}
|
||||
run: docker tag cerc-io/eth-statediff-service git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
|
||||
- name: Tag docker image TAG
|
||||
run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
|
||||
- name: Docker Login
|
||||
run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
|
||||
- name: Docker Push
|
||||
run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
|
||||
# push_to_registries:
|
||||
# name: Push Docker image to Docker Hub
|
||||
# runs-on: ubuntu-latest
|
||||
# if: |
|
||||
# always() &&
|
||||
# (needs.build.result == 'success') &&
|
||||
# github.event_name == 'release'
|
||||
# needs: build
|
||||
# steps:
|
||||
# - name: Get the version
|
||||
# id: vars
|
||||
# run: |
|
||||
# echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
|
||||
# echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
|
||||
# - name: Docker Login to Github Registry
|
||||
# run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
|
||||
# - name: Docker Pull
|
||||
# run: docker pull docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
|
||||
# - name: Docker Login to Docker Registry
|
||||
# run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
|
||||
# - name: Tag docker image
|
||||
# run: docker tag docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
|
||||
# - name: Docker Push to Docker Hub
|
||||
# run: docker push cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
|
||||
|
||||
+7
-7
@@ -5,10 +5,10 @@ RUN apk --update --no-cache add make git g++ linux-headers
|
||||
RUN apk add busybox-extras
|
||||
|
||||
# Get and build ipfs-blockchain-watcher
|
||||
ADD . /go/src/github.com/vulcanize/eth-statediff-service
|
||||
#RUN git clone https://github.com/vulcanize/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
|
||||
ADD . /go/src/github.com/cerc-io/eth-statediff-service
|
||||
#RUN git clone https://github.com/cerc-io/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
|
||||
|
||||
WORKDIR /go/src/github.com/vulcanize/eth-statediff-service
|
||||
WORKDIR /go/src/github.com/cerc-io/eth-statediff-service
|
||||
RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .
|
||||
|
||||
# app container
|
||||
@@ -27,12 +27,12 @@ USER $USER
|
||||
|
||||
# chown first so dir is writable
|
||||
# note: using $USER is merged, but not in the stable release yet
|
||||
COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/$CONFIG_FILE config.toml
|
||||
COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/startup_script.sh .
|
||||
COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/environments environments
|
||||
COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/$CONFIG_FILE config.toml
|
||||
COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/startup_script.sh .
|
||||
COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/environments environments
|
||||
|
||||
# keep binaries immutable
|
||||
COPY --from=builder /go/src/github.com/vulcanize/eth-statediff-service/eth-statediff-service eth-statediff-service
|
||||
COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/eth-statediff-service eth-statediff-service
|
||||
|
||||
EXPOSE $EXPOSE_PORT
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
## Build docker image
|
||||
.PHONY: docker-build
|
||||
docker-build:
|
||||
docker build -t vulcanize/eth-statediff-service .
|
||||
docker build -t cerc-io/eth-statediff-service .
|
||||
|
||||
.PHONY: test
|
||||
test: | $(GOOSE)
|
||||
test:
|
||||
go test -p 1 ./pkg/... -v
|
||||
|
||||
build:
|
||||
|
||||
@@ -8,7 +8,7 @@ Purpose:
|
||||
|
||||
Stand up a statediffing service directly on top of a go-ethereum LevelDB instance.
|
||||
This service can serve historical state data over the same rpc interface as
|
||||
[statediffing geth](https://github.com/vulcanize/go-ethereum) without needing to run a full node.
|
||||
[statediffing geth](https://github.com/cerc-io/go-ethereum) without needing to run a full node.
|
||||
|
||||
## Setup
|
||||
|
||||
@@ -47,7 +47,8 @@ An example config file:
|
||||
trieWorkers = 4 # STATEDIFF_TRIE_WORKERS
|
||||
|
||||
[prerun]
|
||||
only = false # PRERUN_ONLY
|
||||
only = false # PRERUN_ONLY
|
||||
parallel = true # PRERUN_PARALLEL
|
||||
|
||||
# to perform prerun in a specific range (optional)
|
||||
start = 0 # PRERUN_RANGE_START
|
||||
@@ -122,6 +123,9 @@ An example config file:
|
||||
# path to custom chain config file (optional)
|
||||
# keep chainID same as that in chain config file
|
||||
chainConfig = "./chain.json" # ETH_CHAIN_CONFIG
|
||||
|
||||
[debug]
|
||||
pprof = false # DEBUG_PPROF
|
||||
```
|
||||
|
||||
### Local Setup
|
||||
@@ -216,6 +220,8 @@ An example config file:
|
||||
|
||||
* When `eth-statediff-service` is run in file mode (`database.type`) the output is in form of a SQL file or multiple CSV files.
|
||||
|
||||
### SQL
|
||||
|
||||
* Assuming the output files are located in host's `./output_dir` directory.
|
||||
|
||||
* Create a directory to store post-processed output:
|
||||
@@ -224,7 +230,11 @@ An example config file:
|
||||
mkdir -p output_dir/processed_output
|
||||
```
|
||||
|
||||
### SQL
|
||||
* (Optional) Get row counts in the output:
|
||||
|
||||
```bash
|
||||
wc -l output_dir/statediff.sql > output_stats.txt
|
||||
```
|
||||
|
||||
* De-duplicate data:
|
||||
|
||||
@@ -242,86 +252,48 @@ An example config file:
|
||||
|
||||
### CSV
|
||||
|
||||
* De-duplicate data and copy to post-processed output directory:
|
||||
* Create an env file with the required variables. Refer [.sample.env](./scripts/.sample.env).
|
||||
|
||||
* (Optional) Get row counts in the output:
|
||||
|
||||
```bash
|
||||
# public.blocks
|
||||
sort -u output_dir/public.blocks.csv -o output_dir/processed_output/deduped-public.blocks.csv
|
||||
|
||||
# eth.header_cids
|
||||
sort -u output_dir/eth.header_cids.csv -o output_dir/processed_output/deduped-eth.header_cids.csv
|
||||
|
||||
# eth.uncle_cids
|
||||
sort -u output_dir/eth.uncle_cids.csv -o output_dir/processed_output/deduped-eth.uncle_cids.csv
|
||||
|
||||
# eth.transaction_cids
|
||||
sort -u output_dir/eth.transaction_cids.csv -o output_dir/processed_output/deduped-eth.transaction_cids.csv
|
||||
|
||||
# eth.access_list_elements
|
||||
sort -u output_dir/eth.access_list_elements.csv -o output_dir/processed_output/deduped-eth.access_list_elements.csv
|
||||
|
||||
# eth.receipt_cids
|
||||
sort -u output_dir/eth.receipt_cids.csv -o output_dir/processed_output/deduped-eth.receipt_cids.csv
|
||||
|
||||
# eth.log_cids
|
||||
sort -u output_dir/eth.log_cids.csv -o output_dir/processed_output/deduped-eth.log_cids.csv
|
||||
|
||||
# eth.state_cids
|
||||
sort -u output_dir/eth.state_cids.csv -o output_dir/processed_output/deduped-eth.state_cids.csv
|
||||
|
||||
# eth.storage_cids
|
||||
sort -u output_dir/eth.storage_cids.csv -o output_dir/processed_output/deduped-eth.storage_cids.csv
|
||||
|
||||
# eth.state_accounts
|
||||
sort -u output_dir/eth.state_accounts.csv -o output_dir/processed_output/deduped-eth.state_accounts.csv
|
||||
|
||||
# public.nodes
|
||||
cp output_dir/public.nodes.csv output_dir/processed_output/public.nodes.csv
|
||||
./scripts/count-lines.sh <ENV_FILE_PATH>
|
||||
```
|
||||
|
||||
* Copy over the post-processed output files to the DB server (say in `/output_dir`).
|
||||
|
||||
* Start `psql` to run the import commands:
|
||||
* De-duplicate data:
|
||||
|
||||
```bash
|
||||
psql -U <DATABASE_USER> -h <DATABASE_HOSTNAME> -p <DATABASE_PORT> <DATABASE_NAME>
|
||||
./scripts/dedup.sh <ENV_FILE_PATH>
|
||||
```
|
||||
|
||||
* Run the following to import data:
|
||||
* Perform column checks:
|
||||
|
||||
```bash
|
||||
# public.nodes
|
||||
COPY public.nodes FROM '/output_dir/processed_output/public.nodes.csv' CSV;
|
||||
./scripts/check-columns.sh <ENV_FILE_PATH>
|
||||
```
|
||||
|
||||
# public.nodes
|
||||
COPY public.blocks FROM '/output_dir/processed_output/deduped-public.blocks.csv' CSV;
|
||||
Check the output logs for any rows detected with unexpected number of columns.
|
||||
|
||||
# eth.header_cids
|
||||
COPY eth.header_cids FROM '/output_dir/processed_output/deduped-eth.header_cids.csv' CSV;
|
||||
Example:
|
||||
|
||||
# eth.uncle_cids
|
||||
COPY eth.uncle_cids FROM '/output_dir/processed_output/deduped-eth.uncle_cids.csv' CSV;
|
||||
```bash
|
||||
# log
|
||||
eth.header_cids
|
||||
Start: Wednesday 21 September 2022 06:00:38 PM IST
|
||||
Time taken: 00:00:05
|
||||
End: Wednesday 21 September 2022 06:00:43 PM IST
|
||||
Total bad rows: 1 ./check-columns/eth.header_cids.txt
|
||||
|
||||
# eth.transaction_cids
|
||||
COPY eth.transaction_cids FROM '/output_dir/processed_output/deduped-eth.transaction_cids.csv' CSV FORCE NOT NULL dst;
|
||||
# bad row output
|
||||
# line number, num. of columns, data
|
||||
23 17 22,xxxxxx,0x07f5ea5c94aa8dea60b28f6b6315d92f2b6d78ca4b74ea409adeb191b5a114f2,0x5918487321aa57dd0c50977856c6231e7c4ee79e95b694c7c8830227d77a1ecc,bagiacgzaa726uxeuvkg6uyfsr5vwgfozf4vw26gkjn2ouqe232yzdnnbctza,45,geth,0,0xad8fa8df61b98dbda7acd6ca76d5ce4cbba663d5f608cc940957adcdb94cee8d,0xc621412320a20b4aaff5363bdf063b9d13e394ef82e55689ab703aae5db08e26,0x71ec1c7d81269ce115be81c81f13e1cc2601c292a7f20440a77257ecfdc69940,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,\x2000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000,1658408419,/blocks/DMQAP5PKLSKKVDPKMCZI623DCXMS6K3NPDFEW5HKICNN5MMRWWQRJ4Q,1,0x0000000000000000000000000000000000000000
|
||||
```
|
||||
|
||||
# eth.access_list_elements
|
||||
COPY eth.access_list_elements FROM '/output_dir/processed_output/deduped-eth.access_list_elements.csv' CSV;
|
||||
* Import data using `timescaledb-parallel-copy`:
|
||||
(requires [`timescaledb-parallel-copy`](https://github.com/timescale/timescaledb-parallel-copy) installation; readily comes with TimescaleDB docker image)
|
||||
|
||||
# eth.receipt_cids
|
||||
COPY eth.receipt_cids FROM '/output_dir/processed_output/deduped-eth.receipt_cids.csv' CSV FORCE NOT NULL post_state, contract, contract_hash;
|
||||
|
||||
# eth.log_cids
|
||||
COPY eth.log_cids FROM '/output_dir/processed_output/deduped-eth.log_cids.csv' CSV FORCE NOT NULL topic0, topic1, topic2, topic3;
|
||||
|
||||
# eth.state_cids
|
||||
COPY eth.state_cids FROM '/output_dir/processed_output/deduped-eth.state_cids.csv' CSV FORCE NOT NULL state_leaf_key;
|
||||
|
||||
# eth.storage_cids
|
||||
COPY eth.storage_cids FROM '/output_dir/processed_output/deduped-eth.storage_cids.csv' CSV FORCE NOT NULL storage_leaf_key;
|
||||
|
||||
# eth.state_accounts
|
||||
COPY eth.state_accounts FROM '/output_dir/processed_output/deduped-eth.state_accounts.csv' CSV;
|
||||
```bash
|
||||
./scripts/timescaledb-import.sh <ENV_FILE_PATH>
|
||||
```
|
||||
|
||||
* NOTE: `COPY` command on CSVs inserts empty strings as `NULL` in the DB. Passing `FORCE_NOT_NULL <COLUMN_NAME>` forces it to insert empty strings instead. This is required to maintain compatibility of the imported statediff data with the data generated in `postgres` mode. Reference: https://www.postgresql.org/docs/14/sql-copy.html
|
||||
|
||||
@@ -50,6 +50,7 @@ const (
|
||||
PROM_DB_STATS = "PROM_DB_STATS"
|
||||
|
||||
PRERUN_ONLY = "PRERUN_ONLY"
|
||||
PRERUN_PARALLEL = "PRERUN_PARALLEL"
|
||||
PRERUN_RANGE_START = "PRERUN_RANGE_START"
|
||||
PRERUN_RANGE_STOP = "PRERUN_RANGE_STOP"
|
||||
PRERUN_INTERMEDIATE_STATE_NODES = "PRERUN_INTERMEDIATE_STATE_NODES"
|
||||
@@ -81,6 +82,8 @@ const (
|
||||
DATABASE_MAX_CONN_LIFETIME = "DATABASE_MAX_CONN_LIFETIME"
|
||||
DATABASE_CONN_TIMEOUT = "DATABSE_CONN_TIMEOUT"
|
||||
DATABASE_MAX_CONN_IDLE_TIME = "DATABASE_MAX_CONN_IDLE_TIME"
|
||||
|
||||
DEBUG_PPROF = "DEBUG_PPROF"
|
||||
)
|
||||
|
||||
// Bind env vars for eth node and DB configuration
|
||||
@@ -135,6 +138,7 @@ func init() {
|
||||
|
||||
viper.BindEnv("statediff.prerun", STATEDIFF_PRERUN)
|
||||
viper.BindEnv("prerun.only", PRERUN_ONLY)
|
||||
viper.BindEnv("prerun.parallel", PRERUN_PARALLEL)
|
||||
viper.BindEnv("prerun.start", PRERUN_RANGE_START)
|
||||
viper.BindEnv("prerun.stop", PRERUN_RANGE_STOP)
|
||||
viper.BindEnv("prerun.params.intermediateStateNodes", PRERUN_INTERMEDIATE_STATE_NODES)
|
||||
@@ -146,4 +150,6 @@ func init() {
|
||||
|
||||
viper.BindEnv("log.level", LOG_LEVEL)
|
||||
viper.BindEnv("log.file", LOG_FILE_PATH)
|
||||
|
||||
viper.BindEnv("debug.pprof", DEBUG_PPROF)
|
||||
}
|
||||
|
||||
+4
-1
@@ -34,7 +34,7 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"github.com/vulcanize/eth-statediff-service/pkg/prom"
|
||||
"github.com/cerc-io/eth-statediff-service/pkg/prom"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -221,6 +221,7 @@ func init() {
|
||||
viper.BindPFlag("prom.metrics", rootCmd.PersistentFlags().Lookup("prom-metrics"))
|
||||
|
||||
viper.BindPFlag("prerun.only", rootCmd.PersistentFlags().Lookup("prerun-only"))
|
||||
viper.BindPFlag("prerun.parallel", rootCmd.PersistentFlags().Lookup("prerun-parallel"))
|
||||
viper.BindPFlag("prerun.start", rootCmd.PersistentFlags().Lookup("prerun-start"))
|
||||
viper.BindPFlag("prerun.stop", rootCmd.PersistentFlags().Lookup("prerun-stop"))
|
||||
viper.BindPFlag("prerun.params.intermediateStateNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-state-nodes"))
|
||||
@@ -230,6 +231,8 @@ func init() {
|
||||
viper.BindPFlag("prerun.params.includeTD", rootCmd.PersistentFlags().Lookup("prerun-include-td"))
|
||||
viper.BindPFlag("prerun.params.includeCode", rootCmd.PersistentFlags().Lookup("prerun-include-code"))
|
||||
|
||||
viper.BindPFlag("debug.pprof", rootCmd.PersistentFlags().Lookup("debug-pprof"))
|
||||
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
|
||||
+27
-3
@@ -16,8 +16,11 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
@@ -25,8 +28,8 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
sd "github.com/vulcanize/eth-statediff-service/pkg"
|
||||
srpc "github.com/vulcanize/eth-statediff-service/pkg/rpc"
|
||||
sd "github.com/cerc-io/eth-statediff-service/pkg"
|
||||
srpc "github.com/cerc-io/eth-statediff-service/pkg/rpc"
|
||||
)
|
||||
|
||||
// serveCmd represents the serve command
|
||||
@@ -47,17 +50,38 @@ func init() {
|
||||
rootCmd.AddCommand(serveCmd)
|
||||
}
|
||||
|
||||
func maxParallelism() int {
|
||||
maxProcs := runtime.GOMAXPROCS(0)
|
||||
numCPU := runtime.NumCPU()
|
||||
if maxProcs < numCPU {
|
||||
return maxProcs
|
||||
}
|
||||
return numCPU
|
||||
}
|
||||
|
||||
func serve() {
|
||||
logWithCommand.Info("Running eth-statediff-service serve command")
|
||||
logWithCommand.Infof("Parallelism: %d", maxParallelism())
|
||||
|
||||
statediffService, err := createStateDiffService()
|
||||
if err != nil {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
|
||||
// Enable the pprof agent if configured
|
||||
if viper.GetBool("debug.pprof") {
|
||||
// See: https://www.farsightsecurity.com/blog/txt-record/go-remote-profiling-20161028/
|
||||
// For security reasons: do not use the default http multiplexor elsewhere in this process.
|
||||
go func() {
|
||||
logWithCommand.Info("Starting pprof listener on port 6060")
|
||||
logWithCommand.Fatal(http.ListenAndServe("localhost:6060", nil))
|
||||
}()
|
||||
}
|
||||
|
||||
// short circuit if we only want to perform prerun
|
||||
if viper.GetBool("prerun.only") {
|
||||
if err := statediffService.Run(nil); err != nil {
|
||||
parallel := viper.GetBool("prerun.parallel")
|
||||
if err := statediffService.Run(nil, parallel); err != nil {
|
||||
logWithCommand.Fatal("unable to perform prerun: %v", err)
|
||||
}
|
||||
return
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import (
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
sd "github.com/vulcanize/eth-statediff-service/pkg"
|
||||
"github.com/vulcanize/eth-statediff-service/pkg/prom"
|
||||
sd "github.com/cerc-io/eth-statediff-service/pkg"
|
||||
"github.com/cerc-io/eth-statediff-service/pkg/prom"
|
||||
)
|
||||
|
||||
type blockRange [2]uint64
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ import (
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
v "github.com/vulcanize/eth-statediff-service/version"
|
||||
v "github.com/cerc-io/eth-statediff-service/version"
|
||||
)
|
||||
|
||||
// versionCmd represents the version command
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
module github.com/vulcanize/eth-statediff-service
|
||||
module github.com/cerc-io/eth-statediff-service
|
||||
|
||||
go 1.18
|
||||
|
||||
require (
|
||||
github.com/ethereum/go-ethereum v1.10.21
|
||||
github.com/cerc-io/go-eth-state-node-iterator v1.1.7
|
||||
github.com/ethereum/go-ethereum v1.10.23
|
||||
github.com/jmoiron/sqlx v1.2.0 // indirect
|
||||
github.com/prometheus/client_golang v1.4.0
|
||||
github.com/sirupsen/logrus v1.7.0
|
||||
github.com/sirupsen/logrus v1.9.0
|
||||
github.com/spf13/cobra v1.3.0
|
||||
github.com/spf13/viper v1.10.1
|
||||
github.com/vulcanize/go-eth-state-node-iterator v1.1.4
|
||||
github.com/vulcanize/leveldb-ethdb-rpc v0.1.6
|
||||
github.com/vulcanize/leveldb-ethdb-rpc v0.1.7
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -70,7 +70,7 @@ require (
|
||||
github.com/jackpal/go-nat-pmp v1.0.2 // indirect
|
||||
github.com/jbenet/goprocess v0.1.4 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
|
||||
github.com/lib/pq v1.10.2 // indirect
|
||||
github.com/lib/pq v1.10.6 // indirect
|
||||
github.com/magiconair/properties v1.8.5 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
@@ -121,10 +121,10 @@ require (
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
go.uber.org/zap v1.17.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
|
||||
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
|
||||
golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
|
||||
google.golang.org/protobuf v1.27.1 // indirect
|
||||
@@ -135,4 +135,4 @@ require (
|
||||
lukechampine.com/blake3 v1.1.6 // indirect
|
||||
)
|
||||
|
||||
replace github.com/ethereum/go-ethereum v1.10.21 => github.com/vulcanize/go-ethereum v1.10.21-statediff-4.1.2-alpha
|
||||
replace github.com/ethereum/go-ethereum v1.10.23 => github.com/cerc-io/go-ethereum v1.10.23-statediff-4.2.0-alpha
|
||||
|
||||
@@ -86,6 +86,10 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOF
|
||||
github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cerc-io/go-eth-state-node-iterator v1.1.7 h1:i4zCKkfowJ4aUVpqVyMpOZIbD5WVm/iijhMAPgcLJ44=
|
||||
github.com/cerc-io/go-eth-state-node-iterator v1.1.7/go.mod h1:BBh4BetpBoHUbAiKAdatR3ij7IzDlgN9nPzZ7snn/lo=
|
||||
github.com/cerc-io/go-ethereum v1.10.23-statediff-4.2.0-alpha h1:Na4mum/Qkw1mRHnq/TIRPIF5exZAEBznVs3G3/YSENo=
|
||||
github.com/cerc-io/go-ethereum v1.10.23-statediff-4.2.0-alpha/go.mod h1:lKBVBWksSwBDR/5D9CAxaGQzDPIS3ueWb6idy7X1Shg=
|
||||
github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
@@ -190,8 +194,8 @@ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU
|
||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
@@ -519,8 +523,9 @@ github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
|
||||
github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
|
||||
github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
|
||||
github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
|
||||
github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
|
||||
@@ -695,8 +700,8 @@ github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
|
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
@@ -753,12 +758,8 @@ github.com/urfave/cli/v2 v2.10.2/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhA
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
|
||||
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
|
||||
github.com/vulcanize/go-eth-state-node-iterator v1.1.4 h1:2jIbgZ/yxrNZt5zmsXtvUPSxT0sP5k1PR5/O1GN/Ma0=
|
||||
github.com/vulcanize/go-eth-state-node-iterator v1.1.4/go.mod h1:kFzlSi7rUB9mJWZf08xIEiiGuXnnUEn48It+WMHzxPc=
|
||||
github.com/vulcanize/go-ethereum v1.10.21-statediff-4.1.2-alpha h1:ct+8FGuQnHA6SOGOQoAMINWdeexuSF40+IjF48J094A=
|
||||
github.com/vulcanize/go-ethereum v1.10.21-statediff-4.1.2-alpha/go.mod h1:dNJkmCSbaasX0zfQM6pm1g3rWlW3EGhLOEZMScyrRAs=
|
||||
github.com/vulcanize/leveldb-ethdb-rpc v0.1.6 h1:vkiplC4B8v5f0jTABLBFDn4HueYzbhYIBhjHuDFnPpU=
|
||||
github.com/vulcanize/leveldb-ethdb-rpc v0.1.6/go.mod h1:VRNHF03FBH+6Gg+nn4JfXZ6R1+K/+/Yf/Eeodsyv7KE=
|
||||
github.com/vulcanize/leveldb-ethdb-rpc v0.1.7 h1:iyaaNRNK2S6QNUwKlAzUbJFHEKyD7z9T74uCAgTHBI0=
|
||||
github.com/vulcanize/leveldb-ethdb-rpc v0.1.7/go.mod h1:gqXQ5dpFcb517/NYzIezqZIDOgLMBHYzP+A0Orqdblg=
|
||||
github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
|
||||
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
|
||||
@@ -824,8 +825,8 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP
|
||||
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
|
||||
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM=
|
||||
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@@ -1027,8 +1028,9 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -1166,7 +1168,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
package main
|
||||
|
||||
import "github.com/vulcanize/eth-statediff-service/cmd"
|
||||
import "github.com/cerc-io/eth-statediff-service/cmd"
|
||||
|
||||
func main() {
|
||||
cmd.Execute()
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ import (
|
||||
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
iter "github.com/vulcanize/go-eth-state-node-iterator"
|
||||
iter "github.com/cerc-io/go-eth-state-node-iterator"
|
||||
)
|
||||
|
||||
type builder struct {
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ import (
|
||||
"github.com/ethereum/go-ethereum/statediff/test_helpers"
|
||||
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
|
||||
|
||||
pkg "github.com/vulcanize/eth-statediff-service/pkg"
|
||||
pkg "github.com/cerc-io/eth-statediff-service/pkg"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
+1
-1
@@ -24,7 +24,7 @@ import (
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vulcanize/eth-statediff-service/pkg/prom"
|
||||
"github.com/cerc-io/eth-statediff-service/pkg/prom"
|
||||
)
|
||||
|
||||
// StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules.
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@ import (
|
||||
"github.com/ethereum/go-ethereum/rpc"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vulcanize/eth-statediff-service/pkg/prom"
|
||||
"github.com/cerc-io/eth-statediff-service/pkg/prom"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
+45
-8
@@ -33,7 +33,7 @@ import (
|
||||
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/vulcanize/eth-statediff-service/pkg/prom"
|
||||
"github.com/cerc-io/eth-statediff-service/pkg/prom"
|
||||
)
|
||||
|
||||
const defaultQueueSize = 1024
|
||||
@@ -48,7 +48,7 @@ type StateDiffService interface {
|
||||
// Loop is the main event loop for processing state diffs
|
||||
Loop(wg *sync.WaitGroup) error
|
||||
// Run is a one-off command to run on a predefined set of ranges
|
||||
Run(ranges []RangeRequest) error
|
||||
Run(ranges []RangeRequest, parallel bool) error
|
||||
// StateDiffAt method to get state diff object at specific block
|
||||
StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
|
||||
// StateDiffFor method to get state diff object at specific block
|
||||
@@ -118,18 +118,55 @@ func (sds *Service) APIs() []rpc.API {
|
||||
}
|
||||
|
||||
// Run does a one-off processing run on the provided RangeRequests + any pre-runs, exiting afterwards
|
||||
func (sds *Service) Run(rngs []RangeRequest) error {
|
||||
func (sds *Service) Run(rngs []RangeRequest, parallel bool) error {
|
||||
for _, preRun := range sds.preruns {
|
||||
logrus.Infof("processing prerun range (%d, %d)", preRun.Start, preRun.Stop)
|
||||
for i := preRun.Start; i <= preRun.Stop; i++ {
|
||||
if err := sds.WriteStateDiffAt(i, preRun.Params); err != nil {
|
||||
return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, preRun.Start, preRun.Stop, err)
|
||||
if parallel {
|
||||
// Chunk overall range into N subranges for workers
|
||||
chunkSize := (preRun.Stop - preRun.Start) / uint64(sds.workers)
|
||||
logrus.Infof("parallel processing prerun range (%d, %d) (%d blocks) divided into %d sized chunks with %d workers", preRun.Start, preRun.Stop,
|
||||
preRun.Stop-preRun.Start+1, chunkSize, sds.workers)
|
||||
// Sanity floor the chunk size
|
||||
if chunkSize < 100 {
|
||||
chunkSize = 100
|
||||
logrus.Infof("Computed range chunk size for each worker is too small, defaulting to 100")
|
||||
}
|
||||
wg := new(sync.WaitGroup)
|
||||
for i := 0; i < int(sds.workers); i++ {
|
||||
blockRange := RangeRequest{
|
||||
Start: preRun.Start + uint64(i)*chunkSize,
|
||||
Stop: preRun.Start + uint64(i)*chunkSize + chunkSize - 1,
|
||||
Params: preRun.Params,
|
||||
}
|
||||
// TODO(hack) this fixes quantization
|
||||
if blockRange.Stop < preRun.Stop && preRun.Stop-blockRange.Stop < chunkSize {
|
||||
blockRange.Stop = preRun.Stop
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
logrus.Infof("prerun worker %d processing range (%d, %d)", id, blockRange.Start, blockRange.Stop)
|
||||
for j := blockRange.Start; j <= blockRange.Stop; j++ {
|
||||
if err := sds.WriteStateDiffAt(j, blockRange.Params); err != nil {
|
||||
logrus.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, blockRange.Start, blockRange.Stop, err)
|
||||
}
|
||||
}
|
||||
logrus.Infof("prerun worker %d finished processing range (%d, %d)", id, blockRange.Start, blockRange.Stop)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
} else {
|
||||
logrus.Infof("sequential processing prerun range (%d, %d)", preRun.Start, preRun.Stop)
|
||||
for i := preRun.Start; i <= preRun.Stop; i++ {
|
||||
if err := sds.WriteStateDiffAt(i, preRun.Params); err != nil {
|
||||
return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, preRun.Start, preRun.Stop, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sds.preruns = nil
|
||||
// At present this code is never called so we have not written the parallel version:
|
||||
for _, rng := range rngs {
|
||||
logrus.Infof("processing prerun range (%d, %d)", rng.Start, rng.Stop)
|
||||
logrus.Infof("processing requested range (%d, %d)", rng.Start, rng.Stop)
|
||||
for i := rng.Start; i <= rng.Stop; i++ {
|
||||
if err := sds.WriteStateDiffAt(i, rng.Params); err != nil {
|
||||
return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, rng.Start, rng.Stop, err)
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# Used by the script to count rows (count-lines.sh)
|
||||
COUNT_LINES_LOG=./count-lines.log
|
||||
COUNT_LINES_INPUT_DIR=~/eth-statediff-service/output_dir
|
||||
COUNT_LINES_OUTPUT_FILE=./output-stats.txt
|
||||
|
||||
# Used by the script to dedup output files (dedup.sh)
|
||||
DEDUP_LOG=./dedup.log
|
||||
DEDUP_INPUT_DIR=~/eth-statediff-service/output_dir
|
||||
DEDUP_OUTPUT_DIR=~/eth-statediff-service/dedup_dir
|
||||
DEDUP_SORT_DIR=./.sort
|
||||
|
||||
# Used by the script to perform column checks (check-columns.sh)
|
||||
CHECK_COLUMNS_LOG=./check-columns.log
|
||||
CHECK_COLUMNS_INPUT_DIR=~/eth-statediff-service/output_dir
|
||||
CHECK_COLUMNS_INPUT_DEDUP_DIR=~/eth-statediff-service/dedup_dir
|
||||
CHECK_COLUMNS_OUTPUT_DIR=./check-columns
|
||||
|
||||
# Used by the script to import data (timescaledb-import.sh)
|
||||
IMPORT_LOG=./tsdb-import.log
|
||||
IMPORT_INPUT_DIR=~/eth-statediff-service/output_dir
|
||||
IMPORT_INPUT_DEDUP_DIR=~/eth-statediff-service/dedup_dir
|
||||
TIMESCALEDB_WORKERS=8
|
||||
|
||||
DATABASE_USER=vdbm
|
||||
DATABASE_HOSTNAME=localhost
|
||||
DATABASE_PORT=8077
|
||||
DATABASE_NAME=vulcanize_testing
|
||||
DATABASE_PASSWORD=password
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Requires:
|
||||
# CHECK_COLUMNS_LOG
|
||||
# CHECK_COLUMNS_INPUT_DIR
|
||||
# CHECK_COLUMNS_INPUT_DEDUP_DIR
|
||||
# CHECK_COLUMNS_OUTPUT_DIR
|
||||
|
||||
# env file arg
|
||||
ENV=$1
|
||||
echo "Using env file: ${ENV}"
|
||||
|
||||
# read env file
|
||||
export $(grep -v '^#' ${ENV} | xargs)
|
||||
|
||||
# redirect stdout/stderr to a file
|
||||
exec >"${CHECK_COLUMNS_LOG}" 2>&1
|
||||
|
||||
# create output dir if not exists
|
||||
mkdir -p "${CHECK_COLUMNS_OUTPUT_DIR}"
|
||||
|
||||
start_timestamp=$(date +%s)
|
||||
|
||||
declare -A expected_columns
|
||||
expected_columns=(
|
||||
["public.nodes"]="5"
|
||||
["public.blocks"]="3"
|
||||
# ["eth.access_list_elements"]="?" # skipping as values include ','
|
||||
["eth.log_cids"]="12"
|
||||
["eth.state_accounts"]="7"
|
||||
["eth.storage_cids"]="9"
|
||||
["eth.uncle_cids"]="7"
|
||||
["eth.header_cids"]="16"
|
||||
["eth.receipt_cids"]="10"
|
||||
["eth.state_cids"]="8"
|
||||
["eth.transaction_cids"]="11"
|
||||
)
|
||||
|
||||
for table_name in "${!expected_columns[@]}";
|
||||
do
|
||||
if [ "${table_name}" = "public.blocks" ];
|
||||
then
|
||||
command="$(dirname "$0")/find-bad-rows.sh -i ${CHECK_COLUMNS_INPUT_DEDUP_DIR}/deduped-${table_name}.csv -c ${expected_columns[${table_name}]} -d true -o ${CHECK_COLUMNS_OUTPUT_DIR}/${table_name}.txt"
|
||||
else
|
||||
command="$(dirname "$0")/find-bad-rows.sh -i ${CHECK_COLUMNS_INPUT_DIR}/${table_name}.csv -c ${expected_columns[${table_name}]} -d true -o ${CHECK_COLUMNS_OUTPUT_DIR}/${table_name}.txt"
|
||||
fi
|
||||
|
||||
echo "${table_name}"
|
||||
echo Start: "$(date)"
|
||||
eval "${command}"
|
||||
echo End: "$(date)"
|
||||
echo Total bad rows: $(wc -l ${CHECK_COLUMNS_OUTPUT_DIR}/${table_name}.txt)
|
||||
echo
|
||||
done
|
||||
|
||||
difference=$(($(date +%s)-start_timestamp))
|
||||
echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)
|
||||
echo
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Requires:
|
||||
# COUNT_LINES_LOG
|
||||
# COUNT_LINES_INPUT_DIR
|
||||
# COUNT_LINES_OUTPUT_FILE
|
||||
|
||||
# env file arg
|
||||
ENV=$1
|
||||
echo "Using env file: ${ENV}"
|
||||
|
||||
# read env file
|
||||
export $(grep -v '^#' ${ENV} | xargs)
|
||||
|
||||
# redirect stdout/stderr to a file
|
||||
exec >"${COUNT_LINES_LOG}" 2>&1
|
||||
|
||||
start_timestamp=$(date +%s)
|
||||
|
||||
table_names=(
|
||||
"public.nodes"
|
||||
"public.blocks"
|
||||
"eth.access_list_elements"
|
||||
"eth.log_cids"
|
||||
"eth.state_accounts"
|
||||
"eth.storage_cids"
|
||||
"eth.uncle_cids"
|
||||
"eth.header_cids"
|
||||
"eth.receipt_cids"
|
||||
"eth.state_cids"
|
||||
"eth.transaction_cids"
|
||||
)
|
||||
|
||||
echo "Row counts:" > "${COUNT_LINES_OUTPUT_FILE}"
|
||||
|
||||
for table_name in "${table_names[@]}";
|
||||
do
|
||||
echo "${table_name}";
|
||||
echo Start: "$(date)"
|
||||
wc -l "${COUNT_LINES_INPUT_DIR}"/"${table_name}.csv" >> "${COUNT_LINES_OUTPUT_FILE}"
|
||||
echo End: "$(date)"
|
||||
echo
|
||||
done
|
||||
|
||||
difference=$(($(date +%s)-start_timestamp))
|
||||
echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Requires:
|
||||
# DEDUP_LOG
|
||||
# DEDUP_INPUT_DIR
|
||||
# DEDUP_OUTPUT_DIR
|
||||
# DEDUP_SORT_DIR
|
||||
|
||||
# env file arg
|
||||
ENV=$1
|
||||
echo "Using env file: ${ENV}"
|
||||
|
||||
# read env file
|
||||
export $(grep -v '^#' ${ENV} | xargs)
|
||||
|
||||
# redirect stdout/stderr to a file
|
||||
exec >"${DEDUP_LOG}" 2>&1
|
||||
|
||||
# create output dir if not exists
|
||||
mkdir -p "${DEDUP_OUTPUT_DIR}"
|
||||
|
||||
start_timestamp=$(date +%s)
|
||||
|
||||
echo "public.blocks"
|
||||
echo Start: "$(date)"
|
||||
sort -T "${DEDUP_SORT_DIR}" -u "${DEDUP_INPUT_DIR}"/public.blocks.csv -o "${DEDUP_OUTPUT_DIR}"/deduped-public.blocks.csv
|
||||
echo End: "$(date)"
|
||||
echo Total deduped rows: $(wc -l ${DEDUP_OUTPUT_DIR}/deduped-public.blocks.csv)
|
||||
echo
|
||||
|
||||
difference=$(($(date +%s)-start_timestamp))
|
||||
echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)
|
||||
|
||||
# NOTE: This script currently only dedups public.blocks output file.
|
||||
# If the output contains blocks that were statediffed more than once, output files for other tables will have to be deduped as well.
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/bin/bash
|
||||
|
||||
# flags
|
||||
# -i <input-file>: Input data file path
|
||||
# -c <expected-columns>: Expected number of columns in each row of the input file
|
||||
# -o [output-file]: Output destination file path (default: STDOUT)
|
||||
# -d [include-data]: Whether to include the data row in output (true | false) (default: false)
|
||||
|
||||
# eg: ./scripts/find-bad-rows.sh -i eth.state_cids.csv -c 8 -o res.txt -d true
|
||||
# output: 1 9 1500000,xxxxxxxx,0x83952d392f9b0059eea94b10d1a095eefb1943ea91595a16c6698757127d4e1c,,
|
||||
# baglacgzasvqcntdahkxhufdnkm7a22s2eetj6mx6nzkarwxtkvy4x3bubdgq,\x0f,0,f,/blocks/,
|
||||
# DMQJKYBGZRQDVLT2CRWVGPQNNJNCCJU7GL7G4VAI3LZVK4OL5Q2ARTI
|
||||
|
||||
while getopts i:c:o:d: OPTION
|
||||
do
|
||||
case "${OPTION}" in
|
||||
i) inputFile=${OPTARG};;
|
||||
c) expectedColumns=${OPTARG};;
|
||||
o) outputFile=${OPTARG};;
|
||||
d) data=${OPTARG};;
|
||||
esac
|
||||
done
|
||||
|
||||
timestamp=$(date +%s)
|
||||
|
||||
# if data requested, dump row number, number of columns and the row
|
||||
if [ "${data}" = true ] ; then
|
||||
if [ -z "${outputFile}" ]; then
|
||||
awk -F"," "NF!=${expectedColumns} {print NR, NF, \$0}" < ${inputFile}
|
||||
else
|
||||
awk -F"," "NF!=${expectedColumns} {print NR, NF, \$0}" < ${inputFile} > ${outputFile}
|
||||
fi
|
||||
# else, dump only row number, number of columns
|
||||
else
|
||||
if [ -z "${outputFile}" ]; then
|
||||
awk -F"," "NF!=${expectedColumns} {print NR, NF}" < ${inputFile}
|
||||
else
|
||||
awk -F"," "NF!=${expectedColumns} {print NR, NF}" < ${inputFile} > ${outputFile}
|
||||
fi
|
||||
fi
|
||||
|
||||
difference=$(($(date +%s)-timestamp))
|
||||
echo Time taken: $(date -d@${difference} -u +%H:%M:%S)
|
||||
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Requires:
|
||||
# IMPORT_LOG
|
||||
# IMPORT_INPUT_DIR
|
||||
# IMPORT_INPUT_DEDUP_DIR
|
||||
# TIMESCALEDB_WORKERS
|
||||
# DATABASE_USER
|
||||
# DATABASE_HOSTNAME
|
||||
# DATABASE_PORT
|
||||
# DATABASE_NAME
|
||||
# DATABASE_PASSWORD
|
||||
|
||||
DEFAULT_TIMESCALEDB_WORKERS=8
|
||||
|
||||
# env file arg
|
||||
ENV=$1
|
||||
echo "Using env file: ${ENV}"
|
||||
|
||||
# read env file
|
||||
export $(grep -v '^#' ${ENV} | xargs)
|
||||
|
||||
if [ "$TIMESCALEDB_WORKERS" = "" ]; then
|
||||
TIMESCALEDB_WORKERS=$DEFAULT_TIMESCALEDB_WORKERS
|
||||
fi
|
||||
|
||||
# redirect stdout/stderr to a file
|
||||
exec >"${IMPORT_LOG}" 2>&1
|
||||
|
||||
start_timestamp=$(date +%s)
|
||||
|
||||
declare -a tables
|
||||
# schema-table-copyOptions
|
||||
tables=(
|
||||
"public-nodes"
|
||||
"public-blocks"
|
||||
"eth-access_list_elements"
|
||||
"eth-log_cids-FORCE NOT NULL topic0, topic1, topic2, topic3 CSV"
|
||||
"eth-state_accounts"
|
||||
"eth-storage_cids-FORCE NOT NULL storage_leaf_key CSV"
|
||||
"eth-uncle_cids"
|
||||
"eth-header_cids"
|
||||
"eth-receipt_cids-FORCE NOT NULL post_state, contract, contract_hash CSV"
|
||||
"eth-state_cids-FORCE NOT NULL state_leaf_key CSV"
|
||||
"eth-transaction_cids-FORCE NOT NULL dst CSV"
|
||||
)
|
||||
|
||||
for elem in "${tables[@]}";
|
||||
do
|
||||
IFS='-' read -a arr <<< "${elem}"
|
||||
|
||||
if [ "${arr[0]}.${arr[1]}" = "public.blocks" ];
|
||||
then
|
||||
copy_command="timescaledb-parallel-copy --connection \"host=${DATABASE_HOSTNAME} port=${DATABASE_PORT} user=${DATABASE_USER} password=${DATABASE_PASSWORD} sslmode=disable\" --db-name ${DATABASE_NAME} --schema ${arr[0]} --table ${arr[1]} --file ${IMPORT_INPUT_DEDUP_DIR}/deduped-${arr[0]}.${arr[1]}.csv --workers ${TIMESCALEDB_WORKERS} --reporting-period 300s"
|
||||
else
|
||||
copy_command="timescaledb-parallel-copy --connection \"host=${DATABASE_HOSTNAME} port=${DATABASE_PORT} user=${DATABASE_USER} password=${DATABASE_PASSWORD} sslmode=disable\" --db-name ${DATABASE_NAME} --schema ${arr[0]} --table ${arr[1]} --file ${IMPORT_INPUT_DIR}/${arr[0]}.${arr[1]}.csv --workers ${TIMESCALEDB_WORKERS} --reporting-period 300s"
|
||||
fi
|
||||
|
||||
if [ "${arr[2]}" != "" ];
|
||||
then
|
||||
copy_with_options="${copy_command} --copy-options \"${arr[2]}\""
|
||||
else
|
||||
copy_with_options=${copy_command}
|
||||
fi
|
||||
|
||||
echo "${arr[0]}.${arr[1]}"
|
||||
echo Start: "$(date)"
|
||||
eval "${copy_with_options}"
|
||||
echo End: "$(date)"
|
||||
echo
|
||||
done
|
||||
|
||||
difference=$(($(date +%s)-start_timestamp))
|
||||
echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)
|
||||
echo
|
||||
Reference in New Issue
Block a user