CI, build & docker updates

rm makefile
basic test
add chaindata fixture
simplify publish workflow

.dockerignore

@@ -0,0 +1 @@
+.git

.gitea/workflows/publish.yml

@@ -0,0 +1,28 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  docker-build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - id: vars
+        name: Output SHA and version tag
+        run: |
+          echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
+          echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+      - name: Build and tag image
+        run: |
+          docker build . \
+          -t cerc-io/eth-statediff-service \
+          -t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} \
+          -t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
+      - name: Push image tags
+        run: |
+          echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+          docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+          docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}

.gitea/workflows/tests.yml

@@ -0,0 +1,65 @@
+name: Tests
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches:
+      - main
+      - ci-test
+  workflow_call:
+
+# Needed until we can incorporate docker startup into the executor container
+env:
+  DOCKER_HOST: unix:///var/run/dind.sock
+
+jobs:
+  integration-tests:
+    name: Run integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+          check-latest: true
+      - name: Run dockerd
+        run: |
+          dockerd -H $DOCKER_HOST --userland-proxy=false &
+          sleep 5
+      - name: Run DB container
+        run: docker compose -f test/compose.yml up --wait
+      - name: Configure Gitea access
+        env:
+          TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
+        run: |
+          git config --global url."https://$TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"
+      - name: Build package
+        run: go build .
+      - name: Run server
+        env:
+          DATABASE_TYPE: postgres
+          LEVELDB_PATH: ./fixture/chaindata
+          LEVELDB_ANCIENT: ./fixture/chaindata/ancient
+          LOG_FILE_PATH: ./server-log
+        run: |
+          ./eth-statediff-service --config ./test/ci-config.toml serve &
+
+        # Run a sanity test against the fixture data
+        # Complete integration tests are TODO
+      - name: Run test
+        run: |
+          ./scripts/request-range.sh 0 32 || (E=$?; cat ./server-log; exit $E)
+
+          until grep "Finished processing block 32" ./server-log
+          do sleep 1; done
+
+          count_results() {
+              query="select count(*) from $1;"
+              docker exec -e PGPASSWORD=password test-ipld-eth-db-1 \
+                  psql -tA cerc_testing -U vdbm -c "$query"
+          }
+          set -x
+          [[ "$(count_results eth.header_cids)" = 33 ]]
+          [[ "$(count_results eth.state_cids)" = 21 ]]
+          [[ "$(count_results eth.storage_cids)" = 18 ]]

.github/workflows/on-pr.yaml

@@ -1,7 +0,0 @@
-name: Docker Build
-
-on: [pull_request]
-
-jobs:
-  run-tests:
-    uses: ./.github/workflows/tests.yml

.github/workflows/publish.yaml

@@ -1,44 +0,0 @@
-name: Publish Docker image
-on:
-  release:
-    types: [published]
-jobs:
-  run-tests:
-    uses: ./.github/workflows/tests.yml
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    needs: run-tests
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag vulcanize/eth-statediff-service docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Push
-        run: docker push docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-  push_to_registries:
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Get the version
-        id: vars
-        run: |
-          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-      - name: Docker Login to Github Registry
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Pull
-        run: docker pull docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-      - name: Docker Login to Docker Registry
-        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
-      - name: Tag docker image
-        run: docker tag docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} vulcanize/eth-statediff-service:${{steps.vars.outputs.tag}}
-      - name: Docker Push to Docker Hub
-        run: docker push vulcanize/eth-statediff-service:${{steps.vars.outputs.tag}}

.github/workflows/tests.yml

@@ -1,37 +0,0 @@
-name: Tests for Geth that are used in multiple jobs.
-
-on:
-  workflow_call:
-
-env:
-  GOPATH: /tmp/go
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Run docker build
-        run: make docker-build
-
-  statediff-unit-test:
-    name: Run statediff unit tests
-    runs-on: ubuntu-latest
-    env:
-      GO111MODULE: on
-    steps:
-      - name: Create GOPATH
-        run: mkdir -p /tmp/go
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.18.0"
-          check-latest: true
-
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Run unit tests
-        run: |
-          make test

.gitignore

@@ -1,2 +1,3 @@
 .idea/
 eth-statediff-service
+.vscode

Dockerfile

@@ -1,21 +1,24 @@
-FROM golang:1.18-alpine as builder
+FROM golang:1.19-alpine as builder
 
-RUN apk --update --no-cache add make git g++ linux-headers
+RUN apk add --no-cache git gcc musl-dev binutils-gold
 # DEBUG
 RUN apk add busybox-extras
 
-# Get and build ipfs-blockchain-watcher
-ADD . /go/src/github.com/vulcanize/eth-statediff-service
-#RUN git clone https://github.com/vulcanize/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
+WORKDIR /eth-statediff-service
 
-WORKDIR /go/src/github.com/vulcanize/eth-statediff-service
-RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .
+ARG GIT_VDBTO_TOKEN
+
+COPY go.mod go.sum ./
+RUN if [ -n "$GIT_VDBTO_TOKEN" ]; then git config --global url."https://$GIT_VDBTO_TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"; fi && \
+    go mod download && \
+    rm -f ~/.gitconfig
+COPY . .
+
+RUN go build -ldflags '-extldflags "-static"' -o eth-statediff-service .
 
-# app container
 FROM alpine
 
-ARG USER="vdm"
-ARG CONFIG_FILE="./environments/config.toml"
+ARG USER="vdbm"
 ARG EXPOSE_PORT=8545
 
 RUN adduser -Du 5000 $USER adm
@@ -27,12 +30,11 @@ USER $USER
 
 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/$CONFIG_FILE config.toml
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/startup_script.sh .
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/environments environments
+COPY --chown=5000:5000 --from=builder /eth-statediff-service/startup_script.sh .
+COPY --chown=5000:5000 --from=builder /eth-statediff-service/environments environments
 
 # keep binaries immutable
-COPY --from=builder /go/src/github.com/vulcanize/eth-statediff-service/eth-statediff-service eth-statediff-service
+COPY --from=builder /eth-statediff-service/eth-statediff-service eth-statediff-service
 
 EXPOSE $EXPOSE_PORT
 

Makefile

@@ -1,12 +0,0 @@
-## Build docker image
-.PHONY: docker-build
-docker-build:
-	docker build -t vulcanize/eth-statediff-service .
-
-.PHONY: test
-test: | $(GOOSE)
-	go test -p 1 ./pkg/... -v
-
-build:
-	go fmt ./...
-	go build

README.md

@@ -1,23 +1,23 @@
 # eth-statediff-service
 
-[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/eth-statediff-service)](https://goreportcard.com/report/github.com/vulcanize/eth-statediff-service)
+[![Go Report Card](https://goreportcard.com/badge/github.com/cerc-io/eth-statediff-service)](https://goreportcard.com/report/github.com/cerc-io/eth-statediff-service)
 
->> standalone statediffing service ontop of LevelDB
-
-Purpose:
-
-Stand up a statediffing service directly on top of a go-ethereum LevelDB instance.
+A standalone statediffing service which runs directly on top of a `go-ethereum` LevelDB instance.
 This service can serve historical state data over the same rpc interface as
-[statediffing geth](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.5) without needing to run a full node
+[statediffing geth](https://github.com/cerc-io/go-ethereum) without needing to run a full node.
 
 ## Setup
 
-Build the binary:
+Configure access to the private Git server at `git.vdb.to`, then build the executable:
 
 ```bash
-make build
+go build .
 ```
 
+## Configuration
+
+See [./environments/example.toml](./environments/example.toml) for an annotated example config file.
+
 ### Local Setup
 
 * Create a chain config file `chain.json` according to chain config in genesis json file used by local geth.
@@ -42,142 +42,171 @@ make build
   }
   ```
 
-* Change the following in [config file](./environments/config.toml)
-
-    ```toml
-    [leveldb]
-        mode = "local"
-        # Path to geth LevelDB data
-        path = "/path-to-local-geth-data/chaindata"
-        ancient = "/path-to-local-geth-data/chaindata/ancient"
-
-    [ethereum]
-        chainConfig = "./chain.json" # Path to custom chain config file
-        chainID = 41337 # Same chain ID as in chain.json
-
-    [database]
-        # Update database config
-        name     = "vulcanize_testing"
-        hostname = "localhost"
-        port     = 5432
-        user     = "postgres"
-        password = "postgres"
-        type = "postgres"
-    ```
-
-* To write statediff for a range of block make changes in [config file](./environments/config.toml)
-    ```toml
-    [prerun]
-        only = false
-        ranges = [
-            [8, 15] # Block number range for which to write statediff.
-        ]
-    ```
-
-* To use remote LevelDB RPC endpoint change the following in [config file](./environments/config.toml)
-    ```toml
-    [leveldb]
-        mode = "remote"
-        url = "http://127.0.0.1:8082/" # Remote LevelDB RPC url
-    ```
+  Provide the path to the above file in the config.
 
 ## Usage
 
+* Create / update the config file (refer to example config above).
+
 ### `serve`
 
-To serve state diffs over RPC:
+* To serve the statediff RPC API:
 
-`eth-statediff-service serve --config=<config path>`
+    ```bash
+    ./eth-statediff-service serve --config=<config path>
+    ```
 
-Example:
+    Example:
 
-```bash
-./eth-statediff-service serve --config environments/config.toml
-```
+    ```bash
+    ./eth-statediff-service serve --config environments/config.toml
+    ```
 
-Available RPC methods are:
-  * `statediff_stateTrieAt()`
-  * `statediff_streamCodeAndCodeHash()`
-  * `statediff_stateDiffAt()`
-  * `statediff_writeStateDiffAt()`
-  * `statediff_writeStateDiffsInRange()`
+* Available RPC methods:
+    * `statediff_stateTrieAt()`
+    * `statediff_streamCodeAndCodeHash()`
+    * `statediff_stateDiffAt()`
+    * `statediff_writeStateDiffAt()`
+    * `statediff_writeStateDiffsInRange()`
 
-e.g. `curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"intermediateStateNodes":true,"intermediateStorageNodes":true,"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT"`
+    Example:
 
-The process can be configured locally with sets of ranges to process as a "prerun" to processing directed by the server endpoints.
-This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining ranged and params in the
-`prerun` section of the config as shown below.
+    ```bash
+    curl -X POST -H 'Content-Type: application/json' --data '{
+      "jsonrpc": "2.0",
+      "method": "statediff_writeStateDiffsInRange",
+      "params": [0, 1, {
+          "ncludeBlock": true,
+          "includeReceipts": true,
+          "includeTD": true,
+          "includeCode": true
+        }
+      ],
+      "id": 1
+    }' "$HOST":"$PORT"
+    ```
 
-## Configuration
+* Prerun:
+    * The process can be configured locally with sets of ranges to process as a "prerun" to
+      processing directed by the server endpoints.
+    * This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining
+ranges and params in the `prerun` section of the config.
+    * Set the range using `prerun.start` and `prerun.stop`. Use `prerun.ranges` if prerun on more
+      than one range is required.
 
-An example config file:
+* NOTE: Currently, `params.includeTD` must be set to / passed as `true`.
 
-```toml
-[leveldb]
-    mode = "local"
-    # path and ancient LevelDB paths required in local mode
-    path = "/Users/user/Library/Ethereum/geth/chaindata"
-    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
-    # url for leveldb-ethdb-rpc endpoint required in remote mode
-    url = "http://127.0.0.1:8082/"
+## Monitoring
 
-[server]
-    ipcPath = ".ipc"
-    httpPath = "127.0.0.1:8545"
+* Enable metrics using config parameters `prom.metrics` and `prom.http`.
+* `eth-statediff-service` exposes following prometheus metrics at `/metrics` endpoint:
+    * `ranges_queued`: Number of range requests currently queued.
+    * `loaded_height`: The last block that was loaded for processing.
+    * `processed_height`: The last block that was processed.
+    * `stats.t_block_load`: Block loading time.
+    * `stats.t_block_processing`: Block (header, uncles, txs, rcts, tx trie, rct trie) processing time.
+    * `stats.t_state_processing`: State (state trie, storage tries, and code) processing time.
+    * `stats.t_postgres_tx_commit`: Postgres tx commit time.
+    * `http.count`: HTTP request count.
+    * `http.duration`: HTTP request duration.
+    * `ipc.count`: Unix socket connection count.
 
-[statediff]
-    prerun = true
-    serviceWorkers = 1
-    workerQueueSize = 1024
-    trieWorkers = 4
+## Tests
 
-[prerun]
-    only = false
-    ranges = [
-        [0, 1000]
-    ]
-    [prerun.params]
-        intermediateStateNodes = true
-        intermediateStorageNodes = true
-        includeBlock = true
-        includeReceipts = true
-        includeTD = true
-        includeCode = true
-        watchedAddresses = []
+* Run unit tests:
 
-[log]
-    file = ""
-    level = "info"
+    ```bash
+    make test
+    ```
 
-[eth]
-    chainID = 1
+## Import output data in file mode into a database
 
-[database]
-    name     = "vulcanize_test"
-    hostname = "localhost"
-    port     = 5432
-    user     = "vulcanize"
-    password = "..."
-    type = "postgres"
-    driver = "sqlx"
-    dumpDestination = ""
-    filePath = ""
+* When `eth-statediff-service` is run in file mode (`database.type`: `file`) the output is in form of a SQL
+  file or multiple CSV files.
 
-[cache]
-    database = 1024
-    trie = 1024
+### SQL
 
-[prom]
-    dbStats = false
-    metrics = true
-    http = true
-    httpAddr = "localhost"
-    httpPort = "8889"
+* Assuming the output files are located in host's `./output_dir` directory.
 
-[ethereum]
-    nodeID = ""
-    clientName = "eth-statediff-service"
-    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
-    networkID = 1
-    chainID = 1
-```
+* Create a directory to store post-processed output:
+
+    ```bash
+    mkdir -p output_dir/processed_output
+    ```
+
+* (Optional) Get row counts in the output:
+
+    ```bash
+    wc -l output_dir/statediff.sql > output_stats.txt
+    ```
+
+* De-duplicate data:
+
+    ```bash
+    sort -u output_dir/statediff.sql -o output_dir/processed_output/deduped-statediff.sql
+    ```
+
+* Copy over the post-processed output files to the DB server (say in `/output_dir`).
+
+* Run the following to import data:
+
+    ```bash
+    psql -U <DATABASE_USER> -h <DATABASE_HOSTNAME> -p <DATABASE_PORT> <DATABASE_NAME> --set ON_ERROR_STOP=on -f /output_dir/processed_output/deduped-statediff.sql
+    ```
+
+### CSV
+
+* Create an env file with the required variables. Refer [.sample.env](./scripts/.sample.env).
+
+* (Optional) Get row counts in the output:
+
+    ```bash
+    ./scripts/count-lines.sh <ENV_FILE_PATH>
+    ```
+
+* De-duplicate data:
+
+    ```bash
+    ./scripts/dedup.sh <ENV_FILE_PATH>
+    ```
+
+* Perform column checks:
+
+    ```bash
+    ./scripts/check-columns.sh <ENV_FILE_PATH>
+    ```
+
+    Check the output logs for any rows detected with unexpected number of columns.
+
+    Example:
+
+    ```bash
+    # log
+    eth.header_cids
+    Start: Wednesday 21 September 2022 06:00:38 PM IST
+    Time taken: 00:00:05
+    End: Wednesday 21 September 2022 06:00:43 PM IST
+    Total bad rows: 1 ./check-columns/eth.header_cids.txt
+
+    # bad row output
+    # line number, num. of columns, data
+    23 17 22,xxxxxx,0x07f5ea5c94aa8dea60b28f6b6315d92f2b6d78ca4b74ea409adeb191b5a114f2,0x5918487321aa57dd0c50977856c6231e7c4ee79e95b694c7c8830227d77a1ecc,bagiacgzaa726uxeuvkg6uyfsr5vwgfozf4vw26gkjn2ouqe232yzdnnbctza,45,geth,0,0xad8fa8df61b98dbda7acd6ca76d5ce4cbba663d5f608cc940957adcdb94cee8d,0xc621412320a20b4aaff5363bdf063b9d13e394ef82e55689ab703aae5db08e26,0x71ec1c7d81269ce115be81c81f13e1cc2601c292a7f20440a77257ecfdc69940,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,\x2000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000,1658408419,/blocks/DMQAP5PKLSKKVDPKMCZI623DCXMS6K3NPDFEW5HKICNN5MMRWWQRJ4Q,1,0x0000000000000000000000000000000000000000
+    ```
+
+* Import data using `timescaledb-parallel-copy`:  
+  (requires [`timescaledb-parallel-copy`](https://github.com/timescale/timescaledb-parallel-copy) installation; readily comes with TimescaleDB docker image)
+
+    ```bash
+    ./scripts/timescaledb-import.sh <ENV_FILE_PATH>
+    ```
+
+* NOTE: `COPY` command on CSVs inserts empty strings as `NULL` in the DB. Passing `FORCE_NOT_NULL <COLUMN_NAME>` forces it to insert empty strings instead. This is required to maintain compatibility of the imported statediff data with the data generated in `postgres` mode. Reference: https://www.postgresql.org/docs/14/sql-copy.html
+
+### Stats
+
+The binary includes a `stats` command which reports stats for the offline or remote levelDB.
+
+At this time, the only stat supported is to return the latest/highest block height and hash found the levelDB, this is
+useful for determining what the upper limit is for a standalone statediffing process on a given levelDB.
+
+`./eth-statediff-service stats --config={path to toml config file}`

cmd/env.go

@@ -30,10 +30,10 @@ const (
 
 	DB_CACHE_SIZE_MB   = "DB_CACHE_SIZE_MB"
 	TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
-	LVLDB_MODE         = "LVLDB_MODE"
-	LVLDB_PATH         = "LVLDB_PATH"
-	LVLDB_ANCIENT      = "LVLDB_ANCIENT"
-	LVLDB_URL          = "LVLDB_URL"
+	LEVELDB_MODE       = "LEVELDB_MODE"
+	LEVELDB_PATH       = "LEVELDB_PATH"
+	LEVELDB_ANCIENT    = "LEVELDB_ANCIENT"
+	LEVELDB_URL        = "LEVELDB_URL"
 
 	STATEDIFF_PRERUN            = "STATEDIFF_PRERUN"
 	STATEDIFF_TRIE_WORKERS      = "STATEDIFF_TRIE_WORKERS"
@@ -49,15 +49,14 @@ const (
 	PROM_HTTP_PORT = "PROM_HTTP_PORT"
 	PROM_DB_STATS  = "PROM_DB_STATS"
 
-	PRERUN_ONLY                       = "PRERUN_ONLY"
-	PRERUN_RANGE_START                = "PRERUN_RANGE_START"
-	PRERUN_RANGE_STOP                 = "PRERUN_RANGE_STOP"
-	PRERUN_INTERMEDIATE_STATE_NODES   = "PRERUN_INTERMEDIATE_STATE_NODES"
-	PRERUN_INTERMEDIATE_STORAGE_NODES = "PRERUN_INTERMEDIATE_STORAGE_NODES"
-	PRERUN_INCLUDE_BLOCK              = "PRERUN_INCLUDE_BLOCK"
-	PRERUN_INCLUDE_RECEIPTS           = "PRERUN_INCLUDE_RECEIPTS"
-	PRERUN_INCLUDE_TD                 = "PRERUN_INCLUDE_TD"
-	PRERUN_INCLUDE_CODE               = "PRERUN_INCLUDE_CODE"
+	PRERUN_ONLY             = "PRERUN_ONLY"
+	PRERUN_PARALLEL         = "PRERUN_PARALLEL"
+	PRERUN_RANGE_START      = "PRERUN_RANGE_START"
+	PRERUN_RANGE_STOP       = "PRERUN_RANGE_STOP"
+	PRERUN_INCLUDE_BLOCK    = "PRERUN_INCLUDE_BLOCK"
+	PRERUN_INCLUDE_RECEIPTS = "PRERUN_INCLUDE_RECEIPTS"
+	PRERUN_INCLUDE_TD       = "PRERUN_INCLUDE_TD"
+	PRERUN_INCLUDE_CODE     = "PRERUN_INCLUDE_CODE"
 
 	LOG_LEVEL     = "LOG_LEVEL"
 	LOG_FILE_PATH = "LOG_FILE_PATH"
@@ -68,10 +67,12 @@ const (
 	DATABASE_USER     = "DATABASE_USER"
 	DATABASE_PASSWORD = "DATABASE_PASSWORD"
 
-	DATABASE_TYPE        = "DATABASE_TYPE"
-	DATABASE_DRIVER_TYPE = "DATABASE_DRIVER_TYPE"
-	DATABASE_DUMP_DST    = "DATABASE_DUMP_DST"
-	DATABASE_FILE_PATH   = "DATABASE_FILE_PATH"
+	DATABASE_TYPE         = "DATABASE_TYPE"
+	DATABASE_DRIVER_TYPE  = "DATABASE_DRIVER_TYPE"
+	DATABASE_DUMP_DST     = "DATABASE_DUMP_DST"
+	DATABASE_FILE_PATH    = "DATABASE_FILE_PATH"
+	DATABASE_FILE_MODE    = "DATABASE_FILE_MODE"
+	DATABASE_FILE_CSV_DIR = "DATABASE_FILE_CSV_DIR"
 
 	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
 	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
@@ -79,6 +80,8 @@ const (
 	DATABASE_MAX_CONN_LIFETIME    = "DATABASE_MAX_CONN_LIFETIME"
 	DATABASE_CONN_TIMEOUT         = "DATABSE_CONN_TIMEOUT"
 	DATABASE_MAX_CONN_IDLE_TIME   = "DATABASE_MAX_CONN_IDLE_TIME"
+
+	DEBUG_PPROF = "DEBUG_PPROF"
 )
 
 // Bind env vars for eth node and DB configuration
@@ -109,15 +112,17 @@ func init() {
 	viper.BindEnv("database.type", DATABASE_TYPE)
 	viper.BindEnv("database.driver", DATABASE_DRIVER_TYPE)
 	viper.BindEnv("database.dumpDestination", DATABASE_DUMP_DST)
+	viper.BindEnv("database.fileMode", DATABASE_FILE_MODE)
 	viper.BindEnv("database.filePath", DATABASE_FILE_PATH)
+	viper.BindEnv("database.fileCsvDir", DATABASE_FILE_CSV_DIR)
 
 	viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
 	viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)
 
-	viper.BindEnv("leveldb.mode", LVLDB_MODE)
-	viper.BindEnv("leveldb.path", LVLDB_PATH)
-	viper.BindEnv("leveldb.ancient", LVLDB_ANCIENT)
-	viper.BindEnv("leveldb.url", LVLDB_URL)
+	viper.BindEnv("leveldb.mode", LEVELDB_MODE)
+	viper.BindEnv("leveldb.path", LEVELDB_PATH)
+	viper.BindEnv("leveldb.ancient", LEVELDB_ANCIENT)
+	viper.BindEnv("leveldb.url", LEVELDB_URL)
 
 	viper.BindEnv("prom.metrics", PROM_METRICS)
 	viper.BindEnv("prom.http", PROM_HTTP)
@@ -131,10 +136,9 @@ func init() {
 
 	viper.BindEnv("statediff.prerun", STATEDIFF_PRERUN)
 	viper.BindEnv("prerun.only", PRERUN_ONLY)
+	viper.BindEnv("prerun.parallel", PRERUN_PARALLEL)
 	viper.BindEnv("prerun.start", PRERUN_RANGE_START)
 	viper.BindEnv("prerun.stop", PRERUN_RANGE_STOP)
-	viper.BindEnv("prerun.params.intermediateStateNodes", PRERUN_INTERMEDIATE_STATE_NODES)
-	viper.BindEnv("prerun.params.intermediateStorageNodes", PRERUN_INTERMEDIATE_STORAGE_NODES)
 	viper.BindEnv("prerun.params.includeBlock", PRERUN_INCLUDE_BLOCK)
 	viper.BindEnv("prerun.params.includeReceipts", PRERUN_INCLUDE_RECEIPTS)
 	viper.BindEnv("prerun.params.includeTD", PRERUN_INCLUDE_TD)
@@ -142,4 +146,6 @@ func init() {
 
 	viper.BindEnv("log.level", LOG_LEVEL)
 	viper.BindEnv("log.file", LOG_FILE_PATH)
+
+	viper.BindEnv("debug.pprof", DEBUG_PPROF)
 }

cmd/root.go

@@ -23,18 +23,18 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cerc-io/plugeth-statediff/indexer/database/dump"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/file"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
+	"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
+	"github.com/cerc-io/plugeth-statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/indexer/shared"
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/dump"
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
-	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
-	"github.com/ethereum/go-ethereum/statediff/indexer/node"
-	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )
 
 var (
@@ -49,7 +49,6 @@ var rootCmd = &cobra.Command{
 }
 
 func Execute() {
-	log.Info("----- Starting vDB -----")
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)
 	}
@@ -126,7 +125,7 @@ func init() {
 	rootCmd.PersistentFlags().Int("trie-workers", 0, "number of workers to use for trie traversal and processing")
 	rootCmd.PersistentFlags().Int("worker-queue-size", 0, "size of the range request queue for service workers")
 
-	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
+	rootCmd.PersistentFlags().String("database-name", "cerc_public", "database name")
 	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
 	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
 	rootCmd.PersistentFlags().String("database-user", "", "database user")
@@ -140,7 +139,9 @@ func init() {
 	rootCmd.PersistentFlags().String("database-type", "postgres", "database type (currently supported: postgres, dump)")
 	rootCmd.PersistentFlags().String("database-driver", "sqlx", "database driver type (currently supported: sqlx, pgx)")
 	rootCmd.PersistentFlags().String("database-dump-dst", "stdout", "dump destination (for database-type=dump; options: stdout, stderr, discard)")
-	rootCmd.PersistentFlags().String("database-file-path", "", "full file path (for database-type=file)")
+	rootCmd.PersistentFlags().String("database-file-mode", "csv", "mode for writing file (for database-type=file; options: csv, sql)")
+	rootCmd.PersistentFlags().String("database-file-csv-dir", "", "full directory path (for database-file-mode=csv)")
+	rootCmd.PersistentFlags().String("database-file-path", "", "full file path (for database-file-mode=sql)")
 
 	rootCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
 	rootCmd.PersistentFlags().String("eth-client-name", "eth-statediff-service", "eth client name")
@@ -161,8 +162,6 @@ func init() {
 	rootCmd.PersistentFlags().Bool("prerun-only", false, "only process pre-configured ranges; exit afterwards")
 	rootCmd.PersistentFlags().Int("prerun-start", 0, "start height for a prerun range")
 	rootCmd.PersistentFlags().Int("prerun-stop", 0, "stop height for a prerun range")
-	rootCmd.PersistentFlags().Bool("prerun-intermediate-state-nodes", true, "include intermediate state nodes in state diff")
-	rootCmd.PersistentFlags().Bool("prerun-intermediate-storage-nodes", true, "include intermediate storage nodes in state diff")
 	rootCmd.PersistentFlags().Bool("prerun-include-block", true, "include block data in the statediff payload")
 	rootCmd.PersistentFlags().Bool("prerun-include-receipts", true, "include receipts in the statediff payload")
 	rootCmd.PersistentFlags().Bool("prerun-include-td", true, "include td in the statediff payload")
@@ -198,6 +197,8 @@ func init() {
 	viper.BindPFlag("database.type", rootCmd.PersistentFlags().Lookup("database-type"))
 	viper.BindPFlag("database.driver", rootCmd.PersistentFlags().Lookup("database-driver"))
 	viper.BindPFlag("database.dumpDestination", rootCmd.PersistentFlags().Lookup("database-dump-dst"))
+	viper.BindPFlag("database.fileMode", rootCmd.PersistentFlags().Lookup("database-file-mode"))
+	viper.BindPFlag("database.fileCsvDir", rootCmd.PersistentFlags().Lookup("database-file-csv-dir"))
 	viper.BindPFlag("database.filePath", rootCmd.PersistentFlags().Lookup("database-file-path"))
 
 	viper.BindPFlag("ethereum.nodeID", rootCmd.PersistentFlags().Lookup("eth-node-id"))
@@ -217,15 +218,16 @@ func init() {
 	viper.BindPFlag("prom.metrics", rootCmd.PersistentFlags().Lookup("prom-metrics"))
 
 	viper.BindPFlag("prerun.only", rootCmd.PersistentFlags().Lookup("prerun-only"))
+	viper.BindPFlag("prerun.parallel", rootCmd.PersistentFlags().Lookup("prerun-parallel"))
 	viper.BindPFlag("prerun.start", rootCmd.PersistentFlags().Lookup("prerun-start"))
 	viper.BindPFlag("prerun.stop", rootCmd.PersistentFlags().Lookup("prerun-stop"))
-	viper.BindPFlag("prerun.params.intermediateStateNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-state-nodes"))
-	viper.BindPFlag("prerun.params.intermediateStorageNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-storage-nodes"))
 	viper.BindPFlag("prerun.params.includeBlock", rootCmd.PersistentFlags().Lookup("prerun-include-block"))
 	viper.BindPFlag("prerun.params.includeReceipts", rootCmd.PersistentFlags().Lookup("prerun-include-receipts"))
 	viper.BindPFlag("prerun.params.includeTD", rootCmd.PersistentFlags().Lookup("prerun-include-td"))
 	viper.BindPFlag("prerun.params.includeCode", rootCmd.PersistentFlags().Lookup("prerun-include-code"))
 
+	viper.BindPFlag("debug.pprof", rootCmd.PersistentFlags().Lookup("debug-pprof"))
+
 	rand.Seed(time.Now().UnixNano())
 }
 
@@ -297,18 +299,35 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 	if err != nil {
 		return nil, err
 	}
-	logWithCommand.Infof("configuring service for database type: %s", dbType)
+	logWithCommand.Debugf("Configuring service for database type: %s", dbType)
 	var indexerConfig interfaces.Config
 	switch dbType {
 	case shared.FILE:
-		logWithCommand.Info("starting in sql file writing mode")
-		filePathStr := viper.GetString("database.filePath")
-		if filePathStr == "" {
-			logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided")
+		logWithCommand.Info("Starting in sql file writing mode")
+
+		fileModeStr := viper.GetString("database.fileMode")
+		fileMode, err := file.ResolveFileMode(fileModeStr)
+		if err != nil {
+			utils.Fatalf("%v", err)
+		}
+
+		filePathStr := viper.GetString("database.filePath")
+		if fileMode == file.SQL && filePathStr == "" {
+			logWithCommand.Fatal("When operating in sql file writing mode a file path must be provided")
+		}
+
+		fileCsvDirStr := viper.GetString("database.fileCsvDir")
+		if fileMode == file.CSV && fileCsvDirStr == "" {
+			logWithCommand.Fatal("When operating in csv file writing mode a directory path must be provided")
+		}
+
+		indexerConfig = file.Config{
+			Mode:      fileMode,
+			OutputDir: fileCsvDirStr,
+			FilePath:  filePathStr,
 		}
-		indexerConfig = file.Config{FilePath: filePathStr}
 	case shared.DUMP:
-		logWithCommand.Info("starting in data dump mode")
+		logWithCommand.Info("Starting in data dump mode")
 		dumpDstStr := viper.GetString("database.dumpDestination")
 		dumpDst, err := dump.ResolveDumpType(dumpDstStr)
 		if err != nil {
@@ -320,12 +339,12 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 		case dump.STDOUT:
 			indexerConfig = dump.Config{Dump: os.Stderr}
 		case dump.DISCARD:
-			indexerConfig = dump.Config{Dump: dump.NewDiscardWriterCloser()}
+			indexerConfig = dump.Config{Dump: dump.Discard}
 		default:
 			return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst)
 		}
 	case shared.POSTGRES:
-		logWithCommand.Info("starting in postgres mode")
+		logWithCommand.Info("Starting in postgres mode")
 		driverTypeStr := viper.GetString("database.driver")
 		driverType, err := postgres.ResolveDriverType(driverTypeStr)
 		if err != nil {
@@ -337,8 +356,6 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 			DatabaseName: viper.GetString("database.name"),
 			Username:     viper.GetString("database.user"),
 			Password:     viper.GetString("database.password"),
-			ID:           nodeInfo.ID,
-			ClientName:   nodeInfo.ClientName,
 			Driver:       driverType,
 		}
 		if viper.IsSet("database.maxIdle") {

cmd/serve.go

@@ -16,8 +16,11 @@
 package cmd
 
 import (
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 	"os/signal"
+	"runtime"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/rpc"
@@ -25,8 +28,8 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	sd "github.com/vulcanize/eth-statediff-service/pkg"
-	srpc "github.com/vulcanize/eth-statediff-service/pkg/rpc"
+	pkg "github.com/cerc-io/eth-statediff-service/pkg"
+	srpc "github.com/cerc-io/eth-statediff-service/pkg/rpc"
 )
 
 // serveCmd represents the serve command
@@ -47,58 +50,80 @@ func init() {
 	rootCmd.AddCommand(serveCmd)
 }
 
-func serve() {
-	logWithCommand.Info("Running eth-statediff-service serve command")
+func maxParallelism() int {
+	maxProcs := runtime.GOMAXPROCS(0)
+	numCPU := runtime.NumCPU()
+	if maxProcs < numCPU {
+		return maxProcs
+	}
+	return numCPU
+}
 
-	statediffService, err := createStateDiffService()
+func serve() {
+	logWithCommand.Debug("Running eth-statediff-service serve command")
+	logWithCommand.Debugf("Parallelism: %d", maxParallelism())
+
+	reader, chainConf, nodeInfo := instantiateLevelDBReader()
+
+	reportLatestBlock(reader)
+
+	service, err := createStateDiffService(reader, chainConf, nodeInfo)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
 
+	// Enable the pprof agent if configured
+	if viper.GetBool("debug.pprof") {
+		// See: https://www.farsightsecurity.com/blog/txt-record/go-remote-profiling-20161028/
+		// For security reasons: do not use the default http multiplexor elsewhere in this process.
+		go func() {
+			logWithCommand.Info("Starting pprof listener on port 6060")
+			logWithCommand.Fatal(http.ListenAndServe("localhost:6060", nil))
+		}()
+	}
+
 	// short circuit if we only want to perform prerun
 	if viper.GetBool("prerun.only") {
-		if err := statediffService.Run(nil); err != nil {
-			logWithCommand.Fatal("unable to perform prerun: %v", err)
+		parallel := viper.GetBool("prerun.parallel")
+		if err := service.Run(nil, parallel); err != nil {
+			logWithCommand.Fatalf("Unable to perform prerun: %v", err)
 		}
 		return
 	}
 
 	// start service and servers
-	logWithCommand.Info("Starting statediff service")
-	wg := new(sync.WaitGroup)
-	if err := statediffService.Loop(wg); err != nil {
+	var wg sync.WaitGroup
+	if err := service.Loop(&wg); err != nil {
 		logWithCommand.Fatalf("unable to start statediff service: %v", err)
 	}
-	logWithCommand.Info("Starting RPC servers")
-	if err := startServers(statediffService); err != nil {
+
+	if err := startServers(service); err != nil {
 		logWithCommand.Fatal(err)
 	}
-	logWithCommand.Info("RPC servers successfully spun up; awaiting requests")
+	logWithCommand.Debug("RPC servers successfully spun up; awaiting requests")
 
 	// clean shutdown
 	shutdown := make(chan os.Signal)
 	signal.Notify(shutdown, os.Interrupt)
 	<-shutdown
 	logWithCommand.Info("Received interrupt signal, shutting down")
-	statediffService.Stop()
+	service.Stop()
 	wg.Wait()
 }
 
-func startServers(serv sd.StateDiffService) error {
+func startServers(serv *pkg.Service) error {
 	ipcPath := viper.GetString("server.ipcPath")
 	httpPath := viper.GetString("server.httpPath")
 	if ipcPath == "" && httpPath == "" {
-		logWithCommand.Fatal("need an ipc path and/or an http path")
+		logWithCommand.Fatal("Need an IPC path and/or an HTTP path")
 	}
 	if ipcPath != "" {
-		logWithCommand.Info("starting up IPC server")
 		_, _, err := srpc.StartIPCEndpoint(ipcPath, serv.APIs())
 		if err != nil {
 			return err
 		}
 	}
 	if httpPath != "" {
-		logWithCommand.Info("starting up HTTP server")
 		_, err := srpc.StartHTTPEndpoint(httpPath, serv.APIs(), []string{"statediff"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
 		if err != nil {
 			return err

cmd/stats.go

@@ -0,0 +1,46 @@
+// Copyright © 2022 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+// statsCmd represents the serve command
+var statsCmd = &cobra.Command{
+	Use:   "stats",
+	Short: "Report stats for cold levelDB",
+	Long: `Usage
+
+./eth-statediff-service stats --config={path to toml config file}`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *logrus.WithField("SubCommand", subCommand)
+		stats()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(statsCmd)
+}
+
+func stats() {
+	logWithCommand.Info("Running eth-statediff-service stats command")
+
+	reader, _, _ := instantiateLevelDBReader()
+	reportLatestBlock(reader)
+}

cmd/util.go

@@ -2,23 +2,118 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"os"
 
+	statediff "github.com/cerc-io/plugeth-statediff"
+	"github.com/cerc-io/plugeth-statediff/indexer"
+	"github.com/cerc-io/plugeth-statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/indexer/shared"
+	"github.com/cerc-io/plugeth-statediff/utils/log"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/statediff"
-	gethsd "github.com/ethereum/go-ethereum/statediff"
-	ind "github.com/ethereum/go-ethereum/statediff/indexer"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/spf13/viper"
 
-	sd "github.com/vulcanize/eth-statediff-service/pkg"
+	pkg "github.com/cerc-io/eth-statediff-service/pkg"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )
 
 type blockRange [2]uint64
 
-func createStateDiffService() (sd.StateDiffService, error) {
+func createStateDiffService(lvlDBReader pkg.Reader, chainConf *params.ChainConfig, nodeInfo node.Info) (*pkg.Service, error) {
+	// create statediff service
+	logWithCommand.Debug("Setting up database")
+	conf, err := getConfig(nodeInfo)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	logWithCommand.Debug("Creating statediff indexer")
+	db, indexer, err := indexer.NewStateDiffIndexer(context.Background(), chainConf, nodeInfo, conf)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	if conf.Type() == shared.POSTGRES && viper.GetBool("prom.dbStats") {
+		prom.RegisterDBCollector(viper.GetString("database.name"), db)
+	}
+
+	logWithCommand.Debug("Creating statediff service")
+	sdConf := pkg.ServiceConfig{
+		ServiceWorkers:  viper.GetUint("statediff.serviceWorkers"),
+		TrieWorkers:     viper.GetUint("statediff.trieWorkers"),
+		WorkerQueueSize: viper.GetUint("statediff.workerQueueSize"),
+		PreRuns:         setupPreRunRanges(),
+	}
+	return pkg.NewStateDiffService(lvlDBReader, indexer, sdConf), nil
+}
+
+func setupPreRunRanges() []pkg.RangeRequest {
+	if !viper.GetBool("statediff.prerun") {
+		return nil
+	}
+	preRunParams := statediff.Params{
+		IncludeBlock:    viper.GetBool("prerun.params.includeBlock"),
+		IncludeReceipts: viper.GetBool("prerun.params.includeReceipts"),
+		IncludeTD:       viper.GetBool("prerun.params.includeTD"),
+		IncludeCode:     viper.GetBool("prerun.params.includeCode"),
+	}
+	var addrStrs []string
+	viper.UnmarshalKey("prerun.params.watchedAddresses", &addrStrs)
+	addrs := make([]common.Address, len(addrStrs))
+	for i, addrStr := range addrStrs {
+		addrs[i] = common.HexToAddress(addrStr)
+	}
+	preRunParams.WatchedAddresses = addrs
+	var rawRanges []blockRange
+	viper.UnmarshalKey("prerun.ranges", &rawRanges)
+	blockRanges := make([]pkg.RangeRequest, len(rawRanges))
+	for i, rawRange := range rawRanges {
+		blockRanges[i] = pkg.RangeRequest{
+			Start:  rawRange[0],
+			Stop:   rawRange[1],
+			Params: preRunParams,
+		}
+	}
+	if viper.IsSet("prerun.start") && viper.IsSet("prerun.stop") {
+		hardStart := viper.GetInt("prerun.start")
+		hardStop := viper.GetInt("prerun.stop")
+		blockRanges = append(blockRanges, pkg.RangeRequest{
+			Start:  uint64(hardStart),
+			Stop:   uint64(hardStop),
+			Params: preRunParams,
+		})
+	}
+
+	return blockRanges
+}
+
+// LoadConfig loads chain config from json file
+func LoadConfig(chainConfigPath string) (*params.ChainConfig, error) {
+	file, err := os.Open(chainConfigPath)
+	if err != nil {
+		log.Error("Failed to read chain config file", "error", err)
+
+		return nil, err
+	}
+	defer file.Close()
+
+	chainConfig := new(params.ChainConfig)
+	if err := json.NewDecoder(file).Decode(chainConfig); err != nil {
+		log.Error("invalid chain config file", "error", err)
+
+		return nil, err
+	}
+
+	log.Debug(fmt.Sprintf("Using chain config from '%s'. Content: %+v", chainConfigPath, chainConfig))
+
+	return chainConfig, nil
+}
+
+func instantiateLevelDBReader() (pkg.Reader, *params.ChainConfig, node.Info) {
 	// load some necessary params
-	logWithCommand.Info("Loading statediff service parameters")
+	logWithCommand.Debug("Loading statediff service parameters")
 	mode := viper.GetString("leveldb.mode")
 	path := viper.GetString("leveldb.path")
 	ancientPath := viper.GetString("leveldb.ancient")
@@ -38,23 +133,15 @@ func createStateDiffService() (sd.StateDiffService, error) {
 
 	nodeInfo := getEthNodeInfo()
 
-	var chainConf *params.ChainConfig
-	var err error
 	chainConfigPath := viper.GetString("ethereum.chainConfig")
-
-	if chainConfigPath != "" {
-		chainConf, err = statediff.LoadConfig(chainConfigPath)
-	} else {
-		chainConf, err = statediff.ChainConfig(nodeInfo.ChainID)
-	}
-
+	chainConf, err := LoadConfig(chainConfigPath)
 	if err != nil {
-		logWithCommand.Fatal(err)
+		logWithCommand.Fatalf("Unable to instantiate chain config: %s", err)
 	}
 
 	// create LevelDB reader
-	logWithCommand.Info("Creating LevelDB reader")
-	readerConf := sd.LvLDBReaderConfig{
+	logWithCommand.Debug("Creating LevelDB reader")
+	readerConf := pkg.LvLDBReaderConfig{
 		TrieConfig: &trie.Config{
 			Cache:     viper.GetInt("cache.trie"),
 			Journal:   "",
@@ -67,70 +154,24 @@ func createStateDiffService() (sd.StateDiffService, error) {
 		Url:         url,
 		DBCacheSize: viper.GetInt("cache.database"),
 	}
-	lvlDBReader, err := sd.NewLvlDBReader(readerConf)
+	reader, err := pkg.NewLvlDBReader(readerConf)
 	if err != nil {
-		logWithCommand.Fatal(err)
+		logWithCommand.Fatalf("Unable to instantiate levelDB reader: %s", err)
 	}
-
-	// create statediff service
-	logWithCommand.Info("Setting up database")
-	conf, err := getConfig(nodeInfo)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	logWithCommand.Info("Creating statediff indexer")
-	_, indexer, err := ind.NewStateDiffIndexer(context.Background(), chainConf, nodeInfo, conf)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	logWithCommand.Info("Creating statediff service")
-	sdConf := sd.Config{
-		ServiceWorkers:  viper.GetUint("statediff.serviceWorkers"),
-		TrieWorkers:     viper.GetUint("statediff.trieWorkers"),
-		WorkerQueueSize: viper.GetUint("statediff.workerQueueSize"),
-		PreRuns:         setupPreRunRanges(),
-	}
-	return sd.NewStateDiffService(lvlDBReader, indexer, sdConf)
+	return reader, chainConf, nodeInfo
 }
 
-func setupPreRunRanges() []sd.RangeRequest {
-	if !viper.GetBool("statediff.prerun") {
-		return nil
+// report latest block info
+func reportLatestBlock(reader pkg.Reader) {
+	header, err := reader.GetLatestHeader()
+	if err != nil {
+		logWithCommand.Fatalf("Unable to determine latest header height and hash: %s", err.Error())
 	}
-	preRunParams := gethsd.Params{
-		IntermediateStateNodes:   viper.GetBool("prerun.params.intermediateStateNodes"),
-		IntermediateStorageNodes: viper.GetBool("prerun.params.intermediateStorageNodes"),
-		IncludeBlock:             viper.GetBool("prerun.params.includeBlock"),
-		IncludeReceipts:          viper.GetBool("prerun.params.includeReceipts"),
-		IncludeTD:                viper.GetBool("prerun.params.includeTD"),
-		IncludeCode:              viper.GetBool("prerun.params.includeCode"),
+	if header.Number == nil {
+		logWithCommand.Fatal("Latest header found in levelDB has a nil block height")
 	}
-	var addrStrs []string
-	viper.UnmarshalKey("prerun.params.watchedAddresses", &addrStrs)
-	addrs := make([]common.Address, len(addrStrs))
-	for i, addrStr := range addrStrs {
-		addrs[i] = common.HexToAddress(addrStr)
-	}
-	preRunParams.WatchedAddresses = addrs
-	var rawRanges []blockRange
-	viper.UnmarshalKey("prerun.ranges", &rawRanges)
-	blockRanges := make([]sd.RangeRequest, len(rawRanges))
-	for i, rawRange := range rawRanges {
-		blockRanges[i] = sd.RangeRequest{
-			Start:  rawRange[0],
-			Stop:   rawRange[1],
-			Params: preRunParams,
-		}
-	}
-	if viper.IsSet("prerun.start") && viper.IsSet("prerun.stop") {
-		hardStart := viper.GetInt("prerun.start")
-		hardStop := viper.GetInt("prerun.stop")
-		blockRanges = append(blockRanges, sd.RangeRequest{
-			Start:  uint64(hardStart),
-			Stop:   uint64(hardStop),
-			Params: preRunParams,
-		})
-	}
-
-	return blockRanges
+	logWithCommand.
+		WithField("height", header.Number).
+		WithField("hash", header.Hash()).
+		Info("Latest block found in levelDB")
 }

cmd/version.go

@@ -20,7 +20,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	v "github.com/vulcanize/eth-statediff-service/version"
+	v "github.com/cerc-io/eth-statediff-service/version"
 )
 
 // versionCmd represents the version command

docker-compose.yml

@@ -1,23 +0,0 @@
-version: '3.2'
-
-services:
-  eth-statediff-service:
-    build:
-      context: ./
-      cache_from:
-      - alpine:latest
-      - golang:1.16
-      dockerfile: ./Dockerfile
-      args:
-        USER: "vdbm"
-        CONFIG_FILE: ./environments/example.toml
-        EXPOSE_PORT: 8545
-    environment:
-    - VDB_COMMAND=serve
-    volumes:
-    - eth-statediff-service-data:/root/.ethereum/
-    ports:
-    - "127.0.0.1:8545:8545"
-
-volumes:
-  eth-statediff-service-data:

environments/config.toml

@@ -1,61 +0,0 @@
-[leveldb]
-    mode = "local"
-    path = "/app/geth-rw/chaindata"
-    ancient = "/app/geth-rw/chaindata/ancient"
-    url = "http://127.0.0.1:8082/"
-
-[server]
-    ipcPath = ""
-    httpPath = "0.0.0.0:8545"
-
-[statediff]
-    prerun = true
-    serviceWorkers = 1
-    workerQueueSize = 1024
-    trieWorkers = 16
-
-[prerun]
-    only = true
-    ranges = []
-    [prerun.params]
-        intermediateStateNodes = true
-        intermediateStorageNodes = true
-        includeBlock = true
-        includeReceipts = true
-        includeTD = true
-        includeCode = true
-        watchedAddresses = []
-
-[log]
-    file = ""
-    level = "info"
-
-[database]
-    name     = ""
-    hostname = ""
-    port     = 5432
-    user     = ""
-    password = ""
-    type = "postgres"
-    driver = "sqlx"
-    dumpDestination = ""
-    filePath = ""
-
-[cache]
-    database = 1024
-    trie = 4096
-
-[prom]
-    dbStats = false
-    metrics = true
-    http = true
-    httpAddr = "0.0.0.0"
-    httpPort = 9100
-
-[ethereum]
-    chainConfig = ""
-    nodeID = ""
-    clientName = "eth-statediff-service"
-    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
-    networkID = 1
-    chainID = 1

environments/example.toml

@@ -1,63 +1,104 @@
 [leveldb]
-    mode = "local"
-    path = "/Users/user/Library/Ethereum/geth/chaindata"
-    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
-    url = "http://127.0.0.1:8082/"
+    # LevelDB access mode <local | remote>
+    mode = "local"  # LEVELDB_MODE
+
+    # LevelDB paths (local mode)
+    path    = "/Users/user/Library/Ethereum/geth/chaindata"         # LEVELDB_PATH
+    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LEVELDB_ANCIENT
+
+    # URL for leveldb-ethdb-rpc endpoint (remote mode)
+    url = "http://127.0.0.1:8082/"  # LEVELDB_URL
 
 [server]
-    ipcPath = ".ipc"
-    httpPath = "127.0.0.1:8545"
+    ipcPath  = ".ipc"           # SERVICE_IPC_PATH
+    httpPath = "127.0.0.1:8545" # SERVICE_HTTP_PATH
 
 [statediff]
-    prerun = true
-    serviceWorkers = 1
-    workerQueueSize = 1024
-    trieWorkers = 4
+    prerun          = true  # STATEDIFF_PRERUN
+    serviceWorkers  = 1     # STATEDIFF_SERVICE_WORKERS
+    workerQueueSize = 1024  # STATEDIFF_WORKER_QUEUE_SIZE
+    trieWorkers     = 4     # STATEDIFF_TRIE_WORKERS
 
 [prerun]
-    only = false
+    only = false     # PRERUN_ONLY
+    parallel = true  # PRERUN_PARALLEL
+
+    # to perform prerun in a specific range (optional)
+    start = 0   # PRERUN_RANGE_START
+    stop  = 100 # PRERUN_RANGE_STOP
+
+    # to perform prerun over multiple ranges (optional)
     ranges = [
-        [0, 1000]
+        [101, 1000]
     ]
+
+    # statediffing params for prerun
     [prerun.params]
-        intermediateStateNodes = true
-        intermediateStorageNodes = true
-        includeBlock = true
-        includeReceipts = true
-        includeTD = true
-        includeCode = true
-        watchedAddresses = []
+        intermediateStateNodes   = true # PRERUN_INTERMEDIATE_STATE_NODES
+        intermediateStorageNodes = true # PRERUN_INTERMEDIATE_STORAGE_NODES
+        includeBlock             = true # PRERUN_INCLUDE_BLOCK
+        includeReceipts          = true # PRERUN_INCLUDE_RECEIPTS
+        includeTD                = true # PRERUN_INCLUDE_TD
+        includeCode              = true # PRERUN_INCLUDE_CODE
+        watchedAddresses         = []
 
 [log]
-    file = ""
-    level = "info"
+    # Leave empty to output to stdout
+    file  = ""      # LOG_FILE_PATH
+    level = "info"  # LOG_LEVEL
 
 [database]
-    name     = "vulcanize_test"
-    hostname = "localhost"
-    port     = 5432
-    user     = "vulcanize"
-    password = "..."
+    # output type <postgres | file | dump>
     type = "postgres"
-    driver = "sqlx"
-    dumpDestination = ""
-    filePath = ""
+
+    # with postgres type
+    # db credentials
+    name     = "vulcanize_test" # DATABASE_NAME
+    hostname = "localhost"      # DATABASE_HOSTNAME
+    port     = 5432             # DATABASE_PORT
+    user     = "vulcanize"      # DATABASE_USER
+    password = "..."            # DATABASE_PASSWORD
+    # SQL backend to use: <sqlx | pgx>
+    driver   = "sqlx"           # DATABASE_DRIVER_TYPE
+
+    # with file type
+    # file mode <sql | csv>
+    fileMode = "csv"    # DATABASE_FILE_MODE
+
+    # with SQL file mode
+    filePath = ""   # DATABASE_FILE_PATH
+
+    # with CSV file mode
+    fileCsvDir = "output_dir" # DATABASE_FILE_CSV_DIR
+
+    # with dump type
+    # <stdout | stderr | discard>
+    dumpDestination = ""    # DATABASE_DUMP_DST
 
 [cache]
-    database = 1024
-    trie = 1024
+    # settings for geth internal caches
+    database = 1024 # DB_CACHE_SIZE_MB
+    trie     = 1024 # TRIE_CACHE_SIZE_MB
 
 [prom]
-    dbStats = false
-    metrics = true
-    http = true
-    httpAddr = "localhost"
-    httpPort = "8889"
+    # prometheus metrics
+    metrics  = true         # PROM_METRICS
+    http     = true         # PROM_HTTP
+    httpAddr = "localhost"  # PROM_HTTP_ADDR
+    httpPort = "8889"       # PROM_HTTP_PORT
+    dbStats = true          # PROM_DB_STATS
 
 [ethereum]
-    chainConfig = ""
-    nodeID = ""
-    clientName = "eth-statediff-service"
-    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
-    networkID = 1
-    chainID = 1
+    # Identifiers for ethereum node
+    nodeID       = ""                       # ETH_NODE_ID
+    clientName   = "eth-statediff-service"  # ETH_CLIENT_NAME
+    networkID    = 1                        # ETH_NETWORK_ID
+    chainID      = 1                        # ETH_CHAIN_ID
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # ETH_GENESIS_BLOCK
+
+    # Path to custom chain config file (optional)
+    # chainID should match that in this config file
+    chainConfig  = "chain.json"           # ETH_CHAIN_CONFIG
+
+[debug]
+    pprof = false                           # DEBUG_PPROF

fixture/.gitignore

@@ -0,0 +1,6 @@
+*/*.log
+*/CURRENT*
+*/LOCK
+*/LOG
+*/MANIFEST-*
+*/ancient/FLOCK

fixture/chaindata/ancient/bodies.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/diffs.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/hashes.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/headers.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

fixture/chaindata/ancient/receipts.meta

@@ -0,0 +1 @@
+<EFBFBD>
<01>

go.mod

@@ -1,62 +1,62 @@
-module github.com/vulcanize/eth-statediff-service
+module github.com/cerc-io/eth-statediff-service
 
-go 1.18
+go 1.19
 
 require (
-	github.com/ethereum/go-ethereum v1.10.18
-	github.com/jmoiron/sqlx v1.2.0
-	github.com/prometheus/client_golang v1.4.0
-	github.com/sirupsen/logrus v1.7.0
+	github.com/cerc-io/leveldb-ethdb-rpc v1.1.13
+	github.com/cerc-io/plugeth-statediff v0.0.0-00010101000000-000000000000
+	github.com/ethereum/go-ethereum v1.12.0
+	github.com/jmoiron/sqlx v1.3.5 // indirect
+	github.com/prometheus/client_golang v1.16.0
+	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.3.0
 	github.com/spf13/viper v1.10.1
-	github.com/vulcanize/go-eth-state-node-iterator v1.0.3
-	github.com/vulcanize/leveldb-ethdb-rpc v0.1.2
 )
 
 require (
-	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
-	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
+	github.com/DataDog/zstd v1.5.5 // indirect
+	github.com/VictoriaMetrics/fastcache v1.12.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
+	github.com/cerc-io/eth-iterator-utils v1.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.10.0 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230720154706-692f3b61a3c4 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230613231145-182959a1fad6 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/deckarep/golang-set v1.8.0 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/deckarep/golang-set/v2 v2.3.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
 	github.com/edsrzf/mmap-go v1.0.0 // indirect
-	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/fjl/memsize v0.0.1 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
 	github.com/georgysavva/scany v0.2.9 // indirect
-	github.com/go-ole/go-ole v1.2.1 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/getsentry/sentry-go v0.22.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/graph-gophers/graphql-go v1.3.0 // indirect
-	github.com/hashicorp/go-bexpr v0.1.10 // indirect
-	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/hashicorp/go-bexpr v0.1.12 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
-	github.com/holiman/uint256 v1.2.0 // indirect
-	github.com/huin/goupnp v1.0.3 // indirect
+	github.com/holiman/uint256 v1.2.3 // indirect
+	github.com/huin/goupnp v1.2.0 // indirect
+	github.com/inconshreveable/log15 v2.16.0+incompatible // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/influxdata/influxdb v1.8.3 // indirect
 	github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
+	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c // indirect
 	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
-	github.com/ipfs/bbloom v0.0.4 // indirect
-	github.com/ipfs/go-block-format v0.0.2 // indirect
-	github.com/ipfs/go-cid v0.0.7 // indirect
-	github.com/ipfs/go-datastore v0.4.2 // indirect
-	github.com/ipfs/go-ipfs-blockstore v1.0.1 // indirect
-	github.com/ipfs/go-ipfs-ds-help v1.0.0 // indirect
-	github.com/ipfs/go-ipfs-util v0.0.1 // indirect
-	github.com/ipfs/go-ipld-format v0.2.0 // indirect
-	github.com/ipfs/go-log v0.0.1 // indirect
-	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-cid v0.4.1 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgconn v1.10.0 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
@@ -67,65 +67,81 @@ require (
 	github.com/jackc/pgx/v4 v4.13.0 // indirect
 	github.com/jackc/puddle v1.1.3 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
-	github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 // indirect
-	github.com/lib/pq v1.10.2 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
-	github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 // indirect
-	github.com/mitchellh/mapstructure v1.4.3 // indirect
-	github.com/mitchellh/pointerstructure v1.2.0 // indirect
-	github.com/mr-tron/base58 v1.1.3 // indirect
-	github.com/multiformats/go-base32 v0.0.3 // indirect
-	github.com/multiformats/go-base36 v0.1.0 // indirect
-	github.com/multiformats/go-multibase v0.0.3 // indirect
-	github.com/multiformats/go-multihash v0.0.14 // indirect
-	github.com/multiformats/go-varint v0.0.5 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/pointerstructure v1.2.1 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/openrelayxyz/plugeth-utils v1.2.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
-	github.com/pganalyze/pg_query_go/v2 v2.1.0 // indirect
+	github.com/pganalyze/pg_query_go/v4 v4.2.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.9.1 // indirect
-	github.com/prometheus/procfs v0.0.8 // indirect
-	github.com/prometheus/tsdb v0.7.1 // indirect
-	github.com/rjeczalik/notify v0.9.1 // indirect
-	github.com/rs/cors v1.7.0 // indirect
-	github.com/shirou/gopsutil v3.21.5+incompatible // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.0 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/rs/cors v1.9.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/stretchr/testify v1.7.0 // indirect
+	github.com/status-im/keycard-go v0.2.0 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/stretchr/testify v1.8.2 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
-	github.com/thoas/go-funk v0.9.2 // indirect
-	github.com/tklauser/go-sysconf v0.3.6 // indirect
-	github.com/tklauser/numcpus v0.2.2 // indirect
-	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
-	github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
-	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/ini.v1 v1.66.2 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
+	github.com/thoas/go-funk v0.9.3 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
+	github.com/urfave/cli/v2 v2.25.7 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	golang.org/x/crypto v0.11.0 // indirect
+	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect
+	golang.org/x/net v0.10.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
+	golang.org/x/term v0.10.0 // indirect
+	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
-	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.10.18 => github.com/vulcanize/go-ethereum v1.10.18-statediff-3.2.1
+replace (
+	github.com/cerc-io/eth-iterator-utils => git.vdb.to/cerc-io/eth-iterator-utils v0.0.0-20230803115933-6bb6d4e27fd2
+	// github.com/cerc-io/plugeth-statediff => ../plugeth-statediff
+	// github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff v0.1.0
+	github.com/cerc-io/plugeth-statediff => git.vdb.to/cerc-io/plugeth-statediff v0.1.1-0.20230901070823-5ed22c03d29a
+	github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v0.0.0-20230808125822-691dc334fab1
+	github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v0.0.0-20230706160122-cd41de354c46
+)

main.go

@@ -15,7 +15,7 @@
 
 package main
 
-import "github.com/vulcanize/eth-statediff-service/cmd"
+import "github.com/cerc-io/eth-statediff-service/cmd"
 
 func main() {
 	cmd.Execute()

pkg/api.go

@@ -18,7 +18,7 @@ package statediff
 import (
 	"context"
 
-	sd "github.com/ethereum/go-ethereum/statediff"
+	sd "github.com/cerc-io/plugeth-statediff"
 )
 
 // APIName is the namespace used for the state diffing service API
@@ -30,11 +30,11 @@ const APIVersion = "0.0.1"
 // PublicStateDiffAPI provides an RPC interface
 // that can be used to fetch historical diffs from LevelDB directly
 type PublicStateDiffAPI struct {
-	sds StateDiffService
+	sds *Service
 }
 
 // NewPublicStateDiffAPI creates an rpc interface for the underlying statediff service
-func NewPublicStateDiffAPI(sds StateDiffService) *PublicStateDiffAPI {
+func NewPublicStateDiffAPI(sds *Service) *PublicStateDiffAPI {
 	return &PublicStateDiffAPI{
 		sds: sds,
 	}
@@ -45,11 +45,6 @@ func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint
 	return api.sds.StateDiffAt(blockNumber, params)
 }
 
-// StateTrieAt returns a state trie payload at the specific blockheight
-func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params sd.Params) (*sd.Payload, error) {
-	return api.sds.StateTrieAt(blockNumber, params)
-}
-
 // WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight
 func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params sd.Params) error {
 	return api.sds.WriteStateDiffAt(blockNumber, params)

pkg/builder.go

@@ -1,886 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Contains a batch of utility type declarations used by the tests. As the node
-// operates on unique types, a lot of them are needed to check various features.
-
-package statediff
-
-import (
-	"bytes"
-	"fmt"
-	"math/bits"
-	"sync"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/rlp"
-	sd "github.com/ethereum/go-ethereum/statediff"
-	sdtrie "github.com/ethereum/go-ethereum/statediff/trie_helpers"
-	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
-	"github.com/ethereum/go-ethereum/trie"
-	"github.com/sirupsen/logrus"
-
-	iter "github.com/vulcanize/go-eth-state-node-iterator"
-)
-
-var (
-	nullHashBytes     = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
-	emptyNode, _      = rlp.EncodeToBytes(&[]byte{})
-	emptyContractRoot = crypto.Keccak256Hash(emptyNode)
-	nullCodeHash      = crypto.Keccak256Hash([]byte{}).Bytes()
-)
-
-// Builder interface exposes the method for building a state diff between two blocks
-type Builder interface {
-	BuildStateDiffObject(args sd.Args, params sd.Params) (sdtypes.StateObject, error)
-	BuildStateTrieObject(current *types.Block) (sdtypes.StateObject, error)
-	WriteStateDiffObject(args sdtypes.StateRoots, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error
-}
-
-type builder struct {
-	stateCache state.Database
-	numWorkers uint
-}
-
-type iterPair struct {
-	older, newer trie.NodeIterator
-}
-
-func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (sdtypes.StateNode, []interface{}, error) {
-	nodePath := make([]byte, len(it.Path()))
-	copy(nodePath, it.Path())
-	node, err := trieDB.Node(it.Hash())
-	if err != nil {
-		return sdtypes.StateNode{}, nil, err
-	}
-	var nodeElements []interface{}
-	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
-		return sdtypes.StateNode{}, nil, err
-	}
-	ty, err := sdtrie.CheckKeyType(nodeElements)
-	if err != nil {
-		return sdtypes.StateNode{}, nil, err
-	}
-	return sdtypes.StateNode{
-		NodeType:  ty,
-		Path:      nodePath,
-		NodeValue: node,
-	}, nodeElements, nil
-}
-
-// convenience
-func stateNodeAppender(nodes *[]sdtypes.StateNode) sdtypes.StateNodeSink {
-	return func(node sdtypes.StateNode) error {
-		*nodes = append(*nodes, node)
-		return nil
-	}
-}
-func storageNodeAppender(nodes *[]sdtypes.StorageNode) sdtypes.StorageNodeSink {
-	return func(node sdtypes.StorageNode) error {
-		*nodes = append(*nodes, node)
-		return nil
-	}
-}
-func codeMappingAppender(data *[]sdtypes.CodeAndCodeHash) sdtypes.CodeSink {
-	return func(c sdtypes.CodeAndCodeHash) error {
-		*data = append(*data, c)
-		return nil
-	}
-}
-
-// NewBuilder is used to create a statediff builder
-func NewBuilder(stateCache state.Database, workers uint) (Builder, error) {
-	if workers == 0 {
-		workers = 1
-	}
-	if bits.OnesCount(workers) != 1 {
-		return nil, fmt.Errorf("workers must be a power of 2")
-	}
-	return &builder{
-		stateCache: stateCache, // state cache is safe for concurrent reads
-		numWorkers: workers,
-	}, nil
-}
-
-// BuildStateTrieObject builds a state trie object from the provided block
-func (sdb *builder) BuildStateTrieObject(current *types.Block) (sdtypes.StateObject, error) {
-	currentTrie, err := sdb.stateCache.OpenTrie(current.Root())
-	if err != nil {
-		return sdtypes.StateObject{}, fmt.Errorf("error creating trie for block %d: %v", current.Number(), err)
-	}
-	it := currentTrie.NodeIterator([]byte{})
-	stateNodes, codeAndCodeHashes, err := sdb.buildStateTrie(it)
-	if err != nil {
-		return sdtypes.StateObject{}, fmt.Errorf("error collecting state nodes for block %d: %v", current.Number(), err)
-	}
-	return sdtypes.StateObject{
-		BlockNumber:       current.Number(),
-		BlockHash:         current.Hash(),
-		Nodes:             stateNodes,
-		CodeAndCodeHashes: codeAndCodeHashes,
-	}, nil
-}
-
-func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]sdtypes.StateNode, []sdtypes.CodeAndCodeHash, error) {
-	stateNodes := make([]sdtypes.StateNode, 0)
-	codeAndCodeHashes := make([]sdtypes.CodeAndCodeHash, 0)
-	for it.Next(true) {
-		// skip value nodes
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return nil, nil, err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			var account types.StateAccount
-			if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
-				return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
-			}
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			node.LeafKey = leafKey
-			if !bytes.Equal(account.CodeHash, nullCodeHash) {
-				var storageNodes []sdtypes.StorageNode
-				err := sdb.buildStorageNodesEventual(account.Root, true, storageNodeAppender(&storageNodes))
-				if err != nil {
-					return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err)
-				}
-				node.StorageNodes = storageNodes
-				// emit codehash => code mappings for code
-				codeHash := common.BytesToHash(account.CodeHash)
-				code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash)
-				if err != nil {
-					return nil, nil, fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
-				}
-				codeAndCodeHashes = append(codeAndCodeHashes, sdtypes.CodeAndCodeHash{
-					Hash: codeHash,
-					Code: code,
-				})
-			}
-			stateNodes = append(stateNodes, node)
-		case sdtypes.Extension, sdtypes.Branch:
-			stateNodes = append(stateNodes, node)
-		default:
-			return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-	}
-	return stateNodes, codeAndCodeHashes, it.Error()
-}
-
-// BuildStateDiffObject builds a statediff object from two blocks and the provided parameters
-func (sdb *builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sdtypes.StateObject, error) {
-	var stateNodes []sdtypes.StateNode
-	var codeAndCodeHashes []sdtypes.CodeAndCodeHash
-	err := sdb.WriteStateDiffObject(
-		sdtypes.StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot},
-		params, stateNodeAppender(&stateNodes), codeMappingAppender(&codeAndCodeHashes))
-	if err != nil {
-		return sdtypes.StateObject{}, err
-	}
-	return sdtypes.StateObject{
-		BlockHash:         args.BlockHash,
-		BlockNumber:       args.BlockNumber,
-		Nodes:             stateNodes,
-		CodeAndCodeHashes: codeAndCodeHashes,
-	}, nil
-}
-
-// WriteStateDiffObject writes a statediff object to output callback
-func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
-	if len(params.WatchedAddresses) > 0 {
-		// if we are watching only specific accounts then we are only diffing leaf nodes
-		log.Info("Ignoring intermediate state nodes because WatchedAddresses was passed")
-		params.IntermediateStateNodes = false
-	}
-
-	// Load tries for old and new states
-	oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot)
-	if err != nil {
-		return fmt.Errorf("error creating trie for oldStateRoot: %v", err)
-	}
-	newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot)
-	if err != nil {
-		return fmt.Errorf("error creating trie for newStateRoot: %v", err)
-	}
-
-	// Split old and new tries into corresponding subtrie iterators
-	oldIters1 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
-	oldIters2 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
-	newIters1 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
-	newIters2 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
-
-	// Create iterators ahead of time to avoid race condition in state.Trie access
-	// We do two state iterations per subtrie: one for new/updated nodes,
-	// one for deleted/updated nodes; prepare 2 iterator instances for each task
-	var iterPairs [][]iterPair
-	for i := uint(0); i < sdb.numWorkers; i++ {
-		iterPairs = append(iterPairs, []iterPair{
-			{older: oldIters1[i], newer: newIters1[i]},
-			{older: oldIters2[i], newer: newIters2[i]},
-		})
-	}
-
-	// Dispatch workers to process trie data; sync and collect results here via channels
-	nodeChan := make(chan sdtypes.StateNode)
-	codeChan := make(chan sdtypes.CodeAndCodeHash)
-
-	go func() {
-		nodeSender := func(node sdtypes.StateNode) error { nodeChan <- node; return nil }
-		codeSender := func(code sdtypes.CodeAndCodeHash) error { codeChan <- code; return nil }
-		var wg sync.WaitGroup
-
-		for w := uint(0); w < sdb.numWorkers; w++ {
-			wg.Add(1)
-			go func(worker uint) {
-				defer wg.Done()
-				if err := sdb.buildStateDiff(iterPairs[worker], params, nodeSender, codeSender); err != nil {
-					logrus.Errorf("buildStateDiff error for worker %d, pparams %+v", worker, params)
-				}
-			}(w)
-		}
-		wg.Wait()
-		close(nodeChan)
-		close(codeChan)
-	}()
-
-	for nodeChan != nil || codeChan != nil {
-		select {
-		case node, more := <-nodeChan:
-			if more {
-				if err := output(node); err != nil {
-					return err
-				}
-			} else {
-				nodeChan = nil
-			}
-		case codeAndCodeHash, more := <-codeChan:
-			if more {
-				if err := codeOutput(codeAndCodeHash); err != nil {
-					return err
-				}
-			} else {
-				codeChan = nil
-			}
-		}
-	}
-
-	return nil
-}
-
-func (sdb *builder) buildStateDiff(args []iterPair, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
-	// collect a slice of all the intermediate nodes that were touched and exist at B
-	// a map of their leafkey to all the accounts that were touched and exist at B
-	// and a slice of all the paths for the nodes in both of the above sets
-	var diffAccountsAtB AccountMap
-	var diffPathsAtB map[string]bool
-	var err error
-	if params.IntermediateStateNodes {
-		diffAccountsAtB, diffPathsAtB, err = sdb.createdAndUpdatedStateWithIntermediateNodes(args[0], output)
-	} else {
-		diffAccountsAtB, diffPathsAtB, err = sdb.createdAndUpdatedState(args[0], params.WatchedAddressesLeafKeys())
-	}
-
-	if err != nil {
-		return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
-	}
-
-	// collect a slice of all the nodes that existed at a path in A that doesn't exist in B
-	// a map of their leafkey to all the accounts that were touched and exist at A
-	diffAccountsAtA, err := sdb.deletedOrUpdatedState(args[1], diffAccountsAtB, diffPathsAtB, params.WatchedAddressesLeafKeys(), params.IntermediateStorageNodes, output)
-	if err != nil {
-		return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
-	}
-
-	// collect and sort the leafkeys for both account mappings into a slice
-	createKeys := sortKeys(diffAccountsAtB)
-	deleteKeys := sortKeys(diffAccountsAtA)
-
-	// and then find the intersection of these keys
-	// these are the leafkeys for the accounts which exist at both A and B but are different
-	// this also mutates the passed in createKeys and deleteKeys, removing the intersection keys
-	// and leaving the truly created or deleted keys in place
-	updatedKeys := findIntersection(createKeys, deleteKeys)
-
-	// build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
-	err = sdb.buildAccountUpdates(
-		diffAccountsAtB, diffAccountsAtA, updatedKeys, params.IntermediateStorageNodes, output)
-	if err != nil {
-		return fmt.Errorf("error building diff for updated accounts: %v", err)
-	}
-	// build the diff nodes for created accounts
-	err = sdb.buildAccountCreations(diffAccountsAtB, params.IntermediateStorageNodes, output, codeOutput)
-	if err != nil {
-		return fmt.Errorf("error building diff for created accounts: %v", err)
-	}
-	return nil
-}
-
-// createdAndUpdatedState returns
-// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
-// and a slice of the paths for all of the nodes included in both
-func (sdb *builder) createdAndUpdatedState(iters iterPair, watchedAddressesLeafKeys map[common.Hash]struct{}) (AccountMap, map[string]bool, error) {
-	diffPathsAtB := make(map[string]bool)
-	diffAcountsAtB := make(AccountMap)
-	it, _ := trie.NewDifferenceIterator(iters.older, iters.newer)
-	for it.Next(true) {
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return nil, nil, err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			// created vs updated is important for leaf nodes since we need to diff their storage
-			// so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey
-			var account types.StateAccount
-			if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
-				return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
-			}
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			if isWatchedAddress(watchedAddressesLeafKeys, leafKey) {
-				diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
-					NodeType:  node.NodeType,
-					Path:      node.Path,
-					NodeValue: node.NodeValue,
-					LeafKey:   leafKey,
-					Account:   &account,
-				}
-			}
-		}
-		// add both intermediate and leaf node paths to the list of diffPathsAtB
-		diffPathsAtB[common.Bytes2Hex(node.Path)] = true
-	}
-	return diffAcountsAtB, diffPathsAtB, it.Error()
-}
-
-// createdAndUpdatedStateWithIntermediateNodes returns
-// a slice of all the intermediate nodes that exist in a different state at B than A
-// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
-// and a slice of the paths for all of the nodes included in both
-func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(iters iterPair, output sdtypes.StateNodeSink) (AccountMap, map[string]bool, error) {
-	diffPathsAtB := make(map[string]bool)
-	diffAcountsAtB := make(AccountMap)
-	it, _ := trie.NewDifferenceIterator(iters.older, iters.newer)
-	for it.Next(true) {
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return nil, nil, err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			// created vs updated is important for leaf nodes since we need to diff their storage
-			// so we need to map all changed accounts at B to their leafkey, since account can change paths but not leafkey
-			var account types.StateAccount
-			if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
-				return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
-			}
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
-				NodeType:  node.NodeType,
-				Path:      node.Path,
-				NodeValue: node.NodeValue,
-				LeafKey:   leafKey,
-				Account:   &account,
-			}
-		case sdtypes.Extension, sdtypes.Branch:
-			// create a diff for any intermediate node that has changed at b
-			// created vs updated makes no difference for intermediate nodes since we do not need to diff storage
-			if err := output(sdtypes.StateNode{
-				NodeType:  node.NodeType,
-				Path:      node.Path,
-				NodeValue: node.NodeValue,
-			}); err != nil {
-				return nil, nil, err
-			}
-		default:
-			return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-		// add both intermediate and leaf node paths to the list of diffPathsAtB
-		diffPathsAtB[common.Bytes2Hex(node.Path)] = true
-	}
-	return diffAcountsAtB, diffPathsAtB, it.Error()
-}
-
-// deletedOrUpdatedState returns a slice of all the paths that are emptied at B
-// and a mapping of their leafkeys to all the accounts that exist in a different state at A than B
-func (sdb *builder) deletedOrUpdatedState(iters iterPair, diffAccountsAtB AccountMap, diffPathsAtB map[string]bool, watchedAddressesLeafKeys map[common.Hash]struct{}, intermediateStorageNodes bool, output sdtypes.StateNodeSink) (AccountMap, error) {
-	diffAccountAtA := make(AccountMap)
-	it, _ := trie.NewDifferenceIterator(iters.newer, iters.older)
-	for it.Next(true) {
-		// skip value nodes
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return nil, err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			// map all different accounts at A to their leafkey
-			var account types.StateAccount
-			if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
-				return nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
-			}
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			if isWatchedAddress(watchedAddressesLeafKeys, leafKey) {
-				diffAccountAtA[common.Bytes2Hex(leafKey)] = accountWrapper{
-					NodeType:  node.NodeType,
-					Path:      node.Path,
-					NodeValue: node.NodeValue,
-					LeafKey:   leafKey,
-					Account:   &account,
-				}
-				// if this node's path did not show up in diffPathsAtB
-				// that means the node at this path was deleted (or moved) in B
-				if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
-					var diff sdtypes.StateNode
-					// if this node's leaf key also did not show up in diffAccountsAtB
-					// that means the node was deleted
-					// in that case, emit an empty "removed" diff state node
-					// include empty "removed" diff storage nodes for all the storage slots
-					if _, ok := diffAccountsAtB[common.Bytes2Hex(leafKey)]; !ok {
-						diff = sdtypes.StateNode{
-							NodeType:  sdtypes.Removed,
-							Path:      node.Path,
-							LeafKey:   leafKey,
-							NodeValue: []byte{},
-						}
-
-						var storageDiffs []sdtypes.StorageNode
-						err := sdb.buildRemovedAccountStorageNodes(account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
-						if err != nil {
-							return nil, fmt.Errorf("failed building storage diffs for removed node %x\r\nerror: %v", node.Path, err)
-						}
-						diff.StorageNodes = storageDiffs
-					} else {
-						// emit an empty "removed" diff with empty leaf key if the account was moved
-						diff = sdtypes.StateNode{
-							NodeType:  sdtypes.Removed,
-							Path:      node.Path,
-							NodeValue: []byte{},
-						}
-					}
-
-					if err := output(diff); err != nil {
-						return nil, err
-					}
-				}
-			}
-		case sdtypes.Extension, sdtypes.Branch:
-			// if this node's path did not show up in diffPathsAtB
-			// that means the node at this path was deleted (or moved) in B
-			// emit an empty "removed" diff to signify as such
-			if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
-				if err := output(sdtypes.StateNode{
-					Path:      node.Path,
-					NodeValue: []byte{},
-					NodeType:  sdtypes.Removed,
-				}); err != nil {
-					return nil, err
-				}
-			}
-			// fall through, we did everything we need to do with these node types
-		default:
-			return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-	}
-	return diffAccountAtA, it.Error()
-}
-
-// buildAccountUpdates uses the account diffs maps for A => B and B => A and the known intersection of their leafkeys
-// to generate the statediff node objects for all of the accounts that existed at both A and B but in different states
-// needs to be called before building account creations and deletions as this mutates
-// those account maps to remove the accounts which were updated
-func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string, intermediateStorageNodes bool, output sdtypes.StateNodeSink) error {
-	var err error
-	for _, key := range updatedKeys {
-		createdAcc := creations[key]
-		deletedAcc := deletions[key]
-		var storageDiffs []sdtypes.StorageNode
-		if deletedAcc.Account != nil && createdAcc.Account != nil {
-			oldSR := deletedAcc.Account.Root
-			newSR := createdAcc.Account.Root
-			err = sdb.buildStorageNodesIncremental(oldSR, newSR, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
-			if err != nil {
-				return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err)
-			}
-		}
-		if err = output(sdtypes.StateNode{
-			NodeType:     createdAcc.NodeType,
-			Path:         createdAcc.Path,
-			NodeValue:    createdAcc.NodeValue,
-			LeafKey:      createdAcc.LeafKey,
-			StorageNodes: storageDiffs,
-		}); err != nil {
-			return err
-		}
-		delete(creations, key)
-		delete(deletions, key)
-	}
-
-	return nil
-}
-
-// buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A
-// it also returns the code and codehash for created contract accounts
-func (sdb *builder) buildAccountCreations(accounts AccountMap, intermediateStorageNodes bool, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
-	for _, val := range accounts {
-		diff := sdtypes.StateNode{
-			NodeType:  val.NodeType,
-			Path:      val.Path,
-			LeafKey:   val.LeafKey,
-			NodeValue: val.NodeValue,
-		}
-		if !bytes.Equal(val.Account.CodeHash, nullCodeHash) {
-			// For contract creations, any storage node contained is a diff
-			var storageDiffs []sdtypes.StorageNode
-			err := sdb.buildStorageNodesEventual(val.Account.Root, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
-			if err != nil {
-				return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err)
-			}
-			diff.StorageNodes = storageDiffs
-			// emit codehash => code mappings for code
-			codeHash := common.BytesToHash(val.Account.CodeHash)
-			code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash)
-			if err != nil {
-				return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
-			}
-			if err := codeOutput(sdtypes.CodeAndCodeHash{
-				Hash: codeHash,
-				Code: code,
-			}); err != nil {
-				return err
-			}
-		}
-		if err := output(diff); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// buildStorageNodesEventual builds the storage diff node objects for a created account
-// i.e. it returns all the storage nodes at this state, since there is no previous state
-func (sdb *builder) buildStorageNodesEventual(sr common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
-	if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
-		return nil
-	}
-	log.Debug("Storage Root For Eventual Diff", "root", sr.Hex())
-	sTrie, err := sdb.stateCache.OpenTrie(sr)
-	if err != nil {
-		log.Info("error in build storage diff eventual", "error", err)
-		return err
-	}
-	it := sTrie.NodeIterator(make([]byte, 0))
-	err = sdb.buildStorageNodesFromTrie(it, intermediateNodes, output)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// buildStorageNodesFromTrie returns all the storage diff node objects in the provided node iterator
-// if any storage keys are provided it will only return those leaf nodes
-// including intermediate nodes can be turned on or off
-func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
-	for it.Next(true) {
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			if err := output(sdtypes.StorageNode{
-				NodeType:  node.NodeType,
-				Path:      node.Path,
-				NodeValue: node.NodeValue,
-				LeafKey:   leafKey,
-			}); err != nil {
-				return err
-			}
-		case sdtypes.Extension, sdtypes.Branch:
-			if intermediateNodes {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  node.NodeType,
-					Path:      node.Path,
-					NodeValue: node.NodeValue,
-				}); err != nil {
-					return err
-				}
-			}
-		default:
-			return fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-	}
-	return it.Error()
-}
-
-// buildRemovedAccountStorageNodes builds the "removed" diffs for all the storage nodes for a destroyed account
-func (sdb *builder) buildRemovedAccountStorageNodes(sr common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
-	if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
-		return nil
-	}
-	log.Debug("Storage Root For Removed Diffs", "root", sr.Hex())
-	sTrie, err := sdb.stateCache.OpenTrie(sr)
-	if err != nil {
-		log.Info("error in build removed account storage diffs", "error", err)
-		return err
-	}
-	it := sTrie.NodeIterator(make([]byte, 0))
-	err = sdb.buildRemovedStorageNodesFromTrie(it, intermediateNodes, output)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// buildRemovedStorageNodesFromTrie returns diffs for all the storage nodes in the provided node interator
-// including intermediate nodes can be turned on or off
-func (sdb *builder) buildRemovedStorageNodesFromTrie(it trie.NodeIterator, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
-	for it.Next(true) {
-		// skip value nodes
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			if err := output(sdtypes.StorageNode{
-				NodeType:  sdtypes.Removed,
-				Path:      node.Path,
-				NodeValue: []byte{},
-				LeafKey:   leafKey,
-			}); err != nil {
-				return err
-			}
-		case sdtypes.Extension, sdtypes.Branch:
-			if intermediateNodes {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  sdtypes.Removed,
-					Path:      node.Path,
-					NodeValue: []byte{},
-				}); err != nil {
-					return err
-				}
-			}
-		default:
-			return fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-	}
-	return it.Error()
-}
-
-// buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A
-func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
-	if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) {
-		return nil
-	}
-	log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex())
-	oldTrie, err := sdb.stateCache.OpenTrie(oldSR)
-	if err != nil {
-		return err
-	}
-	newTrie, err := sdb.stateCache.OpenTrie(newSR)
-	if err != nil {
-		return err
-	}
-
-	diffSlotsAtB, diffPathsAtB, err := sdb.createdAndUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), intermediateNodes, output)
-	if err != nil {
-		return err
-	}
-	err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), diffSlotsAtB, diffPathsAtB, intermediateNodes, output)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, intermediateNodes bool, output sdtypes.StorageNodeSink) (map[string]bool, map[string]bool, error) {
-	diffPathsAtB := make(map[string]bool)
-	diffSlotsAtB := make(map[string]bool)
-	it, _ := trie.NewDifferenceIterator(a, b)
-	for it.Next(true) {
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return nil, nil, err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			diffSlotsAtB[common.Bytes2Hex(leafKey)] = true
-			if err := output(sdtypes.StorageNode{
-				NodeType:  node.NodeType,
-				Path:      node.Path,
-				NodeValue: node.NodeValue,
-				LeafKey:   leafKey,
-			}); err != nil {
-				return nil, nil, err
-			}
-		case sdtypes.Extension, sdtypes.Branch:
-			if intermediateNodes {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  node.NodeType,
-					Path:      node.Path,
-					NodeValue: node.NodeValue,
-				}); err != nil {
-					return nil, nil, err
-				}
-			}
-		default:
-			return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-		diffPathsAtB[common.Bytes2Hex(node.Path)] = true
-	}
-	return diffSlotsAtB, diffPathsAtB, it.Error()
-}
-
-func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffSlotsAtB, diffPathsAtB map[string]bool, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
-	it, _ := trie.NewDifferenceIterator(b, a)
-	for it.Next(true) {
-		// skip value nodes
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := sdtrie.ResolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return err
-		}
-
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-
-			// if this node's path did not show up in diffPathsAtB
-			// that means the node at this path was deleted (or moved) in B
-			if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
-				// if this node's leaf key also did not show up in diffSlotsAtB
-				// that means the node was deleted
-				// in that case, emit an empty "removed" diff storage node
-				if _, ok := diffSlotsAtB[common.Bytes2Hex(leafKey)]; !ok {
-					if err := output(sdtypes.StorageNode{
-						NodeType:  sdtypes.Removed,
-						Path:      node.Path,
-						NodeValue: []byte{},
-						LeafKey:   leafKey,
-					}); err != nil {
-						return err
-					}
-				} else {
-					// emit an empty "removed" diff with empty leaf key if the account was moved
-					if err := output(sdtypes.StorageNode{
-						NodeType:  sdtypes.Removed,
-						Path:      node.Path,
-						NodeValue: []byte{},
-					}); err != nil {
-						return err
-					}
-				}
-			}
-		case sdtypes.Extension, sdtypes.Branch:
-			// if this node's path did not show up in diffPathsAtB
-			// that means the node at this path was deleted in B
-			// in that case, emit an empty "removed" diff storage node
-			if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
-				if intermediateNodes {
-					if err := output(sdtypes.StorageNode{
-						NodeType:  sdtypes.Removed,
-						Path:      node.Path,
-						NodeValue: []byte{},
-					}); err != nil {
-						return err
-					}
-				}
-			}
-		default:
-			return fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-	}
-	return it.Error()
-}
-
-// isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch
-func isWatchedAddress(watchedAddressesLeafKeys map[common.Hash]struct{}, stateLeafKey []byte) bool {
-	// If we aren't watching any specific addresses, we are watching everything
-	if len(watchedAddressesLeafKeys) == 0 {
-		return true
-	}
-
-	_, ok := watchedAddressesLeafKeys[common.BytesToHash(stateLeafKey)]
-	return ok
-}
-
-// isWatchedStorageKey is used to check if a storage leaf corresponds to one of the storage slots the builder is configured to watch
-func isWatchedStorageKey(watchedKeys []common.Hash, storageLeafKey []byte) bool {
-	// If we aren't watching any specific addresses, we are watching everything
-	if len(watchedKeys) == 0 {
-		return true
-	}
-	for _, hashKey := range watchedKeys {
-		if bytes.Equal(hashKey.Bytes(), storageLeafKey) {
-			return true
-		}
-	}
-	return false
-}

pkg/config.go

@@ -1,7 +1,7 @@
 package statediff
 
-// Config holds config params for the statediffing service
-type Config struct {
+// ServiceConfig holds config params for the statediffing service
+type ServiceConfig struct {
 	ServiceWorkers  uint
 	TrieWorkers     uint
 	WorkerQueueSize uint

pkg/helpers.go

@@ -22,9 +22,11 @@ package statediff
 import (
 	"sort"
 	"strings"
+
+	sdtypes "github.com/cerc-io/plugeth-statediff/types"
 )
 
-func sortKeys(data AccountMap) []string {
+func sortKeys(data sdtypes.AccountMap) []string {
 	keys := make([]string, 0, len(data))
 	for key := range data {
 		keys = append(keys, key)

pkg/prom/db_stats_collector.go

@@ -17,9 +17,9 @@
 package prom
 
 import (
-	"database/sql"
-
 	"github.com/prometheus/client_golang/prometheus"
+
+	dbmetrics "github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
 )
 
 const (
@@ -29,7 +29,7 @@ const (
 
 // DBStatsGetter is an interface that gets sql.DBStats.
 type DBStatsGetter interface {
-	Stats() sql.DBStats
+	Stats() dbmetrics.DbStats
 }
 
 // DBStatsCollector implements the prometheus.Collector interface.
@@ -122,41 +122,41 @@ func (c DBStatsCollector) Collect(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(
 		c.maxOpenDesc,
 		prometheus.GaugeValue,
-		float64(stats.MaxOpenConnections),
+		float64(stats.MaxOpen()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.openDesc,
 		prometheus.GaugeValue,
-		float64(stats.OpenConnections),
+		float64(stats.Open()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.inUseDesc,
 		prometheus.GaugeValue,
-		float64(stats.InUse),
+		float64(stats.InUse()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.idleDesc,
 		prometheus.GaugeValue,
-		float64(stats.Idle),
+		float64(stats.Idle()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.waitedForDesc,
 		prometheus.CounterValue,
-		float64(stats.WaitCount),
+		float64(stats.WaitCount()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.blockedSecondsDesc,
 		prometheus.CounterValue,
-		stats.WaitDuration.Seconds(),
+		stats.WaitDuration().Seconds(),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.closedMaxIdleDesc,
 		prometheus.CounterValue,
-		float64(stats.MaxIdleClosed),
+		float64(stats.MaxIdleClosed()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.closedMaxLifetimeDesc,
 		prometheus.CounterValue,
-		float64(stats.MaxLifetimeClosed),
+		float64(stats.MaxLifetimeClosed()),
 	)
 }

pkg/prom/prom.go

@@ -19,7 +19,6 @@ package prom
 import (
 	"time"
 
-	"github.com/jmoiron/sqlx"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
@@ -123,7 +122,7 @@ func Init() {
 }
 
 // RegisterDBCollector create metric collector for given connection
-func RegisterDBCollector(name string, db *sqlx.DB) {
+func RegisterDBCollector(name string, db DBStatsGetter) {
 	if metrics {
 		prometheus.Register(NewDBStatsCollector(name, db))
 	}

pkg/reader.go

@@ -16,9 +16,11 @@
 package statediff
 
 import (
+	"errors"
 	"fmt"
 	"math/big"
 
+	"github.com/cerc-io/leveldb-ethdb-rpc/pkg/client"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
@@ -26,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
-	"github.com/vulcanize/leveldb-ethdb-rpc/pkg/client"
 )
 
 // Reader interface required by the statediffing service
@@ -36,6 +37,7 @@ type Reader interface {
 	GetReceiptsByHash(hash common.Hash) (types.Receipts, error)
 	GetTdByHash(hash common.Hash) (*big.Int, error)
 	StateDB() state.Database
+	GetLatestHeader() (*types.Header, error)
 }
 
 // LvlDBReader exposes the necessary Reader methods on lvldb
@@ -54,22 +56,29 @@ type LvLDBReaderConfig struct {
 	DBCacheSize            int
 }
 
-// NewLvlDBReader creates a new Read using LevelDB
+// NewLvlDBReader creates a new Reader using LevelDB
 func NewLvlDBReader(conf LvLDBReaderConfig) (*LvlDBReader, error) {
 	var edb ethdb.Database
 	var err error
 
-	if conf.Mode == "local" {
-		edb, err = rawdb.NewLevelDBDatabaseWithFreezer(conf.Path, conf.DBCacheSize, 256, conf.AncientPath, "eth-statediff-service", true)
-	}
+	switch conf.Mode {
+	case "local":
+		edb, err = rawdb.NewLevelDBDatabase(conf.Path, conf.DBCacheSize, 256, "eth-statediff-service", true)
+		if err != nil {
+			return nil, err
+		}
 
-	if conf.Mode == "remote" {
+		edb, err = rawdb.NewDatabaseWithFreezer(edb, conf.AncientPath, "eth-statediff-service", true)
+		if err != nil {
+			return nil, err
+		}
+	case "remote":
 		edb, err = client.NewDatabaseClient(conf.Url)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	if err != nil {
-		return nil, err
-	}
 	return &LvlDBReader{
 		ethDB:       edb,
 		stateDB:     state.NewDatabaseWithConfig(edb, conf.TrieConfig),
@@ -129,3 +138,12 @@ func (ldr *LvlDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
 func (ldr *LvlDBReader) StateDB() state.Database {
 	return ldr.stateDB
 }
+
+// GetLatestHeader gets the latest header from the levelDB
+func (ldr *LvlDBReader) GetLatestHeader() (*types.Header, error) {
+	header := rawdb.ReadHeadHeader(ldr.ethDB)
+	if header == nil {
+		return nil, errors.New("unable to read head header")
+	}
+	return header, nil
+}

pkg/rpc/http.go

@@ -24,14 +24,14 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )
 
 // StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules.
 func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (*rpc.Server, error) {
 
 	srv := rpc.NewServer()
-	err := node.RegisterApis(apis, modules, srv, false)
+	err := node.RegisterApis(apis, modules, srv)
 	if err != nil {
 		utils.Fatalf("Could not register HTTP API: %w", err)
 	}

pkg/rpc/ipc.go

@@ -26,7 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )
 
 var (
@@ -79,7 +79,7 @@ func StartIPCEndpoint(ipcEndpoint string, apis []rpc.API) (net.Listener, *rpc.Se
 		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
 			return nil, nil, err
 		}
-		log.Debug("IPC registered", "namespace", api.Namespace)
+		log.WithField("namespace", api.Namespace).Debug("IPC server registered")
 	}
 	// All APIs registered, start the IPC listener.
 	listener, err := ipcListen(ipcEndpoint)

pkg/service.go

@@ -22,51 +22,26 @@ import (
 	"sync"
 	"time"
 
+	"github.com/cerc-io/plugeth-statediff"
+	"github.com/cerc-io/plugeth-statediff/adapt"
+	"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
+	sdtypes "github.com/cerc-io/plugeth-statediff/types"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	sd "github.com/ethereum/go-ethereum/statediff"
-	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
-	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )
 
 const defaultQueueSize = 1024
 
-// StateDiffService is the state-diffing service interface
-type StateDiffService interface {
-	// Lifecycle Start() and Stop()
-	node.Lifecycle
-	// APIs and Protocols() interface for node service registration
-	APIs() []rpc.API
-	Protocols() []p2p.Protocol
-	// Loop is the main event loop for processing state diffs
-	Loop(wg *sync.WaitGroup) error
-	// Run is a one-off command to run on a predefined set of ranges
-	Run(ranges []RangeRequest) error
-	// StateDiffAt method to get state diff object at specific block
-	StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
-	// StateDiffFor method to get state diff object at specific block
-	StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.Payload, error)
-	// StateTrieAt method to get state trie object at specific block
-	StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
-	// WriteStateDiffAt method to write state diff object directly to DB
-	WriteStateDiffAt(blockNumber uint64, params sd.Params) error
-	// WriteStateDiffFor method to get state trie object at specific block
-	WriteStateDiffFor(blockHash common.Hash, params sd.Params) error
-	// WriteStateDiffsInRange method to wrtie state diff objects within the range directly to the DB
-	WriteStateDiffsInRange(start, stop uint64, params sd.Params) error
-}
-
 // Service is the underlying struct for the state diffing service
 type Service struct {
 	// Used to build the state diff objects
-	Builder Builder
+	Builder statediff.Builder
 	// Used to read data from LevelDB
 	lvlDBReader Reader
 	// Used to signal shutdown of the service
@@ -82,22 +57,20 @@ type Service struct {
 }
 
 // NewStateDiffService creates a new Service
-func NewStateDiffService(lvlDBReader Reader, indexer interfaces.StateDiffIndexer, conf Config) (*Service, error) {
-	b, err := NewBuilder(lvlDBReader.StateDB(), conf.TrieWorkers)
-	if err != nil {
-		return nil, err
-	}
+func NewStateDiffService(lvlDBReader Reader, indexer interfaces.StateDiffIndexer, conf ServiceConfig) *Service {
+	builder := statediff.NewBuilder(adapt.GethStateView(lvlDBReader.StateDB()))
+	builder.SetSubtrieWorkers(conf.TrieWorkers)
 	if conf.WorkerQueueSize == 0 {
 		conf.WorkerQueueSize = defaultQueueSize
 	}
 	return &Service{
 		lvlDBReader: lvlDBReader,
-		Builder:     b,
+		Builder:     builder,
 		indexer:     indexer,
 		workers:     conf.ServiceWorkers,
 		queue:       make(chan RangeRequest, conf.WorkerQueueSize),
 		preruns:     conf.PreRuns,
-	}, nil
+	}
 }
 
 // Protocols exports the services p2p protocols, this service has none
@@ -117,19 +90,82 @@ func (sds *Service) APIs() []rpc.API {
 	}
 }
 
+func segmentRange(workers, start, stop uint64, params statediff.Params) []RangeRequest {
+	segmentSize := ((stop - start) + 1) / workers
+	remainder := ((stop - start) + 1) % workers
+	numOfSegments := workers
+	if remainder > 0 {
+		numOfSegments++
+	}
+	segments := make([]RangeRequest, numOfSegments)
+	for i := range segments {
+		end := start + segmentSize - 1
+		if end > stop {
+			end = stop
+		}
+		segments[i] = RangeRequest{start, end, params}
+		start = end + 1
+	}
+	return segments
+}
+
 // Run does a one-off processing run on the provided RangeRequests + any pre-runs, exiting afterwards
-func (sds *Service) Run(rngs []RangeRequest) error {
+func (sds *Service) Run(rngs []RangeRequest, parallel bool) error {
 	for _, preRun := range sds.preruns {
-		logrus.Infof("processing prerun range (%d, %d)", preRun.Start, preRun.Stop)
-		for i := preRun.Start; i <= preRun.Stop; i++ {
-			if err := sds.WriteStateDiffAt(i, preRun.Params); err != nil {
-				return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, preRun.Start, preRun.Stop, err)
+		// if the rangeSize is smaller than the number of workers
+		// make sure we do synchronous processing to avoid quantization issues
+		rangeSize := (preRun.Stop - preRun.Start) + 1
+		numWorkers := uint64(sds.workers)
+		if rangeSize < numWorkers {
+			parallel = false
+		}
+		if parallel {
+			logrus.Infof("parallel processing prerun range (%d, %d) (%d blocks) divided into %d sized chunks with %d workers", preRun.Start, preRun.Stop,
+				rangeSize, rangeSize/numWorkers, numWorkers)
+			workChan := make(chan RangeRequest)
+			quitChan := make(chan struct{})
+			// spin up numWorkers number of worker goroutines
+			wg := new(sync.WaitGroup)
+			for i := 0; i < int(numWorkers); i++ {
+				wg.Add(1)
+				go func(id int) {
+					defer wg.Done()
+					for {
+						select {
+						case workerSegment := <-workChan:
+							for j := workerSegment.Start; j <= workerSegment.Stop; j++ {
+								if err := sds.WriteStateDiffAt(j, workerSegment.Params); err != nil {
+									logrus.Errorf("error writing statediff at height %d in range (%d, %d) : %v", id, workerSegment.Start, workerSegment.Stop, err)
+								}
+							}
+							logrus.Infof("prerun worker %d finished processing range (%d, %d)", id, workerSegment.Start, workerSegment.Stop)
+						case <-quitChan:
+							return
+						}
+					}
+				}(i)
+			}
+			// break range up into segments
+			segments := segmentRange(numWorkers, preRun.Start, preRun.Stop, preRun.Params)
+			// send the segments to the work channel
+			for _, segment := range segments {
+				workChan <- segment
+			}
+			close(quitChan)
+			wg.Wait()
+		} else {
+			logrus.Infof("sequential processing prerun range (%d, %d)", preRun.Start, preRun.Stop)
+			for i := preRun.Start; i <= preRun.Stop; i++ {
+				if err := sds.WriteStateDiffAt(i, preRun.Params); err != nil {
+					return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, preRun.Start, preRun.Stop, err)
+				}
 			}
 		}
 	}
 	sds.preruns = nil
+	// At present this code is never called so we have not written the parallel version:
 	for _, rng := range rngs {
-		logrus.Infof("processing prerun range (%d, %d)", rng.Start, rng.Stop)
+		logrus.Infof("processing requested range (%d, %d)", rng.Start, rng.Stop)
 		for i := rng.Start; i <= rng.Stop; i++ {
 			if err := sds.WriteStateDiffAt(i, rng.Params); err != nil {
 				return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, rng.Start, rng.Stop, err)
@@ -153,25 +189,24 @@ func (sds *Service) Loop(wg *sync.WaitGroup) error {
 			for {
 				select {
 				case blockRange := <-sds.queue:
-					logrus.Infof("service worker %d received range (%d, %d) off of work queue, beginning processing", id, blockRange.Start, blockRange.Stop)
+					log := logrus.WithField("range", blockRange).WithField("worker", id)
+					log.Debug("processing range")
 					prom.DecQueuedRanges()
 					for j := blockRange.Start; j <= blockRange.Stop; j++ {
 						if err := sds.WriteStateDiffAt(j, blockRange.Params); err != nil {
-							logrus.Errorf("service worker %d error writing statediff at height %d in range (%d, %d) : %v", id, j, blockRange.Start, blockRange.Stop, err)
+							log.Errorf("error writing statediff at block %d: %v", j, err)
 						}
 						select {
 						case <-sds.quitChan:
-							logrus.Infof("closing service worker %d\n"+
-								"working in range (%d, %d)\n"+
-								"last processed height: %d", id, blockRange.Start, blockRange.Stop, j)
+							log.Infof("closing service worker (last processed block: %d)", j)
 							return
 						default:
-							logrus.Infof("service worker %d finished processing statediff height %d in range (%d, %d)", id, j, blockRange.Start, blockRange.Stop)
+							log.Infof("Finished processing block %d", j)
 						}
 					}
-					logrus.Infof("service worker %d finished processing range (%d, %d)", id, blockRange.Start, blockRange.Stop)
+					log.Debugf("Finished processing range")
 				case <-sds.quitChan:
-					logrus.Infof("closing the statediff service loop worker %d", id)
+					logrus.Debugf("closing the statediff service loop worker %d", id)
 					return
 				}
 			}
@@ -188,15 +223,15 @@ func (sds *Service) Loop(wg *sync.WaitGroup) error {
 
 // StateDiffAt returns a state diff object payload at the specific blockheight
 // This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error) {
+func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
 	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
 	if err != nil {
 		return nil, err
 	}
 	logrus.Infof("sending state diff at block %d", blockNumber)
 
-	// compute leaf keys of watched addresses in the params
-	params.ComputeWatchedAddressesLeafKeys()
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
 
 	if blockNumber == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
@@ -210,15 +245,15 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Paylo
 
 // StateDiffFor returns a state diff object payload for the specific blockhash
 // This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.Payload, error) {
+func (sds *Service) StateDiffFor(blockHash common.Hash, params statediff.Params) (*statediff.Payload, error) {
 	currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
 	if err != nil {
 		return nil, err
 	}
-	logrus.Infof("sending state diff at block %s", blockHash.Hex())
+	logrus.Infof("sending state diff at block %s", blockHash)
 
-	// compute leaf keys of watched addresses in the params
-	params.ComputeWatchedAddressesLeafKeys()
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
 
 	if currentBlock.NumberU64() == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
@@ -231,8 +266,8 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.P
 }
 
 // processStateDiff method builds the state diff payload from the current block, parent state root, and provided params
-func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params sd.Params) (*sd.Payload, error) {
-	stateDiff, err := sds.Builder.BuildStateDiffObject(sd.Args{
+func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params statediff.Params) (*statediff.Payload, error) {
+	stateDiff, err := sds.Builder.BuildStateDiffObject(statediff.Args{
 		BlockHash:    currentBlock.Hash(),
 		BlockNumber:  currentBlock.Number(),
 		OldStateRoot: parentRoot,
@@ -249,8 +284,8 @@ func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot commo
 	return sds.newPayload(stateDiffRlp, currentBlock, params)
 }
 
-func (sds *Service) newPayload(stateObject []byte, block *types.Block, params sd.Params) (*sd.Payload, error) {
-	payload := &sd.Payload{
+func (sds *Service) newPayload(stateObject []byte, block *types.Block, params statediff.Params) (*statediff.Payload, error) {
+	payload := &statediff.Payload{
 		StateObjectRlp: stateObject,
 	}
 	if params.IncludeBlock {
@@ -281,34 +316,6 @@ func (sds *Service) newPayload(stateObject []byte, block *types.Block, params sd
 	return payload, nil
 }
 
-// StateTrieAt returns a state trie object payload at the specified blockheight
-// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error) {
-	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Infof("sending state trie at block %d", blockNumber)
-
-	// compute leaf keys of watched addresses in the params
-	params.ComputeWatchedAddressesLeafKeys()
-
-	return sds.processStateTrie(currentBlock, params)
-}
-
-func (sds *Service) processStateTrie(block *types.Block, params sd.Params) (*sd.Payload, error) {
-	stateNodes, err := sds.Builder.BuildStateTrieObject(block)
-	if err != nil {
-		return nil, err
-	}
-	stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Infof("state trie object at block %d is %d bytes in length", block.Number().Uint64(), len(stateTrieRlp))
-	return sds.newPayload(stateTrieRlp, block, params)
-}
-
 // Start is used to begin the service
 func (sds *Service) Start() error {
 	logrus.Info("starting statediff service")
@@ -325,7 +332,7 @@ func (sds *Service) Stop() error {
 // WriteStateDiffAt writes a state diff at the specific blockheight directly to the database
 // This operation cannot be performed back past the point of db pruning; it requires an archival node
 // for historical data
-func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error {
+func (sds *Service) WriteStateDiffAt(blockNumber uint64, params statediff.Params) error {
 	logrus.Infof("Writing state diff at block %d", blockNumber)
 	t := time.Now()
 	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
@@ -333,8 +340,8 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
 		return err
 	}
 
-	// compute leaf keys of watched addresses in the params
-	params.ComputeWatchedAddressesLeafKeys()
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
 
 	parentRoot := common.Hash{}
 	if blockNumber != 0 {
@@ -350,16 +357,16 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
 // WriteStateDiffFor writes a state diff for the specific blockHash directly to the database
 // This operation cannot be performed back past the point of db pruning; it requires an archival node
 // for historical data
-func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) error {
-	logrus.Infof("Writing state diff for block %s", blockHash.Hex())
+func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params statediff.Params) error {
+	logrus.Infof("Writing state diff for block %s", blockHash)
 	t := time.Now()
 	currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
 	if err != nil {
 		return err
 	}
 
-	// compute leaf keys of watched addresses in the params
-	params.ComputeWatchedAddressesLeafKeys()
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
 
 	parentRoot := common.Hash{}
 	if currentBlock.NumberU64() != 0 {
@@ -373,7 +380,7 @@ func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) e
 }
 
 // Writes a state diff from the current block, parent state root, and provided params
-func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params sd.Params, t time.Time) error {
+func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params statediff.Params, t time.Time) error {
 	var totalDifficulty *big.Int
 	var receipts types.Receipts
 	var err error
@@ -398,28 +405,30 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
 		return err
 	}
 	// defer handling of commit/rollback for any return case
-	output := func(node sdtypes.StateNode) error {
+	output := func(node sdtypes.StateLeafNode) error {
 		return sds.indexer.PushStateNode(tx, node, block.Hash().String())
 	}
-	codeOutput := func(c sdtypes.CodeAndCodeHash) error {
-		return sds.indexer.PushCodeAndCodeHash(tx, c)
+	codeOutput := func(c sdtypes.IPLD) error {
+		return sds.indexer.PushIPLD(tx, c)
 	}
 	prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t))
 	t = time.Now()
-	err = sds.Builder.WriteStateDiffObject(sdtypes.StateRoots{
+	err = sds.Builder.WriteStateDiff(statediff.Args{
 		NewStateRoot: block.Root(),
 		OldStateRoot: parentRoot,
+		BlockNumber:  block.Number(),
+		BlockHash:    block.Hash(),
 	}, params, output, codeOutput)
 	prom.SetTimeMetric(prom.T_STATE_PROCESSING, time.Now().Sub(t))
 	t = time.Now()
-	err = tx.Submit(err)
+	err = tx.Submit()
 	prom.SetLastProcessedHeight(height)
 	prom.SetTimeMetric(prom.T_POSTGRES_TX_COMMIT, time.Now().Sub(t))
 	return err
 }
 
 // WriteStateDiffsInRange adds a RangeRequest to the work queue
-func (sds *Service) WriteStateDiffsInRange(start, stop uint64, params sd.Params) error {
+func (sds *Service) WriteStateDiffsInRange(start, stop uint64, params statediff.Params) error {
 	if stop < start {
 		return fmt.Errorf("invalid block range (%d, %d): stop height must be greater or equal to start height", start, stop)
 	}
@@ -427,7 +436,7 @@ func (sds *Service) WriteStateDiffsInRange(start, stop uint64, params sd.Params)
 	select {
 	case sds.queue <- RangeRequest{Start: start, Stop: stop, Params: params}:
 		prom.IncQueuedRanges()
-		logrus.Infof("added range (%d, %d) to the worker queue", start, stop)
+		logrus.Infof("Added range (%d, %d) to the worker queue", start, stop)
 		return nil
 	case <-blocked.C:
 		return fmt.Errorf("unable to add range (%d, %d) to the worker queue", start, stop)

pkg/types.go

@@ -20,25 +20,17 @@
 package statediff
 
 import (
-	"github.com/ethereum/go-ethereum/core/types"
-	sd "github.com/ethereum/go-ethereum/statediff"
-	sdTypes "github.com/ethereum/go-ethereum/statediff/types"
+	"fmt"
+
+	sd "github.com/cerc-io/plugeth-statediff"
 )
 
-// AccountMap is a mapping of hex encoded path => account wrapper
-type AccountMap map[string]accountWrapper
-
-// accountWrapper is used to temporary associate the unpacked node with its raw values
-type accountWrapper struct {
-	Account   *types.StateAccount
-	NodeType  sdTypes.NodeType
-	Path      []byte
-	NodeValue []byte
-	LeafKey   []byte
-}
-
 // RangeRequest holds range quest work params
 type RangeRequest struct {
 	Start, Stop uint64
 	Params      sd.Params
 }
+
+func (r RangeRequest) String() string {
+	return fmt.Sprintf("[%d,%d]", r.Start, r.Stop)
+}

scripts/.env.example

@@ -0,0 +1,28 @@
+# Used by the script to count rows (count-lines.sh)
+COUNT_LINES_LOG=./count-lines.log
+COUNT_LINES_INPUT_DIR=~/eth-statediff-service/output_dir
+COUNT_LINES_OUTPUT_FILE=./output-stats.txt
+
+# Used by the script to dedup output files (dedup.sh)
+DEDUP_LOG=./dedup.log
+DEDUP_INPUT_DIR=~/eth-statediff-service/output_dir
+DEDUP_OUTPUT_DIR=~/eth-statediff-service/dedup_dir
+DEDUP_SORT_DIR=./.sort
+
+# Used by the script to perform column checks (check-columns.sh)
+CHECK_COLUMNS_LOG=./check-columns.log
+CHECK_COLUMNS_INPUT_DIR=~/eth-statediff-service/output_dir
+CHECK_COLUMNS_INPUT_DEDUP_DIR=~/eth-statediff-service/dedup_dir
+CHECK_COLUMNS_OUTPUT_DIR=./check-columns
+
+# Used by the script to import data (timescaledb-import.sh)
+IMPORT_LOG=./tsdb-import.log
+IMPORT_INPUT_DIR=~/eth-statediff-service/output_dir
+IMPORT_INPUT_DEDUP_DIR=~/eth-statediff-service/dedup_dir
+TIMESCALEDB_WORKERS=8
+
+DATABASE_USER=vdbm
+DATABASE_HOSTNAME=localhost
+DATABASE_PORT=8077
+DATABASE_NAME=vulcanize_testing
+DATABASE_PASSWORD=password

scripts/check-columns.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Requires:
+# CHECK_COLUMNS_LOG
+# CHECK_COLUMNS_INPUT_DIR
+# CHECK_COLUMNS_INPUT_DEDUP_DIR
+# CHECK_COLUMNS_OUTPUT_DIR
+
+# env file arg
+ENV=$1
+echo "Using env file: ${ENV}"
+
+# read env file
+export $(grep -v '^#' ${ENV} | xargs)
+
+# redirect stdout/stderr to a file
+exec >"${CHECK_COLUMNS_LOG}" 2>&1
+
+# create output dir if not exists
+mkdir -p "${CHECK_COLUMNS_OUTPUT_DIR}"
+
+start_timestamp=$(date +%s)
+
+declare -A expected_columns
+expected_columns=(
+  ["public.nodes"]="5"
+  ["public.blocks"]="3"
+  # ["eth.access_list_elements"]="?" # skipping as values include ','
+  ["eth.log_cids"]="12"
+  ["eth.state_accounts"]="7"
+  ["eth.storage_cids"]="9"
+  ["eth.uncle_cids"]="7"
+  ["eth.header_cids"]="16"
+  ["eth.receipt_cids"]="10"
+  ["eth.state_cids"]="8"
+  ["eth.transaction_cids"]="11"
+)
+
+for table_name in "${!expected_columns[@]}";
+do
+  if [ "${table_name}" = "public.blocks" ];
+  then
+    command="$(dirname "$0")/find-bad-rows.sh -i ${CHECK_COLUMNS_INPUT_DEDUP_DIR}/deduped-${table_name}.csv -c ${expected_columns[${table_name}]} -d true -o ${CHECK_COLUMNS_OUTPUT_DIR}/${table_name}.txt"
+  else
+    command="$(dirname "$0")/find-bad-rows.sh -i ${CHECK_COLUMNS_INPUT_DIR}/${table_name}.csv -c ${expected_columns[${table_name}]} -d true -o ${CHECK_COLUMNS_OUTPUT_DIR}/${table_name}.txt"
+  fi
+
+  echo "${table_name}"
+  echo Start: "$(date)"
+  eval "${command}"
+  echo End: "$(date)"
+  echo Total bad rows: $(wc -l ${CHECK_COLUMNS_OUTPUT_DIR}/${table_name}.txt)
+  echo
+done
+
+difference=$(($(date +%s)-start_timestamp))
+echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)
+echo

scripts/count-lines.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Requires:
+# COUNT_LINES_LOG
+# COUNT_LINES_INPUT_DIR
+# COUNT_LINES_OUTPUT_FILE
+
+# env file arg
+ENV=$1
+echo "Using env file: ${ENV}"
+
+# read env file
+export $(grep -v '^#' ${ENV} | xargs)
+
+# redirect stdout/stderr to a file
+exec >"${COUNT_LINES_LOG}" 2>&1
+
+start_timestamp=$(date +%s)
+
+table_names=(
+  "public.nodes"
+  "public.blocks"
+  "eth.access_list_elements"
+  "eth.log_cids"
+  "eth.state_accounts"
+  "eth.storage_cids"
+  "eth.uncle_cids"
+  "eth.header_cids"
+  "eth.receipt_cids"
+  "eth.state_cids"
+  "eth.transaction_cids"
+)
+
+echo "Row counts:" > "${COUNT_LINES_OUTPUT_FILE}"
+
+for table_name in "${table_names[@]}";
+do
+  echo "${table_name}";
+  echo Start: "$(date)"
+  wc -l "${COUNT_LINES_INPUT_DIR}"/"${table_name}.csv" >> "${COUNT_LINES_OUTPUT_FILE}"
+  echo End: "$(date)"
+  echo
+done
+
+difference=$(($(date +%s)-start_timestamp))
+echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)

scripts/dedup.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Requires:
+# DEDUP_LOG
+# DEDUP_INPUT_DIR
+# DEDUP_OUTPUT_DIR
+# DEDUP_SORT_DIR
+
+# env file arg
+ENV=$1
+echo "Using env file: ${ENV}"
+
+# read env file
+export $(grep -v '^#' ${ENV} | xargs)
+
+# redirect stdout/stderr to a file
+exec >"${DEDUP_LOG}" 2>&1
+
+# create output dir if not exists
+mkdir -p "${DEDUP_OUTPUT_DIR}"
+
+start_timestamp=$(date +%s)
+
+echo "public.blocks"
+echo Start: "$(date)"
+sort -T "${DEDUP_SORT_DIR}" -u "${DEDUP_INPUT_DIR}"/public.blocks.csv -o "${DEDUP_OUTPUT_DIR}"/deduped-public.blocks.csv
+echo End: "$(date)"
+echo Total deduped rows: $(wc -l ${DEDUP_OUTPUT_DIR}/deduped-public.blocks.csv)
+echo
+
+difference=$(($(date +%s)-start_timestamp))
+echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)
+
+# NOTE: This script currently only dedups public.blocks output file.
+# If the output contains blocks that were statediffed more than once, output files for other tables will have to be deduped as well.

scripts/find-bad-rows.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# flags
+# -i <input-file>:        Input data file path
+# -c <expected-columns>:  Expected number of columns in each row of the input file
+# -o [output-file]:       Output destination file path (default: STDOUT)
+# -d [include-data]:      Whether to include the data row in output (true | false) (default: false)
+
+# eg: ./scripts/find-bad-rows.sh -i eth.state_cids.csv -c 8 -o res.txt -d true
+# output: 1 9 1500000,xxxxxxxx,0x83952d392f9b0059eea94b10d1a095eefb1943ea91595a16c6698757127d4e1c,,
+#         baglacgzasvqcntdahkxhufdnkm7a22s2eetj6mx6nzkarwxtkvy4x3bubdgq,\x0f,0,f,/blocks/,
+#         DMQJKYBGZRQDVLT2CRWVGPQNNJNCCJU7GL7G4VAI3LZVK4OL5Q2ARTI
+
+while getopts i:c:o:d: OPTION
+do
+  case "${OPTION}" in
+    i) inputFile=${OPTARG};;
+    c) expectedColumns=${OPTARG};;
+    o) outputFile=${OPTARG};;
+    d) data=${OPTARG};;
+  esac
+done
+
+timestamp=$(date +%s)
+
+# if data requested, dump row number, number of columns and the row
+if [ "${data}" = true ] ; then
+  if [ -z "${outputFile}" ]; then
+    awk -F"," "NF!=${expectedColumns} {print NR, NF, \$0}" < ${inputFile}
+  else
+    awk -F"," "NF!=${expectedColumns} {print NR, NF, \$0}" < ${inputFile} > ${outputFile}
+  fi
+# else, dump only row number, number of columns
+else
+  if [ -z "${outputFile}" ]; then
+    awk -F"," "NF!=${expectedColumns} {print NR, NF}" < ${inputFile}
+  else
+    awk -F"," "NF!=${expectedColumns} {print NR, NF}" < ${inputFile} > ${outputFile}
+  fi
+fi
+
+difference=$(($(date +%s)-timestamp))
+echo Time taken: $(date -d@${difference} -u +%H:%M:%S)

scripts/request-range.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -eu
+
+FROM=$1
+TO=$2
+URL=127.0.0.1:8545
+
+DATA='{
+    "jsonrpc": "2.0",
+      "method": "statediff_writeStateDiffsInRange",
+      "params": ['"$FROM"', '"$TO"', {
+          "includeBlock": true,
+          "includeReceipts": true,
+          "includeTD": true,
+          "includeCode": true
+        }
+      ],
+      "id": 1
+}'
+
+exec curl -s $URL -X POST -H 'Content-Type: application/json' --data "$DATA"

scripts/timescaledb-import.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Requires:
+# IMPORT_LOG
+# IMPORT_INPUT_DIR
+# IMPORT_INPUT_DEDUP_DIR
+# TIMESCALEDB_WORKERS
+# DATABASE_USER
+# DATABASE_HOSTNAME
+# DATABASE_PORT
+# DATABASE_NAME
+# DATABASE_PASSWORD
+
+DEFAULT_TIMESCALEDB_WORKERS=8
+
+# env file arg
+ENV=$1
+echo "Using env file: ${ENV}"
+
+# read env file
+export $(grep -v '^#' ${ENV} | xargs)
+
+if [ "$TIMESCALEDB_WORKERS" = "" ]; then
+	TIMESCALEDB_WORKERS=$DEFAULT_TIMESCALEDB_WORKERS
+fi
+
+# redirect stdout/stderr to a file
+exec >"${IMPORT_LOG}" 2>&1
+
+start_timestamp=$(date +%s)
+
+declare -a tables
+# schema-table-copyOptions
+tables=(
+  "public-nodes"
+  "public-blocks"
+  "eth-access_list_elements"
+  "eth-log_cids-FORCE NOT NULL topic0, topic1, topic2, topic3 CSV"
+  "eth-state_accounts"
+  "eth-storage_cids-FORCE NOT NULL storage_leaf_key CSV"
+  "eth-uncle_cids"
+  "eth-header_cids"
+  "eth-receipt_cids-FORCE NOT NULL post_state, contract, contract_hash CSV"
+  "eth-state_cids-FORCE NOT NULL state_leaf_key CSV"
+  "eth-transaction_cids-FORCE NOT NULL dst CSV"
+)
+
+for elem in "${tables[@]}";
+do
+	IFS='-' read -a arr <<< "${elem}"
+
+	if [ "${arr[0]}.${arr[1]}" = "public.blocks" ];
+	then
+		copy_command="timescaledb-parallel-copy --connection \"host=${DATABASE_HOSTNAME} port=${DATABASE_PORT} user=${DATABASE_USER} password=${DATABASE_PASSWORD} sslmode=disable\" --db-name ${DATABASE_NAME} --schema ${arr[0]} --table ${arr[1]} --file ${IMPORT_INPUT_DEDUP_DIR}/deduped-${arr[0]}.${arr[1]}.csv --workers ${TIMESCALEDB_WORKERS} --reporting-period 300s"
+	else
+		copy_command="timescaledb-parallel-copy --connection \"host=${DATABASE_HOSTNAME} port=${DATABASE_PORT} user=${DATABASE_USER} password=${DATABASE_PASSWORD} sslmode=disable\" --db-name ${DATABASE_NAME} --schema ${arr[0]} --table ${arr[1]} --file ${IMPORT_INPUT_DIR}/${arr[0]}.${arr[1]}.csv --workers ${TIMESCALEDB_WORKERS} --reporting-period 300s"
+	fi
+
+	if [ "${arr[2]}" != "" ];
+	then
+		copy_with_options="${copy_command} --copy-options \"${arr[2]}\""
+	else
+		copy_with_options=${copy_command}
+	fi
+
+	echo "${arr[0]}.${arr[1]}"
+	echo Start: "$(date)"
+	eval "${copy_with_options}"
+	echo End: "$(date)"
+	echo
+done
+
+difference=$(($(date +%s)-start_timestamp))
+echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)
+echo

startup_script.sh

@@ -17,4 +17,4 @@ mkdir -p /app/geth-rw && \
 sudo mount -t overlay overlay -o lowerdir=/app/geth-ro,upperdir=/tmp/overlay/upper,workdir=/tmp/overlay/work /app/geth-rw && \
 
 echo "Running the statediff service" && \
-sudo ./eth-statediff-service "$VDB_COMMAND" --config=config.toml
+exec sudo ./eth-statediff-service "$VDB_COMMAND" --config=/config/config.toml

test/ci-chain.json

@@ -0,0 +1,16 @@
+{
+  "chainId": 41337,
+  "homesteadBlock": 0,
+  "eip150Block": 0,
+  "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "eip155Block": 0,
+  "eip158Block": 0,
+  "byzantiumBlock": 0,
+  "constantinopleBlock": 0,
+  "petersburgBlock": 0,
+  "istanbulBlock": 0,
+  "clique": {
+    "period": 5,
+    "epoch": 30000
+  }
+}

test/ci-config.toml

@@ -0,0 +1,36 @@
+[leveldb]
+    mode = "local"
+    url = "http://127.0.0.1:8082/"
+
+[server]
+    ipcPath = ".ipc"
+    httpPath = "0.0.0.0:8545"
+
+[statediff]
+    serviceWorkers = 1
+    workerQueueSize = 1024
+    trieWorkers = 4
+
+[log]
+    level = "debug"
+
+[database]
+    name            = "cerc_testing"
+    hostname        = "localhost"
+    port            = 8077
+    user            = "vdbm"
+    password        = "password"
+    type            = "postgres"
+    driver          = "sqlx"
+
+[cache]
+    database = 1024
+    trie = 1024
+
+[ethereum]
+    chainConfig = "test/ci-chain.json"
+    nodeID = ""
+    clientName = "eth-statediff-service"
+    genesisBlock = "0x37cbb63c7150a7b60f2878433963ed8ba7e5f82fb2683ec7a945c974e1cf4e05"
+    networkID = 1
+    chainID = 41337

test/compose.yml

@@ -0,0 +1,23 @@
+services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.0.5-alpha
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "cerc_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
+  ipld-eth-db:
+    image: timescale/timescaledb:latest-pg14
+    restart: always
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "cerc_testing"
+      POSTGRES_PASSWORD: "password"
+    ports:
+      - 127.0.0.1:8077:5432