Updates for Cancun fork (#7 )

- Bumps Geth to v1.14 - Adds support for pebbledb - Drops leveldb-ethdb-rpc Reviewed-on: #7
Fix checksum build error.
2024-08-05 13:08:26 +00:00 · 2023-10-27 17:12:06 -05:00 · 2023-10-04 07:44:43 +00:00 · 2023-09-29 20:27:08 +00:00 · 2023-07-21 19:05:57 -05:00 · 2023-06-01 08:53:18 -05:00
35 changed files with 1303 additions and 4589 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -0,0 +1,4 @@
+Dockerfile
+.git
+.gitea
+test
--- a/.gitea/workflows/publish.yml
+++ b/.gitea/workflows/publish.yml
@ -0,0 +1,28 @@
+name: Publish Docker image
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  docker-build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - id: vars
+        name: Output SHA and version tag
+        run: |
+          echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
+          echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+      - name: Build and tag image
+        run: |
+          docker build . \
+          -t cerc-io/eth-statediff-service \
+          -t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} \
+          -t git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
+      - name: Push image tags
+        run: |
+          echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+          docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+          docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
--- a/.gitea/workflows/tests.yml
+++ b/.gitea/workflows/tests.yml
@ -0,0 +1,115 @@
+name: Tests
+
+on:
+  pull_request:
+    branches: '*'
+  push:
+    branches:
+      - main
+      - ci-test
+  workflow_call:
+
+env:
+  ETH_TESTING_REF: v0.5.2
+
+jobs:
+  integration-tests:
+    name: Run integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
+          check-latest: true
+      - name: Run DB container
+        run: docker compose -f test/compose.yml up --wait
+
+      - name: Build package
+        run: go build .
+      - name: Install test fixtures
+        uses: actions/checkout@v3
+        with:
+          repository: cerc-io/eth-testing
+          path: ./fixtures
+          ref: ${{ env.ETH_TESTING_REF }}
+        # Run a sanity test against the fixture data
+        # Complete integration tests are TODO
+      - name: Run basic integration test
+        env:
+          ETHDB_PATH: ./fixtures/chains/data/premerge2/geth/chaindata
+          ETH_CHAIN_CONFIG: ./fixtures/chains/data/premerge2/genesis.config.json
+          ETH_GENESIS_BLOCK: 0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614
+          LOG_FILE: ./server-log
+        timeout-minutes: 20
+        run: |
+          ./eth-statediff-service --config ./test/ci-config.toml serve &
+          sleep 10
+
+          ./scripts/request-range.sh 0 10 || (E=$?; cat $LOG_FILE; exit $E)
+
+          until grep "Finished processing block 10" $LOG_FILE
+          do sleep 1; done
+
+          count_results() {
+              query="select count(*) from $1;"
+              docker exec -e PGPASSWORD=password test-ipld-eth-db-1 \
+                  psql -tA cerc_testing -U vdbm -c "$query"
+          }
+          set -x
+          [[ "$(count_results eth.header_cids)" = 11 ]]
+          [[ "$(count_results eth.state_cids)" = 287 ]]
+          [[ "$(count_results eth.storage_cids)" = 31 ]]
+          [[ "$(count_results eth.transaction_cids)" = 144 ]]
+          [[ "$(count_results eth.receipt_cids)" = 144 ]]
+
+  compliance-test:
+    name: Run compliance tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: ./eth-statediff-service
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: ./eth-statediff-service/go.mod
+          check-latest: true
+      - name: Build current version
+        working-directory: ./eth-statediff-service
+        run: go build -o ../service-current .
+
+      - name: Checkout canonical version
+        uses: actions/checkout@v3
+        with:
+          path: ./eth-statediff-service-canonical
+          ref: ${{ env.CANONICAL_VERSION }}
+      - name: Build canonical version
+        working-directory: ./eth-statediff-service-canonical
+        run: go build -o ../service-canonical .
+
+      - name: Install test fixtures
+        uses: actions/checkout@v3
+        with:
+          repository: cerc-io/eth-testing
+          path: ./fixtures
+          ref: ${{ env.ETH_TESTING_REF }}
+      - name: Run DB container
+        working-directory: ./eth-statediff-service
+        run: docker compose -f test/compose.yml up --wait
+      - name: Compare statediff output
+        timeout-minutes: 10
+        env:
+          ETHDB_PATH: ./fixtures/chains/data/premerge2/geth/chaindata
+          ETHDB_ANCIENT: ./fixtures/chains/data/premerge2/geth/chaindata/ancient
+          ETH_CHAIN_CONFIG: ./fixtures/chains/data/premerge2/genesis.config.json
+          ETH_GENESIS_BLOCK: "0x8a3c7cddacbd1ab4ec1b03805fa2a287f3a75e43d87f4f987fcc399f5c042614"
+        run: |
+          until
+              ready_query='select max(version_id) from goose_db_version;'
+              version=$(docker exec -e PGPASSWORD=password test-ipld-eth-db-1 \
+                  psql -tA cerc_testing -U vdbm -c "$ready_query")
+              [[ "$version" -ge 21 ]]
+          do sleep 1; done
+
+          ./eth-statediff-service/scripts/compare-statediffs.sh \
+            ./service-canonical ./service-current
--- a/.github/workflows/on-publish-pr.yml
+++ b/.github/workflows/on-publish-pr.yml
@ -1,73 +0,0 @@
-name: Publish Docker image
-on:
-  release:
-    types: [published]
-  pull_request:
-jobs:
-  pre_job:
-    # continue-on-error: true # Uncomment once integration is finished
-    runs-on: ubuntu-latest
-    # Map a step output to a job output
-    outputs:
-      should_skip: ${{ steps.skip_check.outputs.should_skip }}
-    steps:
-      - id: skip_check
-        uses: fkirc/skip-duplicate-actions@v4
-        with:
-          # All of these options are optional, so you can remove them if you are happy with the defaults
-          concurrent_skipping: "never"
-          skip_after_successful_duplicate: "true"
-          do_not_skip: '["workflow_dispatch", "schedule"]'
-  run-tests:
-    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
-    needs: pre_job
-    uses: ./.github/workflows/tests.yml
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    if: |
-      always() &&
-      (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') &&
-      github.event_name == 'release'
-    needs: run-tests
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: |
-          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag cerc-io/eth-statediff-service git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-      - name: Tag docker image TAG
-        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
-      - name: Docker Push
-        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-#  push_to_registries:
-#    name: Push Docker image to Docker Hub
-#    runs-on: ubuntu-latest
-#    if: |
-#      always() &&
-#      (needs.build.result == 'success') &&
-#      github.event_name == 'release'
-#    needs: build
-#    steps:
-#      - name: Get the version
-#        id: vars
-#        run: |
-#          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-#          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-#      - name: Docker Login to Github Registry
-#        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-#      - name: Docker Pull
-#        run: docker pull docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-#      - name: Docker Login to Docker Registry
-#        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
-#      - name: Tag docker image
-#        run: docker tag docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
-#      - name: Docker Push to Docker Hub
-#        run: docker push cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@ -1,37 +0,0 @@
-name: Tests for Geth that are used in multiple jobs.
-
-on:
-  workflow_call:
-
-env:
-  GOPATH: /tmp/go
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Run docker build
-        run: make docker-build
-
-  statediff-unit-test:
-    name: Run statediff unit tests
-    runs-on: ubuntu-latest
-    env:
-      GO111MODULE: on
-    steps:
-      - name: Create GOPATH
-        run: mkdir -p /tmp/go
-
-      - uses: actions/setup-go@v3
-        with:
-          go-version: ">=1.18.0"
-          check-latest: true
-
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Run unit tests
-        run: |
-          make test
--- a/38
+++ b/38
@ -1,38 +1,36 @@
-FROM golang:1.18-alpine as builder
+FROM golang:1.21-alpine as builder

-RUN apk --update --no-cache add make git g++ linux-headers
+RUN apk add --no-cache git gcc musl-dev binutils-gold
 # DEBUG
 RUN apk add busybox-extras

-# Get and build ipfs-blockchain-watcher
-ADD . /go/src/github.com/cerc-io/eth-statediff-service
-#RUN git clone https://github.com/cerc-io/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
+WORKDIR /eth-statediff-service

-WORKDIR /go/src/github.com/cerc-io/eth-statediff-service
-RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .
+ARG GIT_VDBTO_TOKEN
+
+COPY go.mod go.sum ./
+RUN if [ -n "$GIT_VDBTO_TOKEN" ]; then git config --global url."https://$GIT_VDBTO_TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"; fi && \
+    go mod download && \
+    rm -f ~/.gitconfig
+COPY . .
+
+RUN go build -ldflags '-extldflags "-static"' -o eth-statediff-service .

-# app container
 FROM alpine

-ARG USER="vdm"
-ARG CONFIG_FILE="./environments/config.toml"
+ARG USER="vdbm"
 ARG EXPOSE_PORT=8545
+ARG CONFIG_FILE="./environments/docker.toml"

-RUN adduser -Du 5000 $USER adm
-RUN adduser $USER adm; apk --no-cache add sudo; echo '%adm ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+RUN apk --no-cache add su-exec bash

 WORKDIR /app
-RUN chown $USER /app
-USER $USER

-# chown first so dir is writable
-# note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/$CONFIG_FILE config.toml
-COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/startup_script.sh .
-COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/environments environments
+COPY --from=builder /eth-statediff-service/$CONFIG_FILE config.toml
+COPY --from=builder /eth-statediff-service/startup_script.sh .

 # keep binaries immutable
-COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/eth-statediff-service eth-statediff-service
+COPY --from=builder /eth-statediff-service/eth-statediff-service eth-statediff-service

 EXPOSE $EXPOSE_PORT

--- a/12
+++ b/12
@ -1,12 +0,0 @@
-## Build docker image
-.PHONY: docker-build
-docker-build:
-	docker build -t cerc-io/eth-statediff-service .
-
-.PHONY: test
-test:
-	go test -p 1 ./pkg/... -v
-
-build:
-	go fmt ./...
-	go build
--- a/README.md
+++ b/README.md
@ -1,132 +1,26 @@
 # eth-statediff-service

-[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/eth-statediff-service)](https://goreportcard.com/report/github.com/vulcanize/eth-statediff-service)
+[![Go Report Card](https://goreportcard.com/badge/github.com/cerc-io/eth-statediff-service)](https://goreportcard.com/report/github.com/cerc-io/eth-statediff-service)

->> standalone statediffing service on top of LevelDB
-
-Purpose:
-
-Stand up a statediffing service directly on top of a go-ethereum LevelDB instance.
+A standalone statediffing service which runs directly on top of a `go-ethereum` database instance.
 This service can serve historical state data over the same rpc interface as
 [statediffing geth](https://github.com/cerc-io/go-ethereum) without needing to run a full node.

 ## Setup

-Build the binary:
+Configure access to the private Git server at `git.vdb.to`, then build the executable:

 ```bash
-make build
+go build .
 ```

 ## Configuration

-An example config file:
+See [./environments/example.toml](./environments/example.toml) for an annotated example config file.

-```toml
-[leveldb]
-    # LevelDB access mode <local | remote>
-    mode = "local"  # LVLDB_MODE
-
-    # in local mode
-    # LevelDB paths
-    path    = "/Users/user/Library/Ethereum/geth/chaindata"         # LVLDB_PATH
-    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LVLDB_ANCIENT
-
-    # in remote mode
-    # URL for leveldb-ethdb-rpc endpoint
-    url = "http://127.0.0.1:8082/"  # LVLDB_URL
-
-[server]
-    ipcPath  = ".ipc"           # SERVICE_IPC_PATH
-    httpPath = "127.0.0.1:8545" # SERVICE_HTTP_PATH
-
-[statediff]
-    prerun          = true  # STATEDIFF_PRERUN
-    serviceWorkers  = 1     # STATEDIFF_SERVICE_WORKERS
-    workerQueueSize = 1024  # STATEDIFF_WORKER_QUEUE_SIZE
-    trieWorkers     = 4     # STATEDIFF_TRIE_WORKERS
-
-[prerun]
-    only = false     # PRERUN_ONLY
-    parallel = true  # PRERUN_PARALLEL
-
-    # to perform prerun in a specific range (optional)
-    start = 0   # PRERUN_RANGE_START
-    stop  = 100 # PRERUN_RANGE_STOP
-
-    # to perform prerun over multiple ranges (optional)
-    ranges = [
-        [101, 1000]
-    ]
-
-    # statediffing params for prerun
-    [prerun.params]
-        intermediateStateNodes   = true # PRERUN_INTERMEDIATE_STATE_NODES
-        intermediateStorageNodes = true # PRERUN_INTERMEDIATE_STORAGE_NODES
-        includeBlock             = true # PRERUN_INCLUDE_BLOCK
-        includeReceipts          = true # PRERUN_INCLUDE_RECEIPTS
-        includeTD                = true # PRERUN_INCLUDE_TD
-        includeCode              = true # PRERUN_INCLUDE_CODE
-        watchedAddresses         = []
-
-[log]
-    file  = ""      # LOG_FILE_PATH
-    level = "info"  # LOG_LEVEL
-
-[database]
-    # output type <postgres | file | dump>
-    type = "postgres"
-
-    # with postgres type
-    # db credentials
-    name     = "vulcanize_test" # DATABASE_NAME
-    hostname = "localhost"      # DATABASE_HOSTNAME
-    port     = 5432             # DATABASE_PORT
-    user     = "vulcanize"      # DATABASE_USER
-    password = "..."            # DATABASE_PASSWORD
-    driver   = "sqlx"           # DATABASE_DRIVER_TYPE <sqlx | pgx>
-
-    # with file type
-    # file mode <sql | csv>
-    fileMode = "csv"    # DATABASE_FILE_MODE
-
-    # with SQL file mode
-    filePath = ""   # DATABASE_FILE_PATH
-
-    # with CSV file mode
-    fileCsvDir = "output_dir" # DATABASE_FILE_CSV_DIR
-
-    # with dump type
-    # <stdout | stderr | discard>
-    dumpDestination = ""    # DATABASE_DUMP_DST
-
-[cache]
-    database = 1024 # DB_CACHE_SIZE_MB
-    trie     = 1024 # TRIE_CACHE_SIZE_MB
-
-[prom]
-    # prometheus metrics
-    metrics  = true         # PROM_METRICS
-    http     = true         # PROM_HTTP
-    httpAddr = "localhost"  # PROM_HTTP_ADDR
-    httpPort = "8889"       # PROM_HTTP_PORT
-    dbStats = true          # PROM_DB_STATS
-
-[ethereum]
-    # node info
-    nodeID       = ""                       # ETH_NODE_ID
-    clientName   = "eth-statediff-service"  # ETH_CLIENT_NAME
-    networkID    = 1                        # ETH_NETWORK_ID
-    chainID      = 1                        # ETH_CHAIN_ID
-    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # ETH_GENESIS_BLOCK
-
-    # path to custom chain config file (optional)
-    # keep chainID same as that in chain config file
-    chainConfig  = "./chain.json"           # ETH_CHAIN_CONFIG
-
-[debug]
-    pprof = false                           # DEBUG_PPROF
-```
+> **Note:** previous versions of this service used different variable names. To update, change the following:
+> * `LVLDB_*`, `LEVELDB_*` => `ETHDB_*`
+> * `LOG_FILE_PATH` => `LOG_FILE`

 ### Local Setup

@ -182,14 +76,27 @@ An example config file:
    Example:

    ```bash
-    curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"intermediateStateNodes":true,"intermediateStorageNodes":true,"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT"
+    curl -X POST -H 'Content-Type: application/json' --data '{
+      "jsonrpc": "2.0",
+      "method": "statediff_writeStateDiffsInRange",
+      "params": [0, 1, {
+          "ncludeBlock": true,
+          "includeReceipts": true,
+          "includeTD": true,
+          "includeCode": true
+        }
+      ],
+      "id": 1
+    }' "$HOST":"$PORT"
    ```

 * Prerun:
-    * The process can be configured locally with sets of ranges to process as a "prerun" to processing directed by the server endpoints.
-    * This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining ranges and params in the
-`prerun` section of the config.
-    * Set the range using `prerun.start` and `prerun.stop`. Use `prerun.ranges` if prerun on more than one range is required.
+    * The process can be configured locally with sets of ranges to process as a "prerun" to
+      processing directed by the server endpoints.
+    * This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining
+ranges and params in the `prerun` section of the config.
+    * Set the range using `prerun.start` and `prerun.stop`. Use `prerun.ranges` if prerun on more
+      than one range is required.

 * NOTE: Currently, `params.includeTD` must be set to / passed as `true`.

@ -218,7 +125,8 @@ An example config file:

 ## Import output data in file mode into a database

-* When `eth-statediff-service` is run in file mode (`database.type`) the output is in form of a SQL file or multiple CSV files.
+* When `eth-statediff-service` is run in file mode (`database.type`: `file`) the output is in form of a SQL
+  file or multiple CSV files.

 ### SQL

@ -297,3 +205,12 @@ An example config file:
    ```

 * NOTE: `COPY` command on CSVs inserts empty strings as `NULL` in the DB. Passing `FORCE_NOT_NULL <COLUMN_NAME>` forces it to insert empty strings instead. This is required to maintain compatibility of the imported statediff data with the data generated in `postgres` mode. Reference: https://www.postgresql.org/docs/14/sql-copy.html
+
+### Stats
+
+The binary includes a `stats` command which reports stats for the offline DB.
+
+At this time, the only stat supported is to return the latest/highest block height and hash found in the EthDB. This is
+useful for determining what the upper limit is for a standalone statediffing process using a given EthDB.
+
+`./eth-statediff-service stats --config={path to toml config file}`
--- a/cmd/env.go
+++ b/cmd/env.go
@ -30,10 +30,9 @@ const (

 	DB_CACHE_SIZE_MB   = "DB_CACHE_SIZE_MB"
 	TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
-	LVLDB_MODE         = "LVLDB_MODE"
-	LVLDB_PATH         = "LVLDB_PATH"
-	LVLDB_ANCIENT      = "LVLDB_ANCIENT"
-	LVLDB_URL          = "LVLDB_URL"
+	// ETHDB_ENGINE         = "ETHDB_ENGINE"
+	ETHDB_PATH    = "ETHDB_PATH"
+	ETHDB_ANCIENT = "ETHDB_ANCIENT"

 	STATEDIFF_PRERUN            = "STATEDIFF_PRERUN"
 	STATEDIFF_TRIE_WORKERS      = "STATEDIFF_TRIE_WORKERS"
@ -49,19 +48,17 @@ const (
 	PROM_HTTP_PORT = "PROM_HTTP_PORT"
 	PROM_DB_STATS  = "PROM_DB_STATS"

-	PRERUN_ONLY                       = "PRERUN_ONLY"
-	PRERUN_PARALLEL                   = "PRERUN_PARALLEL"
-	PRERUN_RANGE_START                = "PRERUN_RANGE_START"
-	PRERUN_RANGE_STOP                 = "PRERUN_RANGE_STOP"
-	PRERUN_INTERMEDIATE_STATE_NODES   = "PRERUN_INTERMEDIATE_STATE_NODES"
-	PRERUN_INTERMEDIATE_STORAGE_NODES = "PRERUN_INTERMEDIATE_STORAGE_NODES"
-	PRERUN_INCLUDE_BLOCK              = "PRERUN_INCLUDE_BLOCK"
-	PRERUN_INCLUDE_RECEIPTS           = "PRERUN_INCLUDE_RECEIPTS"
-	PRERUN_INCLUDE_TD                 = "PRERUN_INCLUDE_TD"
-	PRERUN_INCLUDE_CODE               = "PRERUN_INCLUDE_CODE"
+	PRERUN_ONLY             = "PRERUN_ONLY"
+	PRERUN_PARALLEL         = "PRERUN_PARALLEL"
+	PRERUN_RANGE_START      = "PRERUN_RANGE_START"
+	PRERUN_RANGE_STOP       = "PRERUN_RANGE_STOP"
+	PRERUN_INCLUDE_BLOCK    = "PRERUN_INCLUDE_BLOCK"
+	PRERUN_INCLUDE_RECEIPTS = "PRERUN_INCLUDE_RECEIPTS"
+	PRERUN_INCLUDE_TD       = "PRERUN_INCLUDE_TD"
+	PRERUN_INCLUDE_CODE     = "PRERUN_INCLUDE_CODE"

-	LOG_LEVEL     = "LOG_LEVEL"
-	LOG_FILE_PATH = "LOG_FILE_PATH"
+	LOG_LEVEL = "LOG_LEVEL"
+	LOG_FILE  = "LOG_FILE"

 	DATABASE_NAME     = "DATABASE_NAME"
 	DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
@ -121,10 +118,9 @@ func init() {
 	viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
 	viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)

-	viper.BindEnv("leveldb.mode", LVLDB_MODE)
-	viper.BindEnv("leveldb.path", LVLDB_PATH)
-	viper.BindEnv("leveldb.ancient", LVLDB_ANCIENT)
-	viper.BindEnv("leveldb.url", LVLDB_URL)
+	// viper.BindEnv("ethdb.engine", ETHDB_ENGINE)
+	viper.BindEnv("ethdb.path", ETHDB_PATH)
+	viper.BindEnv("ethdb.ancient", ETHDB_ANCIENT)

 	viper.BindEnv("prom.metrics", PROM_METRICS)
 	viper.BindEnv("prom.http", PROM_HTTP)
@ -141,15 +137,13 @@ func init() {
 	viper.BindEnv("prerun.parallel", PRERUN_PARALLEL)
 	viper.BindEnv("prerun.start", PRERUN_RANGE_START)
 	viper.BindEnv("prerun.stop", PRERUN_RANGE_STOP)
-	viper.BindEnv("prerun.params.intermediateStateNodes", PRERUN_INTERMEDIATE_STATE_NODES)
-	viper.BindEnv("prerun.params.intermediateStorageNodes", PRERUN_INTERMEDIATE_STORAGE_NODES)
 	viper.BindEnv("prerun.params.includeBlock", PRERUN_INCLUDE_BLOCK)
 	viper.BindEnv("prerun.params.includeReceipts", PRERUN_INCLUDE_RECEIPTS)
 	viper.BindEnv("prerun.params.includeTD", PRERUN_INCLUDE_TD)
 	viper.BindEnv("prerun.params.includeCode", PRERUN_INCLUDE_CODE)

 	viper.BindEnv("log.level", LOG_LEVEL)
-	viper.BindEnv("log.file", LOG_FILE_PATH)
+	viper.BindEnv("log.file", LOG_FILE)

 	viper.BindEnv("debug.pprof", DEBUG_PPROF)
 }
--- a/cmd/root.go
+++ b/cmd/root.go
@ -23,13 +23,13 @@ import (
 	"strings"
 	"time"

+	"github.com/cerc-io/plugeth-statediff/indexer/database/dump"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/file"
+	"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
+	"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
+	"github.com/cerc-io/plugeth-statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/indexer/shared"
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/dump"
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
-	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
-	"github.com/ethereum/go-ethereum/statediff/indexer/node"
-	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@ -49,7 +49,6 @@ var rootCmd = &cobra.Command{
 }

 func Execute() {
-	log.Info("----- Starting vDB -----")
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)
 	}
@ -116,17 +115,16 @@ func init() {
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(),
 		"log level (trace, debug, info, warn, error, fatal, panic")

-	rootCmd.PersistentFlags().String("leveldb-mode", "local", "LevelDB access mode (local, remote)")
-	rootCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
+	// rootCmd.PersistentFlags().String("ethdb-engine", "local", "Ethdb engine type (leveldb, pebble)")
+	rootCmd.PersistentFlags().String("ethdb-path", "", "path to primary datastore")
 	rootCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
-	rootCmd.PersistentFlags().String("leveldb-url", "", "url to primary leveldb-ethdb-rpc server")

 	rootCmd.PersistentFlags().Bool("prerun", false, "turn on prerun of toml configured ranges")
-	rootCmd.PersistentFlags().Int("service-workers", 0, "number of range requests to process concurrently")
-	rootCmd.PersistentFlags().Int("trie-workers", 0, "number of workers to use for trie traversal and processing")
-	rootCmd.PersistentFlags().Int("worker-queue-size", 0, "size of the range request queue for service workers")
+	rootCmd.PersistentFlags().Int("service-workers", 1, "number of range requests to process concurrently")
+	rootCmd.PersistentFlags().Int("trie-workers", 1, "number of workers to use for trie traversal and processing")
+	rootCmd.PersistentFlags().Int("worker-queue-size", 1024, "size of the range request queue for service workers")

-	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
+	rootCmd.PersistentFlags().String("database-name", "cerc_public", "database name")
 	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
 	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
 	rootCmd.PersistentFlags().String("database-user", "", "database user")
@ -163,8 +161,6 @@ func init() {
 	rootCmd.PersistentFlags().Bool("prerun-only", false, "only process pre-configured ranges; exit afterwards")
 	rootCmd.PersistentFlags().Int("prerun-start", 0, "start height for a prerun range")
 	rootCmd.PersistentFlags().Int("prerun-stop", 0, "stop height for a prerun range")
-	rootCmd.PersistentFlags().Bool("prerun-intermediate-state-nodes", true, "include intermediate state nodes in state diff")
-	rootCmd.PersistentFlags().Bool("prerun-intermediate-storage-nodes", true, "include intermediate storage nodes in state diff")
 	rootCmd.PersistentFlags().Bool("prerun-include-block", true, "include block data in the statediff payload")
 	rootCmd.PersistentFlags().Bool("prerun-include-receipts", true, "include receipts in the statediff payload")
 	rootCmd.PersistentFlags().Bool("prerun-include-td", true, "include td in the statediff payload")
@ -181,10 +177,10 @@ func init() {
 	viper.BindPFlag("statediff.trieWorkers", rootCmd.PersistentFlags().Lookup("trie-workers"))
 	viper.BindPFlag("statediff.workerQueueSize", rootCmd.PersistentFlags().Lookup("worker-queue-size"))

-	viper.BindPFlag("leveldb.mode", rootCmd.PersistentFlags().Lookup("leveldb-mode"))
-	viper.BindPFlag("leveldb.path", rootCmd.PersistentFlags().Lookup("leveldb-path"))
-	viper.BindPFlag("leveldb.ancient", rootCmd.PersistentFlags().Lookup("ancient-path"))
-	viper.BindPFlag("leveldb.url", rootCmd.PersistentFlags().Lookup("leveldb-url"))
+	// viper.BindPFlag("ethdb.engine", rootCmd.PersistentFlags().Lookup("ethdb-engine"))
+	viper.BindPFlag("ethdb.path", rootCmd.PersistentFlags().Lookup("ethdb-path"))
+	viper.BindPFlag("ethdb.ancient", rootCmd.PersistentFlags().Lookup("ancient-path"))
+	viper.BindPFlag("ethdb.url", rootCmd.PersistentFlags().Lookup("ethdb-url"))

 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
@ -224,8 +220,6 @@ func init() {
 	viper.BindPFlag("prerun.parallel", rootCmd.PersistentFlags().Lookup("prerun-parallel"))
 	viper.BindPFlag("prerun.start", rootCmd.PersistentFlags().Lookup("prerun-start"))
 	viper.BindPFlag("prerun.stop", rootCmd.PersistentFlags().Lookup("prerun-stop"))
-	viper.BindPFlag("prerun.params.intermediateStateNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-state-nodes"))
-	viper.BindPFlag("prerun.params.intermediateStorageNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-storage-nodes"))
 	viper.BindPFlag("prerun.params.includeBlock", rootCmd.PersistentFlags().Lookup("prerun-include-block"))
 	viper.BindPFlag("prerun.params.includeReceipts", rootCmd.PersistentFlags().Lookup("prerun-include-receipts"))
 	viper.BindPFlag("prerun.params.includeTD", rootCmd.PersistentFlags().Lookup("prerun-include-td"))
@ -240,7 +234,7 @@ func initConfig() {
 	if cfgFile != "" {
 		viper.SetConfigFile(cfgFile)
 		if err := viper.ReadInConfig(); err == nil {
-			log.Printf("Using config file: %s", viper.ConfigFileUsed())
+			log.Infof("Using config file: %s", viper.ConfigFileUsed())
 		} else {
 			log.Fatal(fmt.Sprintf("Couldn't read config file: %s", err.Error()))
 		}
@ -304,11 +298,11 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 	if err != nil {
 		return nil, err
 	}
-	logWithCommand.Infof("configuring service for database type: %s", dbType)
+	logWithCommand.Debugf("Configuring service for database type: %s", dbType)
 	var indexerConfig interfaces.Config
 	switch dbType {
 	case shared.FILE:
-		logWithCommand.Info("starting in sql file writing mode")
+		logWithCommand.Info("Starting in sql file writing mode")

 		fileModeStr := viper.GetString("database.fileMode")
 		fileMode, err := file.ResolveFileMode(fileModeStr)
@ -318,12 +312,12 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {

 		filePathStr := viper.GetString("database.filePath")
 		if fileMode == file.SQL && filePathStr == "" {
-			logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided")
+			logWithCommand.Fatal("When operating in sql file writing mode a file path must be provided")
 		}

 		fileCsvDirStr := viper.GetString("database.fileCsvDir")
 		if fileMode == file.CSV && fileCsvDirStr == "" {
-			logWithCommand.Fatal("when operating in csv file writing mode a directory path must be provided")
+			logWithCommand.Fatal("When operating in csv file writing mode a directory path must be provided")
 		}

 		indexerConfig = file.Config{
@ -332,7 +326,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 			FilePath:  filePathStr,
 		}
 	case shared.DUMP:
-		logWithCommand.Info("starting in data dump mode")
+		logWithCommand.Info("Starting in data dump mode")
 		dumpDstStr := viper.GetString("database.dumpDestination")
 		dumpDst, err := dump.ResolveDumpType(dumpDstStr)
 		if err != nil {
@ -344,12 +338,12 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 		case dump.STDOUT:
 			indexerConfig = dump.Config{Dump: os.Stderr}
 		case dump.DISCARD:
-			indexerConfig = dump.Config{Dump: dump.NewDiscardWriterCloser()}
+			indexerConfig = dump.Config{Dump: dump.Discard}
 		default:
 			return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst)
 		}
 	case shared.POSTGRES:
-		logWithCommand.Info("starting in postgres mode")
+		logWithCommand.Info("Starting in postgres mode")
 		driverTypeStr := viper.GetString("database.driver")
 		driverType, err := postgres.ResolveDriverType(driverTypeStr)
 		if err != nil {
@ -361,8 +355,6 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 			DatabaseName: viper.GetString("database.name"),
 			Username:     viper.GetString("database.user"),
 			Password:     viper.GetString("database.password"),
-			ID:           nodeInfo.ID,
-			ClientName:   nodeInfo.ClientName,
 			Driver:       driverType,
 		}
 		if viper.IsSet("database.maxIdle") {
--- a/cmd/serve.go
+++ b/cmd/serve.go
@ -28,15 +28,15 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"

-	sd "github.com/cerc-io/eth-statediff-service/pkg"
+	pkg "github.com/cerc-io/eth-statediff-service/pkg"
 	srpc "github.com/cerc-io/eth-statediff-service/pkg/rpc"
 )

 // serveCmd represents the serve command
 var serveCmd = &cobra.Command{
 	Use:   "serve",
-	Short: "Stand up a standalone statediffing RPC service on top of LevelDB",
-	Long: `Usage
+	Short: "Standalone statediffing RPC service on top of an Ethereum database",
+	Long: `Usage:

 ./eth-statediff-service serve --config={path to toml config file}`,
 	Run: func(cmd *cobra.Command, args []string) {
@ -60,10 +60,14 @@ func maxParallelism() int {
 }

 func serve() {
-	logWithCommand.Info("Running eth-statediff-service serve command")
-	logWithCommand.Infof("Parallelism: %d", maxParallelism())
+	logWithCommand.Debug("Running eth-statediff-service serve command")
+	logWithCommand.Debugf("Parallelism: %d", maxParallelism())

-	statediffService, err := createStateDiffService()
+	reader, chainConf, nodeInfo := createReader()
+
+	reportLatestBlock(reader)
+
+	service, err := createStateDiffService(reader, chainConf, nodeInfo)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@ -81,48 +85,45 @@ func serve() {
 	// short circuit if we only want to perform prerun
 	if viper.GetBool("prerun.only") {
 		parallel := viper.GetBool("prerun.parallel")
-		if err := statediffService.Run(nil, parallel); err != nil {
-			logWithCommand.Fatal("unable to perform prerun: %v", err)
+		if err := service.Run(nil, parallel); err != nil {
+			logWithCommand.Fatalf("Unable to perform prerun: %v", err)
 		}
 		return
 	}

 	// start service and servers
-	logWithCommand.Info("Starting statediff service")
-	wg := new(sync.WaitGroup)
-	if err := statediffService.Loop(wg); err != nil {
+	var wg sync.WaitGroup
+	if err := service.Loop(&wg); err != nil {
 		logWithCommand.Fatalf("unable to start statediff service: %v", err)
 	}
-	logWithCommand.Info("Starting RPC servers")
-	if err := startServers(statediffService); err != nil {
+
+	if err := startServers(service); err != nil {
 		logWithCommand.Fatal(err)
 	}
-	logWithCommand.Info("RPC servers successfully spun up; awaiting requests")
+	logWithCommand.Debug("RPC servers successfully spun up; awaiting requests")

 	// clean shutdown
 	shutdown := make(chan os.Signal)
 	signal.Notify(shutdown, os.Interrupt)
 	<-shutdown
 	logWithCommand.Info("Received interrupt signal, shutting down")
-	statediffService.Stop()
+	service.Stop()
 	wg.Wait()
 }

-func startServers(serv sd.StateDiffService) error {
+func startServers(serv *pkg.Service) error {
 	ipcPath := viper.GetString("server.ipcPath")
 	httpPath := viper.GetString("server.httpPath")
 	if ipcPath == "" && httpPath == "" {
-		logWithCommand.Fatal("need an ipc path and/or an http path")
+		logWithCommand.Fatal("Need an IPC path and/or an HTTP path")
 	}
 	if ipcPath != "" {
-		logWithCommand.Info("starting up IPC server")
 		_, _, err := srpc.StartIPCEndpoint(ipcPath, serv.APIs())
 		if err != nil {
 			return err
 		}
 	}
 	if httpPath != "" {
-		logWithCommand.Info("starting up HTTP server")
 		_, err := srpc.StartHTTPEndpoint(httpPath, serv.APIs(), []string{"statediff"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
 		if err != nil {
 			return err
--- a/cmd/stats.go
+++ b/cmd/stats.go
@ -0,0 +1,46 @@
+// Copyright © 2022 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+// statsCmd represents the serve command
+var statsCmd = &cobra.Command{
+	Use:   "stats",
+	Short: "Report stats for cold DB",
+	Long: `Usage
+
+./eth-statediff-service stats --config={path to toml config file}`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *logrus.WithField("SubCommand", subCommand)
+		stats()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(statsCmd)
+}
+
+func stats() {
+	logWithCommand.Info("Running eth-statediff-service stats command")
+
+	reader, _, _ := createReader()
+	reportLatestBlock(reader)
+}
--- a/cmd/util.go
+++ b/cmd/util.go
@ -3,85 +3,33 @@ package cmd
 import (
 	"context"

+	statediff "github.com/cerc-io/plugeth-statediff"
+	"github.com/cerc-io/plugeth-statediff/indexer"
+	"github.com/cerc-io/plugeth-statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/indexer/shared"
+	"github.com/cerc-io/plugeth-statediff/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/statediff"
-	ind "github.com/ethereum/go-ethereum/statediff/indexer"
-	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
-	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/triedb"
+	"github.com/ethereum/go-ethereum/triedb/hashdb"
 	"github.com/spf13/viper"

-	sd "github.com/cerc-io/eth-statediff-service/pkg"
+	pkg "github.com/cerc-io/eth-statediff-service/pkg"
 	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 type blockRange [2]uint64

-func createStateDiffService() (sd.StateDiffService, error) {
-	// load some necessary params
-	logWithCommand.Info("Loading statediff service parameters")
-	mode := viper.GetString("leveldb.mode")
-	path := viper.GetString("leveldb.path")
-	ancientPath := viper.GetString("leveldb.ancient")
-	url := viper.GetString("leveldb.url")
-
-	if mode == "local" {
-		if path == "" || ancientPath == "" {
-			logWithCommand.Fatal("Require a valid eth LevelDB primary datastore path and ancient datastore path")
-		}
-	} else if mode == "remote" {
-		if url == "" {
-			logWithCommand.Fatal("Require a valid RPC url for accessing LevelDB")
-		}
-	} else {
-		logWithCommand.Fatal("Invalid mode provided for LevelDB access")
-	}
-
-	nodeInfo := getEthNodeInfo()
-
-	var chainConf *params.ChainConfig
-	var err error
-	chainConfigPath := viper.GetString("ethereum.chainConfig")
-
-	if chainConfigPath != "" {
-		chainConf, err = statediff.LoadConfig(chainConfigPath)
-	} else {
-		chainConf, err = statediff.ChainConfig(nodeInfo.ChainID)
-	}
-
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-
-	// create LevelDB reader
-	logWithCommand.Info("Creating LevelDB reader")
-	readerConf := sd.LvLDBReaderConfig{
-		TrieConfig: &trie.Config{
-			Cache:     viper.GetInt("cache.trie"),
-			Journal:   "",
-			Preimages: false,
-		},
-		ChainConfig: chainConf,
-		Mode:        mode,
-		Path:        path,
-		AncientPath: ancientPath,
-		Url:         url,
-		DBCacheSize: viper.GetInt("cache.database"),
-	}
-	lvlDBReader, err := sd.NewLvlDBReader(readerConf)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-
+func createStateDiffService(lvlDBReader pkg.Reader, chainConf *params.ChainConfig, nodeInfo node.Info) (*pkg.Service, error) {
 	// create statediff service
-	logWithCommand.Info("Setting up database")
+	logWithCommand.Debug("Setting up database")
 	conf, err := getConfig(nodeInfo)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}

-	logWithCommand.Info("Creating statediff indexer")
-	db, indexer, err := ind.NewStateDiffIndexer(context.Background(), chainConf, nodeInfo, conf)
+	logWithCommand.Debug("Creating statediff indexer")
+	db, indexer, err := indexer.NewStateDiffIndexer(context.Background(), chainConf, nodeInfo, conf, true)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@ -89,27 +37,25 @@ func createStateDiffService() (sd.StateDiffService, error) {
 		prom.RegisterDBCollector(viper.GetString("database.name"), db)
 	}

-	logWithCommand.Info("Creating statediff service")
-	sdConf := sd.Config{
+	logWithCommand.Debug("Creating statediff service")
+	sdConf := pkg.ServiceConfig{
 		ServiceWorkers:  viper.GetUint("statediff.serviceWorkers"),
 		TrieWorkers:     viper.GetUint("statediff.trieWorkers"),
 		WorkerQueueSize: viper.GetUint("statediff.workerQueueSize"),
 		PreRuns:         setupPreRunRanges(),
 	}
-	return sd.NewStateDiffService(lvlDBReader, indexer, sdConf)
+	return pkg.NewStateDiffService(lvlDBReader, indexer, sdConf), nil
 }

-func setupPreRunRanges() []sd.RangeRequest {
+func setupPreRunRanges() []pkg.RangeRequest {
 	if !viper.GetBool("statediff.prerun") {
 		return nil
 	}
 	preRunParams := statediff.Params{
-		IntermediateStateNodes:   viper.GetBool("prerun.params.intermediateStateNodes"),
-		IntermediateStorageNodes: viper.GetBool("prerun.params.intermediateStorageNodes"),
-		IncludeBlock:             viper.GetBool("prerun.params.includeBlock"),
-		IncludeReceipts:          viper.GetBool("prerun.params.includeReceipts"),
-		IncludeTD:                viper.GetBool("prerun.params.includeTD"),
-		IncludeCode:              viper.GetBool("prerun.params.includeCode"),
+		IncludeBlock:    viper.GetBool("prerun.params.includeBlock"),
+		IncludeReceipts: viper.GetBool("prerun.params.includeReceipts"),
+		IncludeTD:       viper.GetBool("prerun.params.includeTD"),
+		IncludeCode:     viper.GetBool("prerun.params.includeCode"),
 	}
 	var addrStrs []string
 	viper.UnmarshalKey("prerun.params.watchedAddresses", &addrStrs)
@ -120,9 +66,9 @@ func setupPreRunRanges() []sd.RangeRequest {
 	preRunParams.WatchedAddresses = addrs
 	var rawRanges []blockRange
 	viper.UnmarshalKey("prerun.ranges", &rawRanges)
-	blockRanges := make([]sd.RangeRequest, len(rawRanges))
+	blockRanges := make([]pkg.RangeRequest, len(rawRanges))
 	for i, rawRange := range rawRanges {
-		blockRanges[i] = sd.RangeRequest{
+		blockRanges[i] = pkg.RangeRequest{
 			Start:  rawRange[0],
 			Stop:   rawRange[1],
 			Params: preRunParams,
@ -131,7 +77,7 @@ func setupPreRunRanges() []sd.RangeRequest {
 	if viper.IsSet("prerun.start") && viper.IsSet("prerun.stop") {
 		hardStart := viper.GetInt("prerun.start")
 		hardStop := viper.GetInt("prerun.stop")
-		blockRanges = append(blockRanges, sd.RangeRequest{
+		blockRanges = append(blockRanges, pkg.RangeRequest{
 			Start:  uint64(hardStart),
 			Stop:   uint64(hardStop),
 			Params: preRunParams,
@ -140,3 +86,62 @@ func setupPreRunRanges() []sd.RangeRequest {

 	return blockRanges
 }
+
+func createReader() (pkg.Reader, *params.ChainConfig, node.Info) {
+	// load some necessary params
+	logWithCommand.Debug("Loading statediff service parameters")
+	path := viper.GetString("ethdb.path")
+	ancientPath := viper.GetString("ethdb.ancient")
+	url := viper.GetString("ethdb.url")
+
+	if path == "" {
+		logWithCommand.Fatal("Require a valid Ethereum chain data path")
+	}
+	if ancientPath == "" {
+		ancientPath = path + "/ancient"
+	}
+
+	nodeInfo := getEthNodeInfo()
+
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+	chainConf, err := utils.LoadConfig(chainConfigPath)
+	if err != nil {
+		logWithCommand.Fatalf("Unable to instantiate chain config: %s", err)
+	}
+
+	logWithCommand.Debug("Creating DB reader")
+	readerConf := pkg.EthDBReaderConfig{
+		TrieConfig: &triedb.Config{
+			Preimages: false,
+			IsVerkle:  false,
+			HashDB: &hashdb.Config{
+				CleanCacheSize: viper.GetInt("cache.trie"),
+			},
+		},
+		ChainConfig: chainConf,
+		Path:        path,
+		AncientPath: ancientPath,
+		Url:         url,
+		DBCacheSize: viper.GetInt("cache.database"),
+	}
+	reader, err := pkg.NewEthDBReader(readerConf)
+	if err != nil {
+		logWithCommand.Fatalf("Unable to instantiate DB reader: %s", err)
+	}
+	return reader, chainConf, nodeInfo
+}
+
+// report latest block info
+func reportLatestBlock(reader pkg.Reader) {
+	header, err := reader.GetLatestHeader()
+	if err != nil {
+		logWithCommand.Fatalf("Unable to determine latest header height and hash: %s", err.Error())
+	}
+	if header.Number == nil {
+		logWithCommand.Fatal("Latest header found in DB has a nil block height")
+	}
+	logWithCommand.
+		WithField("height", header.Number).
+		WithField("hash", header.Hash()).
+		Info("Latest block found in DB")
+}
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,23 +0,0 @@
-version: '3.2'
-
-services:
-  eth-statediff-service:
-    build:
-      context: ./
-      cache_from:
-      - alpine:latest
-      - golang:1.16
-      dockerfile: ./Dockerfile
-      args:
-        USER: "vdbm"
-        CONFIG_FILE: ./environments/example.toml
-        EXPOSE_PORT: 8545
-    environment:
-    - VDB_COMMAND=serve
-    volumes:
-    - eth-statediff-service-data:/root/.ethereum/
-    ports:
-    - "127.0.0.1:8545:8545"
-
-volumes:
-  eth-statediff-service-data:
--- a/environments/config.toml
+++ b/environments/config.toml
@ -1,58 +0,0 @@
-[leveldb]
-    mode = "local"
-    path = "/app/geth-rw/chaindata"
-    ancient = "/app/geth-rw/chaindata/ancient"
-
-[server]
-    ipcPath = ""
-    httpPath = "0.0.0.0:8545"
-
-[statediff]
-    prerun = true
-    serviceWorkers = 1
-    workerQueueSize = 1024
-    trieWorkers = 16
-
-[prerun]
-    only = true
-    ranges = []
-    [prerun.params]
-        intermediateStateNodes = true
-        intermediateStorageNodes = true
-        includeBlock = true
-        includeReceipts = true
-        includeTD = true
-        includeCode = true
-        watchedAddresses = []
-
-[log]
-    file = ""
-    level = "info"
-
-[database]
-    type = "postgres"
-    name     = ""
-    hostname = ""
-    port     = 5432
-    user     = ""
-    password = ""
-    driver = "sqlx"
-
-[cache]
-    database = 1024
-    trie = 4096
-
-[prom]
-    metrics = true
-    http = true
-    httpAddr = "0.0.0.0"
-    httpPort = 9100
-    dbStats = false
-
-[ethereum]
-    nodeID = ""
-    clientName = "eth-statediff-service"
-    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
-    networkID = 1
-    chainID = 1
-    chainConfig = ""
--- a/environments/docker.toml
+++ b/environments/docker.toml
@ -0,0 +1,80 @@
+[ethdb]
+    path    = "/app/geth-rw/chaindata"         # ETHDB_PATH
+    ancient = "/app/geth-rw/chaindata/ancient" # ETHDB_ANCIENT
+
+[server]
+    ipcPath  = ".ipc"           # SERVICE_IPC_PATH
+    httpPath = "0.0.0.0:8545" # SERVICE_HTTP_PATH
+
+[statediff]
+    prerun          = true  # STATEDIFF_PRERUN
+    serviceWorkers  = 1     # STATEDIFF_SERVICE_WORKERS
+    workerQueueSize = 1024  # STATEDIFF_WORKER_QUEUE_SIZE
+    trieWorkers     = 16     # STATEDIFF_TRIE_WORKERS
+
+[prerun]
+    only = true     # PRERUN_ONLY
+    parallel = true  # PRERUN_PARALLEL
+    ranges = []
+
+    # statediffing params for prerun
+    [prerun.params]
+        includeBlock             = true # PRERUN_INCLUDE_BLOCK
+        includeReceipts          = true # PRERUN_INCLUDE_RECEIPTS
+        includeTD                = true # PRERUN_INCLUDE_TD
+        includeCode              = true # PRERUN_INCLUDE_CODE
+        watchedAddresses         = []
+
+[log]
+    # Leave empty to output to stdout
+    file  = ""      # LOG_FILE
+    level = "debug" # LOG_LEVEL
+
+[database]
+    # output type <postgres | file | dump>
+    type = "postgres"
+    name     = ""      # DATABASE_NAME
+    hostname = ""      # DATABASE_HOSTNAME
+    port     = 5432    # DATABASE_PORT
+    user     = ""      # DATABASE_USER
+    password = ""      # DATABASE_PASSWORD
+    driver   = ""      # DATABASE_DRIVER_TYPE
+
+    # with file type
+    # file mode <sql | csv>
+    fileMode = "csv"   # DATABASE_FILE_MODE
+
+    # with SQL file mode
+    filePath = ""   # DATABASE_FILE_PATH
+
+    # with CSV file mode
+    fileCsvDir = "" # DATABASE_FILE_CSV_DIR
+
+    # with dump type
+    # <stdout | stderr | discard>
+    dumpDestination = ""    # DATABASE_DUMP_DST
+
+[cache]
+    # settings for geth internal caches
+    database = 1024 # DB_CACHE_SIZE_MB
+    trie     = 4096 # TRIE_CACHE_SIZE_MB
+
+[prom]
+    # prometheus metrics
+    metrics  = true         # PROM_METRICS
+    http     = true         # PROM_HTTP
+    httpAddr = "localhost"  # PROM_HTTP_ADDR
+    httpPort = "8889"       # PROM_HTTP_PORT
+    dbStats = true          # PROM_DB_STATS
+
+[ethereum]
+    # Identifiers for ethereum node
+    nodeID       = ""                       # ETH_NODE_ID
+    clientName   = "eth-statediff-service"  # ETH_CLIENT_NAME
+    networkID    = 1                        # ETH_NETWORK_ID
+    chainID      = 1                        # ETH_CHAIN_ID
+    genesisBlock = ""                       # ETH_GENESIS_BLOCK
+    chainConfig  = ""                       # ETH_CHAIN_CONFIG
+
+[debug]
+    pprof = false                           # DEBUG_PPROF
--- a/environments/example.toml
+++ b/environments/example.toml
@ -1,63 +1,99 @@
-[leveldb]
-    mode = "local"
-    path = "/Users/user/Library/Ethereum/geth/chaindata"
-    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
-    url = "http://127.0.0.1:8082/"
+[ethdb]
+    # #  access mode <local | remote>
+    # mode = "local"  # ETHDB_MODE
+
+    # Ethdb paths (local mode)
+    path    = "/Users/user/Library/Ethereum/geth/chaindata"         # ETHDB_PATH
+    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # ETHDB_ANCIENT

 [server]
-    ipcPath = ".ipc"
-    httpPath = "127.0.0.1:8545"
+    ipcPath  = ".ipc"           # SERVICE_IPC_PATH
+    httpPath = "127.0.0.1:8545" # SERVICE_HTTP_PATH

 [statediff]
-    prerun = true
-    serviceWorkers = 1
-    workerQueueSize = 1024
-    trieWorkers = 4
+    prerun          = true  # STATEDIFF_PRERUN
+    serviceWorkers  = 1     # STATEDIFF_SERVICE_WORKERS
+    workerQueueSize = 1024  # STATEDIFF_WORKER_QUEUE_SIZE
+    trieWorkers     = 4     # STATEDIFF_TRIE_WORKERS

 [prerun]
-    only = false
+    only = false     # PRERUN_ONLY
+    parallel = true  # PRERUN_PARALLEL
+
+    # to perform prerun in a specific range (optional)
+    start = 0   # PRERUN_RANGE_START
+    stop  = 100 # PRERUN_RANGE_STOP
+
+    # to perform prerun over multiple ranges (optional)
    ranges = [
-        [0, 1000]
+        [101, 1000]
    ]
+
+    # statediffing params for prerun
    [prerun.params]
-        intermediateStateNodes = true
-        intermediateStorageNodes = true
-        includeBlock = true
-        includeReceipts = true
-        includeTD = true
-        includeCode = true
-        watchedAddresses = []
+        includeBlock             = true # PRERUN_INCLUDE_BLOCK
+        includeReceipts          = true # PRERUN_INCLUDE_RECEIPTS
+        includeTD                = true # PRERUN_INCLUDE_TD
+        includeCode              = true # PRERUN_INCLUDE_CODE
+        watchedAddresses         = []

 [log]
-    file = ""
-    level = "info"
+    # Leave empty to output to stdout
+    file  = ""      # LOG_FILE
+    level = "info"  # LOG_LEVEL

 [database]
-    name     = "vulcanize_test"
-    hostname = "localhost"
-    port     = 5432
-    user     = "vulcanize"
-    password = "..."
+    # output type <postgres | file | dump>
    type = "postgres"
-    driver = "sqlx"
-    dumpDestination = ""
-    filePath = ""
+
+    # with postgres type
+    # db credentials
+    name     = "vulcanize_test" # DATABASE_NAME
+    hostname = "localhost"      # DATABASE_HOSTNAME
+    port     = 5432             # DATABASE_PORT
+    user     = "vulcanize"      # DATABASE_USER
+    password = "..."            # DATABASE_PASSWORD
+    # SQL backend to use: <sqlx | pgx>
+    driver   = "sqlx"           # DATABASE_DRIVER_TYPE
+
+    # with file type
+    # file mode <sql | csv>
+    fileMode = "csv"    # DATABASE_FILE_MODE
+
+    # with SQL file mode
+    filePath = ""   # DATABASE_FILE_PATH
+
+    # with CSV file mode
+    fileCsvDir = "output_dir" # DATABASE_FILE_CSV_DIR
+
+    # with dump type
+    # <stdout | stderr | discard>
+    dumpDestination = ""    # DATABASE_DUMP_DST

 [cache]
-    database = 1024
-    trie = 1024
+    # settings for geth internal caches
+    database = 1024 # DB_CACHE_SIZE_MB
+    trie     = 1024 # TRIE_CACHE_SIZE_MB

 [prom]
-    dbStats = false
-    metrics = true
-    http = true
-    httpAddr = "localhost"
-    httpPort = "8889"
+    # prometheus metrics
+    metrics  = true         # PROM_METRICS
+    http     = true         # PROM_HTTP
+    httpAddr = "localhost"  # PROM_HTTP_ADDR
+    httpPort = "8889"       # PROM_HTTP_PORT
+    dbStats = true          # PROM_DB_STATS

 [ethereum]
-    chainConfig = ""
-    nodeID = ""
-    clientName = "eth-statediff-service"
-    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
-    networkID = 1
-    chainID = 1
+    # Identifiers for ethereum node
+    nodeID       = ""                       # ETH_NODE_ID
+    clientName   = "eth-statediff-service"  # ETH_CLIENT_NAME
+    networkID    = 1                        # ETH_NETWORK_ID
+    chainID      = 1                        # ETH_CHAIN_ID
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # ETH_GENESIS_BLOCK
+
+    # Path to custom chain config file (optional)
+    # chainID should match that in this config file
+    chainConfig  = "chain.json"           # ETH_CHAIN_CONFIG
+
+[debug]
+    pprof = false                           # DEBUG_PPROF
--- a/go.mod
+++ b/go.mod
@ -1,63 +1,69 @@
 module github.com/cerc-io/eth-statediff-service

-go 1.18
+go 1.21

 require (
-	github.com/cerc-io/go-eth-state-node-iterator v1.1.7
-	github.com/ethereum/go-ethereum v1.10.23
-	github.com/jmoiron/sqlx v1.2.0 // indirect
-	github.com/prometheus/client_golang v1.4.0
-	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v1.3.0
+	github.com/cerc-io/plugeth-statediff v0.3.0
+	github.com/ethereum/go-ethereum v1.14.5
+	github.com/prometheus/client_golang v1.16.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.5.0
 	github.com/spf13/viper v1.10.1
-	github.com/vulcanize/leveldb-ethdb-rpc v0.1.7
 )

 require (
-	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
+	github.com/DataDog/zstd v1.5.5 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/VictoriaMetrics/fastcache v1.12.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/bits-and-blooms/bitset v1.10.0 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
+	github.com/cerc-io/eth-iterator-utils v0.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cockroachdb/errors v1.11.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v1.1.0 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
+	github.com/consensys/bavard v0.1.13 // indirect
+	github.com/consensys/gnark-crypto v0.12.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect
+	github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/deckarep/golang-set v1.8.0 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/deckarep/golang-set/v2 v2.6.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
-	github.com/edsrzf/mmap-go v1.0.0 // indirect
-	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
+	github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 // indirect
+	github.com/ferranbt/fastssz v0.1.2 // indirect
+	github.com/fjl/memsize v0.0.2 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
 	github.com/georgysavva/scany v0.2.9 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/getsentry/sentry-go v0.22.0 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/graph-gophers/graphql-go v1.3.0 // indirect
-	github.com/hashicorp/go-bexpr v0.1.10 // indirect
-	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/hashicorp/go-bexpr v0.1.12 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
-	github.com/holiman/uint256 v1.2.0 // indirect
-	github.com/huin/goupnp v1.0.3 // indirect
+	github.com/holiman/uint256 v1.2.4 // indirect
+	github.com/huin/goupnp v1.3.0 // indirect
+	github.com/inconshreveable/log15 v2.16.0+incompatible // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/influxdata/influxdb v1.8.3 // indirect
 	github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
+	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c // indirect
 	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
-	github.com/ipfs/bbloom v0.0.4 // indirect
-	github.com/ipfs/go-block-format v0.0.3 // indirect
-	github.com/ipfs/go-cid v0.2.0 // indirect
-	github.com/ipfs/go-datastore v0.5.1 // indirect
-	github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
-	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
-	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
-	github.com/ipfs/go-ipld-format v0.4.0 // indirect
-	github.com/ipfs/go-log v1.0.5 // indirect
-	github.com/ipfs/go-log/v2 v2.1.3 // indirect
-	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-cid v0.4.1 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgconn v1.10.0 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
@ -68,71 +74,75 @@ require (
 	github.com/jackc/pgx/v4 v4.13.0 // indirect
 	github.com/jackc/puddle v1.1.3 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
-	github.com/jbenet/goprocess v0.1.4 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
-	github.com/lib/pq v1.10.6 // indirect
+	github.com/jmoiron/sqlx v1.2.0 // indirect
+	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
-	github.com/mitchellh/mapstructure v1.4.3 // indirect
-	github.com/mitchellh/pointerstructure v1.2.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/pointerstructure v1.2.1 // indirect
+	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
-	github.com/multiformats/go-base32 v0.0.3 // indirect
-	github.com/multiformats/go-base36 v0.1.0 // indirect
-	github.com/multiformats/go-multibase v0.0.3 // indirect
-	github.com/multiformats/go-multihash v0.1.0 // indirect
-	github.com/multiformats/go-varint v0.0.6 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/openrelayxyz/plugeth-utils v1.5.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
-	github.com/pganalyze/pg_query_go/v2 v2.1.0 // indirect
+	github.com/pganalyze/pg_query_go/v4 v4.2.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.9.1 // indirect
-	github.com/prometheus/procfs v0.0.8 // indirect
-	github.com/prometheus/tsdb v0.10.0 // indirect
-	github.com/rjeczalik/notify v0.9.1 // indirect
-	github.com/rs/cors v1.7.0 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.0 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/rs/cors v1.9.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/stretchr/testify v1.7.2 // indirect
+	github.com/status-im/keycard-go v0.2.0 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/supranational/blst v0.3.11 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
-	github.com/thoas/go-funk v0.9.2 // indirect
-	github.com/tklauser/go-sysconf v0.3.6 // indirect
-	github.com/tklauser/numcpus v0.2.2 // indirect
-	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
-	github.com/urfave/cli/v2 v2.10.2 // indirect
+	github.com/thoas/go-funk v0.9.3 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
+	github.com/urfave/cli/v2 v2.25.7 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
-	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.17.0 // indirect
-	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
-	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/ini.v1 v1.66.2 // indirect
-	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	lukechampine.com/blake3 v1.1.6 // indirect
+	lukechampine.com/blake3 v1.2.1 // indirect
+	rsc.io/tmplfunc v0.0.3 // indirect
 )
-
-replace github.com/ethereum/go-ethereum v1.10.23 => github.com/cerc-io/go-ethereum v1.10.23-statediff-4.2.0-alpha
--- a/go.sum
+++ b/go.sum
--- a/pkg/api.go
+++ b/pkg/api.go
@ -18,7 +18,7 @@ package statediff
 import (
 	"context"

-	sd "github.com/ethereum/go-ethereum/statediff"
+	sd "github.com/cerc-io/plugeth-statediff"
 )

 // APIName is the namespace used for the state diffing service API
@ -28,13 +28,13 @@ const APIName = "statediff"
 const APIVersion = "0.0.1"

 // PublicStateDiffAPI provides an RPC interface
-// that can be used to fetch historical diffs from LevelDB directly
+// that can be used to fetch historical diffs from database directly
 type PublicStateDiffAPI struct {
-	sds StateDiffService
+	sds *Service
 }

 // NewPublicStateDiffAPI creates an rpc interface for the underlying statediff service
-func NewPublicStateDiffAPI(sds StateDiffService) *PublicStateDiffAPI {
+func NewPublicStateDiffAPI(sds *Service) *PublicStateDiffAPI {
 	return &PublicStateDiffAPI{
 		sds: sds,
 	}
@ -45,11 +45,6 @@ func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint
 	return api.sds.StateDiffAt(blockNumber, params)
 }

-// StateTrieAt returns a state trie payload at the specific blockheight
-func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params sd.Params) (*sd.Payload, error) {
-	return api.sds.StateTrieAt(blockNumber, params)
-}
-
 // WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight
 func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params sd.Params) error {
 	return api.sds.WriteStateDiffAt(blockNumber, params)
--- a/pkg/builder.go
+++ b/pkg/builder.go
@ -1,155 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Contains a batch of utility type declarations used by the tests. As the node
-// operates on unique types, a lot of them are needed to check various features.
-
-package statediff
-
-import (
-	"fmt"
-	"math/bits"
-	"sync"
-
-	"github.com/ethereum/go-ethereum/core/state"
-	sd "github.com/ethereum/go-ethereum/statediff"
-	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
-	"github.com/sirupsen/logrus"
-
-	iter "github.com/cerc-io/go-eth-state-node-iterator"
-)
-
-type builder struct {
-	sd.StateDiffBuilder
-	numWorkers uint
-}
-
-// NewBuilder is used to create a statediff builder
-func NewBuilder(stateCache state.Database, workers uint) (sd.Builder, error) {
-	if workers == 0 {
-		workers = 1
-	}
-	if bits.OnesCount(workers) != 1 {
-		return nil, fmt.Errorf("workers must be a power of 2")
-	}
-	return &builder{
-		StateDiffBuilder: sd.StateDiffBuilder{
-			StateCache: stateCache,
-		},
-		numWorkers: workers,
-	}, nil
-}
-
-// BuildStateDiffObject builds a statediff object from two blocks and the provided parameters
-func (sdb *builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sdtypes.StateObject, error) {
-	var stateNodes []sdtypes.StateNode
-	var codeAndCodeHashes []sdtypes.CodeAndCodeHash
-	err := sdb.WriteStateDiffObject(
-		sdtypes.StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot},
-		params, sd.StateNodeAppender(&stateNodes), sd.CodeMappingAppender(&codeAndCodeHashes))
-	if err != nil {
-		return sdtypes.StateObject{}, err
-	}
-	return sdtypes.StateObject{
-		BlockHash:         args.BlockHash,
-		BlockNumber:       args.BlockNumber,
-		Nodes:             stateNodes,
-		CodeAndCodeHashes: codeAndCodeHashes,
-	}, nil
-}
-
-// WriteStateDiffObject writes a statediff object to output callback
-func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
-	// Load tries for old and new states
-	oldTrie, err := sdb.StateCache.OpenTrie(args.OldStateRoot)
-	if err != nil {
-		return fmt.Errorf("error creating trie for oldStateRoot: %v", err)
-	}
-	newTrie, err := sdb.StateCache.OpenTrie(args.NewStateRoot)
-	if err != nil {
-		return fmt.Errorf("error creating trie for newStateRoot: %v", err)
-	}
-
-	// Split old and new tries into corresponding subtrie iterators
-	oldIters1 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
-	oldIters2 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
-	newIters1 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
-	newIters2 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
-
-	// Create iterators ahead of time to avoid race condition in state.Trie access
-	// We do two state iterations per subtrie: one for new/updated nodes,
-	// one for deleted/updated nodes; prepare 2 iterator instances for each task
-	var iterPairs [][]sd.IterPair
-	for i := uint(0); i < sdb.numWorkers; i++ {
-		iterPairs = append(iterPairs, []sd.IterPair{
-			{Older: oldIters1[i], Newer: newIters1[i]},
-			{Older: oldIters2[i], Newer: newIters2[i]},
-		})
-	}
-
-	// Dispatch workers to process trie data; sync and collect results here via channels
-	nodeChan := make(chan sdtypes.StateNode)
-	codeChan := make(chan sdtypes.CodeAndCodeHash)
-
-	go func() {
-		nodeSender := func(node sdtypes.StateNode) error { nodeChan <- node; return nil }
-		codeSender := func(code sdtypes.CodeAndCodeHash) error { codeChan <- code; return nil }
-		var wg sync.WaitGroup
-
-		for w := uint(0); w < sdb.numWorkers; w++ {
-			wg.Add(1)
-			go func(worker uint) {
-				defer wg.Done()
-				var err error
-				if !params.IntermediateStateNodes {
-					err = sdb.BuildStateDiffWithoutIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender)
-				} else {
-					err = sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender)
-				}
-
-				if err != nil {
-					logrus.Errorf("buildStateDiff error for worker %d, params %+v", worker, params)
-				}
-			}(w)
-		}
-		wg.Wait()
-		close(nodeChan)
-		close(codeChan)
-	}()
-
-	for nodeChan != nil || codeChan != nil {
-		select {
-		case node, more := <-nodeChan:
-			if more {
-				if err := output(node); err != nil {
-					return err
-				}
-			} else {
-				nodeChan = nil
-			}
-		case codeAndCodeHash, more := <-codeChan:
-			if more {
-				if err := codeOutput(codeAndCodeHash); err != nil {
-					return err
-				}
-			} else {
-				codeChan = nil
-			}
-		}
-	}
-
-	return nil
-}
--- a/pkg/builder_test.go
+++ b/pkg/builder_test.go
--- a/pkg/config.go
+++ b/pkg/config.go
@ -1,7 +1,7 @@
 package statediff

-// Config holds config params for the statediffing service
-type Config struct {
+// ServiceConfig holds config params for the statediffing service
+type ServiceConfig struct {
 	ServiceWorkers  uint
 	TrieWorkers     uint
 	WorkerQueueSize uint
--- a/pkg/helpers.go
+++ b/pkg/helpers.go
@ -1,72 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Contains a batch of utility type declarations used by the tests. As the node
-// operates on unique types, a lot of them are needed to check various features.
-
-package statediff
-
-import (
-	"sort"
-	"strings"
-)
-
-func sortKeys(data AccountMap) []string {
-	keys := make([]string, 0, len(data))
-	for key := range data {
-		keys = append(keys, key)
-	}
-	sort.Strings(keys)
-
-	return keys
-}
-
-// findIntersection finds the set of strings from both arrays that are equivalent
-// a and b must first be sorted
-// this is used to find which keys have been both "deleted" and "created" i.e. they were updated
-func findIntersection(a, b []string) []string {
-	lenA := len(a)
-	lenB := len(b)
-	iOfA, iOfB := 0, 0
-	updates := make([]string, 0)
-	if iOfA >= lenA || iOfB >= lenB {
-		return updates
-	}
-	for {
-		switch strings.Compare(a[iOfA], b[iOfB]) {
-		// -1 when a[iOfA] < b[iOfB]
-		case -1:
-			iOfA++
-			if iOfA >= lenA {
-				return updates
-			}
-			// 0 when a[iOfA] == b[iOfB]
-		case 0:
-			updates = append(updates, a[iOfA])
-			iOfA++
-			iOfB++
-			if iOfA >= lenA || iOfB >= lenB {
-				return updates
-			}
-			// 1 when a[iOfA] > b[iOfB]
-		case 1:
-			iOfB++
-			if iOfB >= lenB {
-				return updates
-			}
-		}
-	}
-}
--- a/pkg/prom/db_stats_collector.go
+++ b/pkg/prom/db_stats_collector.go
@ -19,7 +19,7 @@ package prom
 import (
 	"github.com/prometheus/client_golang/prometheus"

-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	dbmetrics "github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
 )

 const (
@ -29,7 +29,7 @@ const (

 // DBStatsGetter is an interface that gets sql.DBStats.
 type DBStatsGetter interface {
-	Stats() sql.Stats
+	Stats() dbmetrics.DbStats
 }

 // DBStatsCollector implements the prometheus.Collector interface.
--- a/pkg/reader.go
+++ b/pkg/reader.go
@ -16,6 +16,7 @@
 package statediff

 import (
+	"errors"
 	"fmt"
 	"math/big"

@ -25,8 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/trie"
-	"github.com/vulcanize/leveldb-ethdb-rpc/pkg/client"
+	"github.com/ethereum/go-ethereum/triedb"
 )

 // Reader interface required by the statediffing service
@ -36,41 +36,39 @@ type Reader interface {
 	GetReceiptsByHash(hash common.Hash) (types.Receipts, error)
 	GetTdByHash(hash common.Hash) (*big.Int, error)
 	StateDB() state.Database
+	GetLatestHeader() (*types.Header, error)
 }

-// LvlDBReader exposes the necessary Reader methods on lvldb
-type LvlDBReader struct {
+// EthDBReader exposes the necessary Reader methods on an ethdb
+type EthDBReader struct {
 	ethDB       ethdb.Database
 	stateDB     state.Database
 	chainConfig *params.ChainConfig
 }

-// LvLDBReaderConfig struct for initializing a LvlDBReader
-type LvLDBReaderConfig struct {
-	TrieConfig             *trie.Config
+type EthDBReaderConfig struct {
+	TrieConfig             *triedb.Config
 	ChainConfig            *params.ChainConfig
-	Mode                   string
 	Path, AncientPath, Url string
 	DBCacheSize            int
 }

-// NewLvlDBReader creates a new Read using LevelDB
-func NewLvlDBReader(conf LvLDBReaderConfig) (*LvlDBReader, error) {
-	var edb ethdb.Database
-	var err error
-
-	if conf.Mode == "local" {
-		edb, err = rawdb.NewLevelDBDatabaseWithFreezer(conf.Path, conf.DBCacheSize, 256, conf.AncientPath, "eth-statediff-service", true)
+// NewEthDBReader creates a new Reader using LevelDB
+func NewEthDBReader(conf EthDBReaderConfig) (*EthDBReader, error) {
+	opts := rawdb.OpenOptions{
+		Directory:         conf.Path,
+		AncientsDirectory: conf.AncientPath,
+		Namespace:         "eth-statediff-service",
+		Cache:             conf.DBCacheSize,
+		Handles:           256,
+		ReadOnly:          true,
 	}
-
-	if conf.Mode == "remote" {
-		edb, err = client.NewDatabaseClient(conf.Url)
-	}
-
+	edb, err := rawdb.Open(opts)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to open DB: %w", err)
 	}
-	return &LvlDBReader{
+
+	return &EthDBReader{
 		ethDB:       edb,
 		stateDB:     state.NewDatabaseWithConfig(edb, conf.TrieConfig),
 		chainConfig: conf.ChainConfig,
@ -78,54 +76,67 @@ func NewLvlDBReader(conf LvLDBReaderConfig) (*LvlDBReader, error) {
 }

 // GetBlockByHash gets block by hash
-func (ldr *LvlDBReader) GetBlockByHash(hash common.Hash) (*types.Block, error) {
+func (ldr *EthDBReader) GetBlockByHash(hash common.Hash) (*types.Block, error) {
 	height := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
 	if height == nil {
-		return nil, fmt.Errorf("unable to read header height for header hash %s", hash.String())
+		return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
 	}
 	block := rawdb.ReadBlock(ldr.ethDB, hash, *height)
 	if block == nil {
-		return nil, fmt.Errorf("unable to read block at height %d hash %s", *height, hash.String())
+		return nil, fmt.Errorf("unable to read block at height %d hash %s", *height, hash)
 	}
 	return block, nil
 }

-func (ldr *LvlDBReader) GetBlockByNumber(number uint64) (*types.Block, error) {
+func (ldr *EthDBReader) GetBlockByNumber(number uint64) (*types.Block, error) {
 	hash := rawdb.ReadCanonicalHash(ldr.ethDB, number)
 	block := rawdb.ReadBlock(ldr.ethDB, hash, number)
 	if block == nil {
-		return nil, fmt.Errorf("unable to read block at height %d hash %s", number, hash.String())
+		return nil, fmt.Errorf("unable to read block at height %d hash %s", number, hash)
 	}
 	return block, nil
 }

 // GetReceiptsByHash gets receipt by hash
-func (ldr *LvlDBReader) GetReceiptsByHash(hash common.Hash) (types.Receipts, error) {
+func (ldr *EthDBReader) GetReceiptsByHash(hash common.Hash) (types.Receipts, error) {
 	number := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
 	if number == nil {
-		return nil, fmt.Errorf("unable to read header height for header hash %s", hash.String())
+		return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
 	}
-	receipts := rawdb.ReadReceipts(ldr.ethDB, hash, *number, ldr.chainConfig)
+	header := rawdb.ReadHeader(ldr.ethDB, hash, *number)
+	if header == nil {
+		return nil, fmt.Errorf("unable to read header for header hash %s", hash)
+	}
+	receipts := rawdb.ReadReceipts(ldr.ethDB, hash, *number, header.Time, ldr.chainConfig)
 	if receipts == nil {
-		return nil, fmt.Errorf("unable to read receipts at height %d hash %s", number, hash.String())
+		return nil, fmt.Errorf("unable to read receipts at height %d hash %s", number, hash)
 	}
 	return receipts, nil
 }

 // GetTdByHash gets td by hash
-func (ldr *LvlDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
+func (ldr *EthDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
 	number := rawdb.ReadHeaderNumber(ldr.ethDB, hash)
 	if number == nil {
-		return nil, fmt.Errorf("unable to read header height for header hash %s", hash.String())
+		return nil, fmt.Errorf("unable to read header height for header hash %s", hash)
 	}
 	td := rawdb.ReadTd(ldr.ethDB, hash, *number)
 	if td == nil {
-		return nil, fmt.Errorf("unable to read total difficulty at height %d hash %s", number, hash.String())
+		return nil, fmt.Errorf("unable to read total difficulty at height %d hash %s", number, hash)
 	}
 	return td, nil
 }

 // StateDB returns the underlying statedb
-func (ldr *LvlDBReader) StateDB() state.Database {
+func (ldr *EthDBReader) StateDB() state.Database {
 	return ldr.stateDB
 }
+
+// GetLatestHeader gets the latest header from the levelDB
+func (ldr *EthDBReader) GetLatestHeader() (*types.Header, error) {
+	header := rawdb.ReadHeadHeader(ldr.ethDB)
+	if header == nil {
+		return nil, errors.New("unable to read head header")
+	}
+	return header, nil
+}
--- a/pkg/rpc/check.go
+++ b/pkg/rpc/check.go
@ -1,37 +0,0 @@
-// VulcanizeDB
-// Copyright © 2020 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package rpc
-
-import "github.com/ethereum/go-ethereum/rpc"
-
-// checkModuleAvailability check that all names given in modules are actually
-// available API services.
-func checkModuleAvailability(modules []string, apis []rpc.API) (bad, available []string) {
-	availableSet := make(map[string]struct{})
-	for _, api := range apis {
-		if _, ok := availableSet[api.Namespace]; !ok {
-			availableSet[api.Namespace] = struct{}{}
-			available = append(available, api.Namespace)
-		}
-	}
-	for _, name := range modules {
-		if _, ok := availableSet[name]; !ok {
-			bad = append(bad, name)
-		}
-	}
-	return bad, available
-}
--- a/pkg/rpc/ipc.go
+++ b/pkg/rpc/ipc.go
@ -79,7 +79,7 @@ func StartIPCEndpoint(ipcEndpoint string, apis []rpc.API) (net.Listener, *rpc.Se
 		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
 			return nil, nil, err
 		}
-		log.Debug("IPC registered", "namespace", api.Namespace)
+		log.WithField("namespace", api.Namespace).Debug("IPC server registered")
 	}
 	// All APIs registered, start the IPC listener.
 	listener, err := ipcListen(ipcEndpoint)
--- a/pkg/service.go
+++ b/pkg/service.go
@ -22,15 +22,15 @@ import (
 	"sync"
 	"time"

+	"github.com/cerc-io/plugeth-statediff"
+	"github.com/cerc-io/plugeth-statediff/adapt"
+	"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
+	sdtypes "github.com/cerc-io/plugeth-statediff/types"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	sd "github.com/ethereum/go-ethereum/statediff"
-	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
-	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/sirupsen/logrus"

 	"github.com/cerc-io/eth-statediff-service/pkg/prom"
@ -38,37 +38,12 @@ import (

 const defaultQueueSize = 1024

-// StateDiffService is the state-diffing service interface
-type StateDiffService interface {
-	// Lifecycle Start() and Stop()
-	node.Lifecycle
-	// APIs and Protocols() interface for node service registration
-	APIs() []rpc.API
-	Protocols() []p2p.Protocol
-	// Loop is the main event loop for processing state diffs
-	Loop(wg *sync.WaitGroup) error
-	// Run is a one-off command to run on a predefined set of ranges
-	Run(ranges []RangeRequest, parallel bool) error
-	// StateDiffAt method to get state diff object at specific block
-	StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
-	// StateDiffFor method to get state diff object at specific block
-	StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.Payload, error)
-	// StateTrieAt method to get state trie object at specific block
-	StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
-	// WriteStateDiffAt method to write state diff object directly to DB
-	WriteStateDiffAt(blockNumber uint64, params sd.Params) error
-	// WriteStateDiffFor method to get state trie object at specific block
-	WriteStateDiffFor(blockHash common.Hash, params sd.Params) error
-	// WriteStateDiffsInRange method to wrtie state diff objects within the range directly to the DB
-	WriteStateDiffsInRange(start, stop uint64, params sd.Params) error
-}
-
 // Service is the underlying struct for the state diffing service
 type Service struct {
 	// Used to build the state diff objects
-	Builder sd.Builder
+	builder statediff.Builder
 	// Used to read data from LevelDB
-	lvlDBReader Reader
+	reader Reader
 	// Used to signal shutdown of the service
 	quitChan chan struct{}
 	// Interface for publishing statediffs as PG-IPLD objects
@ -82,22 +57,20 @@ type Service struct {
 }

 // NewStateDiffService creates a new Service
-func NewStateDiffService(lvlDBReader Reader, indexer interfaces.StateDiffIndexer, conf Config) (*Service, error) {
-	b, err := NewBuilder(lvlDBReader.StateDB(), conf.TrieWorkers)
-	if err != nil {
-		return nil, err
-	}
+func NewStateDiffService(reader Reader, indexer interfaces.StateDiffIndexer, conf ServiceConfig) *Service {
+	builder := statediff.NewBuilder(adapt.GethStateView(reader.StateDB()))
+	builder.SetSubtrieWorkers(conf.TrieWorkers)
 	if conf.WorkerQueueSize == 0 {
 		conf.WorkerQueueSize = defaultQueueSize
 	}
 	return &Service{
-		lvlDBReader: lvlDBReader,
-		Builder:     b,
-		indexer:     indexer,
-		workers:     conf.ServiceWorkers,
-		queue:       make(chan RangeRequest, conf.WorkerQueueSize),
-		preruns:     conf.PreRuns,
-	}, nil
+		reader:  reader,
+		builder: builder,
+		indexer: indexer,
+		workers: conf.ServiceWorkers,
+		queue:   make(chan RangeRequest, conf.WorkerQueueSize),
+		preruns: conf.PreRuns,
+	}
 }

 // Protocols exports the services p2p protocols, this service has none
@ -117,42 +90,68 @@ func (sds *Service) APIs() []rpc.API {
 	}
 }

+func segmentRange(workers, start, stop uint64, params statediff.Params) []RangeRequest {
+	segmentSize := ((stop - start) + 1) / workers
+	remainder := ((stop - start) + 1) % workers
+	numOfSegments := workers
+	if remainder > 0 {
+		numOfSegments++
+	}
+	segments := make([]RangeRequest, numOfSegments)
+	for i := range segments {
+		end := start + segmentSize - 1
+		if end > stop {
+			end = stop
+		}
+		segments[i] = RangeRequest{start, end, params}
+		start = end + 1
+	}
+	return segments
+}
+
 // Run does a one-off processing run on the provided RangeRequests + any pre-runs, exiting afterwards
 func (sds *Service) Run(rngs []RangeRequest, parallel bool) error {
 	for _, preRun := range sds.preruns {
+		// if the rangeSize is smaller than the number of workers
+		// make sure we do synchronous processing to avoid quantization issues
+		rangeSize := (preRun.Stop - preRun.Start) + 1
+		numWorkers := uint64(sds.workers)
+		if rangeSize < numWorkers {
+			parallel = false
+		}
 		if parallel {
-			// Chunk overall range into N subranges for workers
-			chunkSize := (preRun.Stop - preRun.Start) / uint64(sds.workers)
 			logrus.Infof("parallel processing prerun range (%d, %d) (%d blocks) divided into %d sized chunks with %d workers", preRun.Start, preRun.Stop,
-				preRun.Stop-preRun.Start+1, chunkSize, sds.workers)
-			// Sanity floor the chunk size
-			if chunkSize < 100 {
-				chunkSize = 100
-				logrus.Infof("Computed range chunk size for each worker is too small, defaulting to 100")
-			}
+				rangeSize, rangeSize/numWorkers, numWorkers)
+			workChan := make(chan RangeRequest)
+			quitChan := make(chan struct{})
+			// spin up numWorkers number of worker goroutines
 			wg := new(sync.WaitGroup)
-			for i := 0; i < int(sds.workers); i++ {
-				blockRange := RangeRequest{
-					Start:  preRun.Start + uint64(i)*chunkSize,
-					Stop:   preRun.Start + uint64(i)*chunkSize + chunkSize - 1,
-					Params: preRun.Params,
-				}
-				// TODO(hack) this fixes quantization
-				if blockRange.Stop < preRun.Stop && preRun.Stop-blockRange.Stop < chunkSize {
-					blockRange.Stop = preRun.Stop
-				}
+			for i := 0; i < int(numWorkers); i++ {
 				wg.Add(1)
 				go func(id int) {
 					defer wg.Done()
-					logrus.Infof("prerun worker %d processing range (%d, %d)", id, blockRange.Start, blockRange.Stop)
-					for j := blockRange.Start; j <= blockRange.Stop; j++ {
-						if err := sds.WriteStateDiffAt(j, blockRange.Params); err != nil {
-							logrus.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, blockRange.Start, blockRange.Stop, err)
+					for {
+						select {
+						case workerSegment := <-workChan:
+							for j := workerSegment.Start; j <= workerSegment.Stop; j++ {
+								if err := sds.WriteStateDiffAt(j, workerSegment.Params); err != nil {
+									logrus.Errorf("error writing statediff at height %d in range (%d, %d) : %v", id, workerSegment.Start, workerSegment.Stop, err)
+								}
+							}
+							logrus.Infof("prerun worker %d finished processing range (%d, %d)", id, workerSegment.Start, workerSegment.Stop)
+						case <-quitChan:
+							return
 						}
 					}
-					logrus.Infof("prerun worker %d finished processing range (%d, %d)", id, blockRange.Start, blockRange.Stop)
 				}(i)
 			}
+			// break range up into segments
+			segments := segmentRange(numWorkers, preRun.Start, preRun.Stop, preRun.Params)
+			// send the segments to the work channel
+			for _, segment := range segments {
+				workChan <- segment
+			}
+			close(quitChan)
 			wg.Wait()
 		} else {
 			logrus.Infof("sequential processing prerun range (%d, %d)", preRun.Start, preRun.Stop)
@ -190,25 +189,24 @@ func (sds *Service) Loop(wg *sync.WaitGroup) error {
 			for {
 				select {
 				case blockRange := <-sds.queue:
-					logrus.Infof("service worker %d received range (%d, %d) off of work queue, beginning processing", id, blockRange.Start, blockRange.Stop)
+					log := logrus.WithField("range", blockRange).WithField("worker", id)
+					log.Debug("processing range")
 					prom.DecQueuedRanges()
 					for j := blockRange.Start; j <= blockRange.Stop; j++ {
 						if err := sds.WriteStateDiffAt(j, blockRange.Params); err != nil {
-							logrus.Errorf("service worker %d error writing statediff at height %d in range (%d, %d) : %v", id, j, blockRange.Start, blockRange.Stop, err)
+							log.Errorf("error writing statediff at block %d: %v", j, err)
 						}
 						select {
 						case <-sds.quitChan:
-							logrus.Infof("closing service worker %d\n"+
-								"working in range (%d, %d)\n"+
-								"last processed height: %d", id, blockRange.Start, blockRange.Stop, j)
+							log.Infof("closing service worker (last processed block: %d)", j)
 							return
 						default:
-							logrus.Infof("service worker %d finished processing statediff height %d in range (%d, %d)", id, j, blockRange.Start, blockRange.Stop)
+							log.Infof("Finished processing block %d", j)
 						}
 					}
-					logrus.Infof("service worker %d finished processing range (%d, %d)", id, blockRange.Start, blockRange.Stop)
+					log.Debugf("Finished processing range")
 				case <-sds.quitChan:
-					logrus.Infof("closing the statediff service loop worker %d", id)
+					logrus.Debugf("closing the statediff service loop worker %d", id)
 					return
 				}
 			}
@ -225,8 +223,8 @@ func (sds *Service) Loop(wg *sync.WaitGroup) error {

 // StateDiffAt returns a state diff object payload at the specific blockheight
 // This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error) {
-	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
+func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
+	currentBlock, err := sds.reader.GetBlockByNumber(blockNumber)
 	if err != nil {
 		return nil, err
 	}
@ -238,7 +236,7 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Paylo
 	if blockNumber == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
 	}
-	parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
+	parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
 	if err != nil {
 		return nil, err
 	}
@ -247,12 +245,12 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Paylo

 // StateDiffFor returns a state diff object payload for the specific blockhash
 // This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.Payload, error) {
-	currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
+func (sds *Service) StateDiffFor(blockHash common.Hash, params statediff.Params) (*statediff.Payload, error) {
+	currentBlock, err := sds.reader.GetBlockByHash(blockHash)
 	if err != nil {
 		return nil, err
 	}
-	logrus.Infof("sending state diff at block %s", blockHash.Hex())
+	logrus.Infof("sending state diff at block %s", blockHash)

 	// compute leaf paths of watched addresses in the params
 	params.ComputeWatchedAddressesLeafPaths()
@ -260,7 +258,7 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.P
 	if currentBlock.NumberU64() == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
 	}
-	parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
+	parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
 	if err != nil {
 		return nil, err
 	}
@ -268,8 +266,8 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.P
 }

 // processStateDiff method builds the state diff payload from the current block, parent state root, and provided params
-func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params sd.Params) (*sd.Payload, error) {
-	stateDiff, err := sds.Builder.BuildStateDiffObject(sd.Args{
+func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params statediff.Params) (*statediff.Payload, error) {
+	stateDiff, err := sds.builder.BuildStateDiffObject(statediff.Args{
 		BlockHash:    currentBlock.Hash(),
 		BlockNumber:  currentBlock.Number(),
 		OldStateRoot: parentRoot,
@ -286,8 +284,8 @@ func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot commo
 	return sds.newPayload(stateDiffRlp, currentBlock, params)
 }

-func (sds *Service) newPayload(stateObject []byte, block *types.Block, params sd.Params) (*sd.Payload, error) {
-	payload := &sd.Payload{
+func (sds *Service) newPayload(stateObject []byte, block *types.Block, params statediff.Params) (*statediff.Payload, error) {
+	payload := &statediff.Payload{
 		StateObjectRlp: stateObject,
 	}
 	if params.IncludeBlock {
@ -299,14 +297,14 @@ func (sds *Service) newPayload(stateObject []byte, block *types.Block, params sd
 	}
 	if params.IncludeTD {
 		var err error
-		payload.TotalDifficulty, err = sds.lvlDBReader.GetTdByHash(block.Hash())
+		payload.TotalDifficulty, err = sds.reader.GetTdByHash(block.Hash())
 		if err != nil {
 			return nil, err
 		}
 	}
 	if params.IncludeReceipts {
 		receiptBuff := new(bytes.Buffer)
-		receipts, err := sds.lvlDBReader.GetReceiptsByHash(block.Hash())
+		receipts, err := sds.reader.GetReceiptsByHash(block.Hash())
 		if err != nil {
 			return nil, err
 		}
@ -318,34 +316,6 @@ func (sds *Service) newPayload(stateObject []byte, block *types.Block, params sd
 	return payload, nil
 }

-// StateTrieAt returns a state trie object payload at the specified blockheight
-// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error) {
-	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Infof("sending state trie at block %d", blockNumber)
-
-	// compute leaf paths of watched addresses in the params
-	params.ComputeWatchedAddressesLeafPaths()
-
-	return sds.processStateTrie(currentBlock, params)
-}
-
-func (sds *Service) processStateTrie(block *types.Block, params sd.Params) (*sd.Payload, error) {
-	stateNodes, err := sds.Builder.BuildStateTrieObject(block)
-	if err != nil {
-		return nil, err
-	}
-	stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Infof("state trie object at block %d is %d bytes in length", block.Number().Uint64(), len(stateTrieRlp))
-	return sds.newPayload(stateTrieRlp, block, params)
-}
-
 // Start is used to begin the service
 func (sds *Service) Start() error {
 	logrus.Info("starting statediff service")
@ -362,10 +332,10 @@ func (sds *Service) Stop() error {
 // WriteStateDiffAt writes a state diff at the specific blockheight directly to the database
 // This operation cannot be performed back past the point of db pruning; it requires an archival node
 // for historical data
-func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error {
+func (sds *Service) WriteStateDiffAt(blockNumber uint64, params statediff.Params) error {
 	logrus.Infof("Writing state diff at block %d", blockNumber)
 	t := time.Now()
-	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
+	currentBlock, err := sds.reader.GetBlockByNumber(blockNumber)
 	if err != nil {
 		return err
 	}
@ -375,7 +345,7 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error

 	parentRoot := common.Hash{}
 	if blockNumber != 0 {
-		parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
+		parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
 		if err != nil {
 			return err
 		}
@ -387,10 +357,10 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
 // WriteStateDiffFor writes a state diff for the specific blockHash directly to the database
 // This operation cannot be performed back past the point of db pruning; it requires an archival node
 // for historical data
-func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) error {
-	logrus.Infof("Writing state diff for block %s", blockHash.Hex())
+func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params statediff.Params) error {
+	logrus.Infof("Writing state diff for block %s", blockHash)
 	t := time.Now()
-	currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
+	currentBlock, err := sds.reader.GetBlockByHash(blockHash)
 	if err != nil {
 		return err
 	}
@ -400,7 +370,7 @@ func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) e

 	parentRoot := common.Hash{}
 	if currentBlock.NumberU64() != 0 {
-		parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
+		parentBlock, err := sds.reader.GetBlockByHash(currentBlock.ParentHash())
 		if err != nil {
 			return err
 		}
@ -410,18 +380,18 @@ func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) e
 }

 // Writes a state diff from the current block, parent state root, and provided params
-func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params sd.Params, t time.Time) error {
+func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params statediff.Params, t time.Time) error {
 	var totalDifficulty *big.Int
 	var receipts types.Receipts
 	var err error
 	if params.IncludeTD {
-		totalDifficulty, err = sds.lvlDBReader.GetTdByHash(block.Hash())
+		totalDifficulty, err = sds.reader.GetTdByHash(block.Hash())
 	}
 	if err != nil {
 		return err
 	}
 	if params.IncludeReceipts {
-		receipts, err = sds.lvlDBReader.GetReceiptsByHash(block.Hash())
+		receipts, err = sds.reader.GetReceiptsByHash(block.Hash())
 	}
 	if err != nil {
 		return err
@ -435,28 +405,37 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
 		return err
 	}
 	// defer handling of commit/rollback for any return case
-	output := func(node sdtypes.StateNode) error {
+	defer tx.RollbackOnFailure(err)
+
+	var nodeMtx, ipldMtx sync.Mutex
+	output := func(node sdtypes.StateLeafNode) error {
+		nodeMtx.Lock()
+		defer nodeMtx.Unlock()
 		return sds.indexer.PushStateNode(tx, node, block.Hash().String())
 	}
-	codeOutput := func(c sdtypes.CodeAndCodeHash) error {
-		return sds.indexer.PushCodeAndCodeHash(tx, c)
+	ipldOutput := func(c sdtypes.IPLD) error {
+		ipldMtx.Lock()
+		defer ipldMtx.Unlock()
+		return sds.indexer.PushIPLD(tx, c)
 	}
 	prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t))
 	t = time.Now()
-	err = sds.Builder.WriteStateDiffObject(sdtypes.StateRoots{
+	err = sds.builder.WriteStateDiff(statediff.Args{
 		NewStateRoot: block.Root(),
 		OldStateRoot: parentRoot,
-	}, params, output, codeOutput)
+		BlockNumber:  block.Number(),
+		BlockHash:    block.Hash(),
+	}, params, output, ipldOutput)
 	prom.SetTimeMetric(prom.T_STATE_PROCESSING, time.Now().Sub(t))
 	t = time.Now()
-	err = tx.Submit(err)
+	err = tx.Submit()
 	prom.SetLastProcessedHeight(height)
 	prom.SetTimeMetric(prom.T_POSTGRES_TX_COMMIT, time.Now().Sub(t))
 	return err
 }

 // WriteStateDiffsInRange adds a RangeRequest to the work queue
-func (sds *Service) WriteStateDiffsInRange(start, stop uint64, params sd.Params) error {
+func (sds *Service) WriteStateDiffsInRange(start, stop uint64, params statediff.Params) error {
 	if stop < start {
 		return fmt.Errorf("invalid block range (%d, %d): stop height must be greater or equal to start height", start, stop)
 	}
@ -464,7 +443,7 @@ func (sds *Service) WriteStateDiffsInRange(start, stop uint64, params sd.Params)
 	select {
 	case sds.queue <- RangeRequest{Start: start, Stop: stop, Params: params}:
 		prom.IncQueuedRanges()
-		logrus.Infof("added range (%d, %d) to the worker queue", start, stop)
+		logrus.Infof("Added range (%d, %d) to the worker queue", start, stop)
 		return nil
 	case <-blocked.C:
 		return fmt.Errorf("unable to add range (%d, %d) to the worker queue", start, stop)
--- a/pkg/types.go
+++ b/pkg/types.go
@ -20,25 +20,17 @@
 package statediff

 import (
-	"github.com/ethereum/go-ethereum/core/types"
-	sd "github.com/ethereum/go-ethereum/statediff"
-	sdTypes "github.com/ethereum/go-ethereum/statediff/types"
+	"fmt"
+
+	sd "github.com/cerc-io/plugeth-statediff"
 )

-// AccountMap is a mapping of hex encoded path => account wrapper
-type AccountMap map[string]accountWrapper
-
-// accountWrapper is used to temporary associate the unpacked node with its raw values
-type accountWrapper struct {
-	Account   *types.StateAccount
-	NodeType  sdTypes.NodeType
-	Path      []byte
-	NodeValue []byte
-	LeafKey   []byte
-}
-
 // RangeRequest holds range quest work params
 type RangeRequest struct {
 	Start, Stop uint64
 	Params      sd.Params
 }
+
+func (r RangeRequest) String() string {
+	return fmt.Sprintf("[%d,%d]", r.Start, r.Stop)
+}
--- a/scripts/compare-statediffs.sh
+++ b/scripts/compare-statediffs.sh
@ -0,0 +1,122 @@
+#!/bin/bash
+#
+# Usage: compare-versions.sh [-d <output-dir>] <binary-A> <binary-B>
+#
+# Compares full statediff output from two versions of the service.
+# Configure the input data using environment vars.
+(
+  set -u
+  : $ETHDB_PATH
+  : $ETHDB_ANCIENT
+  : $ETH_GENESIS_BLOCK
+  : $ETH_CHAIN_CONFIG
+) || exit 1
+
+# Range of diffs to request
+range_start=50
+range_end=100
+
+# Get the parent directory
+script_dir=$(readlink -f "$(dirname -- "${BASH_SOURCE[0]}")")
+
+while getopts d: opt; do
+    case $opt in
+      d) output_dir="$OPTARG"
+    esac
+done
+shift $((OPTIND - 1))
+
+binary_A=$1
+binary_B=$2
+shift 2
+
+if [[ -z $output_dir ]]; then
+  output_dir=$(mktemp -d)
+fi
+
+export STATEDIFF_TRIE_WORKERS=32
+export STATEDIFF_SERVICE_WORKERS=8
+export STATEDIFF_WORKER_QUEUE_SIZE=1024
+
+export DATABASE_TYPE=postgres
+export DATABASE_NAME="cerc_testing"
+export DATABASE_HOSTNAME="localhost"
+export DATABASE_PORT=8077
+export DATABASE_USER="vdbm"
+export DATABASE_PASSWORD="password"
+
+export ETH_NODE_ID=test-node
+export ETH_CLIENT_NAME=test-client
+export ETH_NETWORK_ID=test-network
+
+export SERVICE_HTTP_PATH='127.0.0.1:8545'
+export LOG_LEVEL=debug
+
+dump_table() {
+  statement="copy (select * from $1) to stdout with csv"
+  docker exec -e PGPASSWORD="$DATABASE_PASSWORD" test-ipld-eth-db-1 \
+    psql -q $DATABASE_NAME -U $DATABASE_USER -c "$statement" | sort -u > "$2/$1.csv"
+}
+
+clear_table() {
+  docker exec -e PGPASSWORD="$DATABASE_PASSWORD" test-ipld-eth-db-1 \
+    psql -q $DATABASE_NAME -U $DATABASE_USER -c "truncate $1"
+}
+
+tables=(
+  eth.header_cids
+  eth.log_cids
+  eth.receipt_cids
+  eth.state_cids
+  eth.storage_cids
+  eth.transaction_cids
+  eth.uncle_cids
+  ipld.blocks
+  public.nodes
+)
+
+for table in "${tables[@]}"; do
+  clear_table $table
+done
+
+run_service() {
+  export LOG_FILE=$(mktemp)
+  export LOG_FILE_PATH=$LOG_FILE
+
+  service_binary=$1
+  service_output_dir=$2
+
+  mkdir -p $service_output_dir
+  $service_binary serve &
+
+  until grep "HTTP endpoint opened" $LOG_FILE
+  do sleep 1; done
+
+  $script_dir/request-range.sh $range_start $range_end
+  if E=$?; [[ $E != 0 ]]; then
+    cat $LOG_FILE
+    return $E
+  fi
+
+  echo "Waiting for service to complete requests..."
+
+  until grep \
+    -e "Finished processing block $range_end" \
+    -e "finished processing statediff height $range_end" \
+    $LOG_FILE
+  do sleep 1; done
+
+  kill -INT $!
+
+  mkdir -p $service_output_dir
+  for table in "${tables[@]}"; do
+    dump_table $table $service_output_dir
+    clear_table $table
+  done
+}
+
+set -e
+run_service $binary_A $output_dir/A
+run_service $binary_B $output_dir/B
+
+diff -rs $output_dir/A $output_dir/B
--- a/scripts/request-range.sh
+++ b/scripts/request-range.sh
@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -eu
+
+FROM=$1
+TO=$2
+URL='127.0.0.1:8545'
+
+DATA='{
+    "jsonrpc": "2.0",
+      "method": "statediff_writeStateDiffsInRange",
+      "params": ['"$FROM"', '"$TO"', {
+          "includeBlock": true,
+          "includeReceipts": true,
+          "includeTD": true,
+          "includeCode": true
+        }
+      ],
+      "id": 1
+}'
+
+exec curl -s $URL -X POST -H 'Content-Type: application/json' --data "$DATA"
--- a/startup_script.sh
+++ b/startup_script.sh
@ -1,20 +1,62 @@
-#!/bin/sh
+#!/bin/bash
 # Exit if the variable tests fail
 set -e
-set +x
+set -o pipefail
+
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi

 # Check the database variables are set
 test "$VDB_COMMAND"
-set +e

-# docker must be run in privilaged mode for mounts to work
-echo "Setting up /app/geth-rw overlayed /app/geth-ro"
-mkdir -p /tmp/overlay && \
-sudo mount -t tmpfs tmpfs /tmp/overlay && \
-mkdir -p /tmp/overlay/upper && \
-mkdir -p /tmp/overlay/work && \
-mkdir -p /app/geth-rw && \
-sudo mount -t overlay overlay -o lowerdir=/app/geth-ro,upperdir=/tmp/overlay/upper,workdir=/tmp/overlay/work /app/geth-rw && \
+# docker must be run in privileged mode for mounts to work
+echo "Setting up /app/geth-rw overlayed onto /app/geth-ro"
+mkdir -p /tmp/overlay
+mount -t tmpfs tmpfs /tmp/overlay
+mkdir -p /tmp/overlay/upper
+mkdir -p /tmp/overlay/work
+mkdir -p /app/geth-rw

+mount -t overlay overlay -o lowerdir=/app/geth-ro,upperdir=/tmp/overlay/upper,workdir=/tmp/overlay/work /app/geth-rw
+
+mkdir /var/run/statediff
+cd /var/run/statediff
+
+SETUID=""
+if [[ -n "$TARGET_UID" ]] && [[ -n "$TARGET_GID" ]]; then
+    SETUID="su-exec $TARGET_UID:$TARGET_GID"
+    chown -R $TARGET_UID:$TARGET_GID /var/run/statediff
+fi
+
+START_TIME=`date -u +"%Y-%m-%dT%H:%M:%SZ"`
 echo "Running the statediff service" && \
-sudo ./eth-statediff-service "$VDB_COMMAND" --config=config.toml
+if [[ ! -z "$LOG_FILE" ]]; then
+  $SETUID /app/eth-statediff-service "$VDB_COMMAND" --config=/app/config.toml $* |& $SETUID tee ${LOG_FILE}.console
+  rc=$?
+else
+  $SETUID /app/eth-statediff-service "$VDB_COMMAND" --config=/app/config.toml $*
+  rc=$?
+fi
+STOP_TIME=`date -u +"%Y-%m-%dT%H:%M:%SZ"`
+
+if [ $rc -eq 0 ] && [ "$VDB_COMMAND" == "serve" ] && [ "$PRERUN_ONLY" == "true" ] && [ ! -z "$PRERUN_RANGE_START" ] && [ ! -z "$PRERUN_RANGE_STOP" ] && [ ! -z "$DATABASE_FILE_CSV_DIR" ] && [ "$DATABASE_FILE_MODE" == "csv" ]; then
+  cat >metadata.json <<EOF
+{
+  "type": "statediff",
+  "range": { "start": $PRERUN_RANGE_START, "stop": $PRERUN_RANGE_STOP },
+  "nodeId": "$ETH_NODE_ID",
+  "genesisBlock": "$ETH_GENESIS_BLOCK",
+  "networkId": "$ETH_NETWORK_ID",
+  "chainId": "$ETH_CHAIN_ID",
+  "time": { "start": "$START_TIME", "stop": "$STOP_TIME" }
+}
+EOF
+  if [[ -n "$TARGET_UID" ]] && [[ -n "$TARGET_GID" ]]; then
+    echo 'metadata.json' | cpio -p --owner $TARGET_UID:$TARGET_GID $DATABASE_FILE_CSV_DIR
+  else
+    cp metadata.json $DATABASE_FILE_CSV_DIR
+  fi
+fi
+
+exit $rc
--- a/test/ci-config.toml
+++ b/test/ci-config.toml
@ -0,0 +1,32 @@
+# Config file for github workflow
+
+[server]
+    ipcPath = ".ipc"
+    httpPath = "0.0.0.0:8545"
+
+[statediff]
+    serviceWorkers = 1
+    workerQueueSize = 1024
+    trieWorkers = 4
+
+[log]
+    level = "debug"
+
+[database]
+    name            = "cerc_testing"
+    hostname        = "localhost"
+    port            = 8077
+    user            = "vdbm"
+    password        = "password"
+    type            = "postgres"
+    driver          = "sqlx"
+
+[cache]
+    database = 1024
+    trie = 1024
+
+[ethereum]
+    nodeID = "test-node"
+    clientName = "test-eth-statediff-service"
+    networkID = "test-network"
+    chainID = 41337
--- a/test/compose.yml
+++ b/test/compose.yml
@ -0,0 +1,23 @@
+services:
+  migrations:
+    restart: on-failure
+    depends_on:
+      - ipld-eth-db
+    image: git.vdb.to/cerc-io/ipld-eth-db/ipld-eth-db:v5.3.0-alpha
+    environment:
+      DATABASE_USER: "vdbm"
+      DATABASE_NAME: "cerc_testing"
+      DATABASE_PASSWORD: "password"
+      DATABASE_HOSTNAME: "ipld-eth-db"
+      DATABASE_PORT: 5432
+
+  ipld-eth-db:
+    image: timescale/timescaledb:latest-pg14
+    restart: always
+    command: ["postgres", "-c", "log_statement=all"]
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "cerc_testing"
+      POSTGRES_PASSWORD: "password"
+    ports:
+      - 127.0.0.1:8077:5432
Author	SHA1	Message	Date
Roy Crihfield	cdacea3aae	Updates for Cancun fork (#7 ) - Bumps Geth to v1.14 - Adds support for pebbledb - Drops leveldb-ethdb-rpc Reviewed-on: #7	2024-08-05 13:08:26 +00:00
Thomas E Lackey	be105a4744	Fix checksum build error.	2023-10-27 17:12:06 -05:00
Roy Crihfield	161febc47a	Fix concurrency and tests (#4 ) 1. Synchronizes pushes to indexer - The Tx object is not threadsafe. 2. Fixes the chain config for the CI test fixtures, which were not accurate for the data, and causing processing errors Reviewed-on: #4	2023-10-04 07:44:43 +00:00
Thomas E Lackey	167cd2839c	Refactor to use plugeth-statediff (#2 ) (#3 ) Rebase of #1 onto real `v5` branch. * Updates to the v5 schema, by linking the statediff plugin (as of this branch cerc-io/plugeth-statediff#15). This replaces the existing builder code. * Adds basic CI workflows * Updates Docker config and docs, cleans up some things Co-authored-by: Thomas E Lackey <telackey@bozemanpass.com> Reviewed-on: #2 Co-authored-by: Roy Crihfield <roy@manteia.ltd> Co-committed-by: Roy Crihfield <roy@manteia.ltd> Co-authored-by: Roy Crihfield <roy@manteia.ltd> Reviewed-on: #3	2023-09-29 20:27:08 +00:00
Thomas E Lackey	151a805b53	Update to geth v1.11.6-statediff-5.0.8 (#135 ) * Update to geth v1.11.6-statediff-5.0.8	2023-07-21 19:05:57 -05:00
Ian Norden	232cb1bcbf	Merge pull request #133 from cerc-io/ian/v5_dev update geth dep	2023-06-01 08:53:18 -05:00
i-norden	36c104f4db	update geth dep	2023-06-01 08:46:19 -05:00
David Boreham	f144863b3b	Merge pull request #131 from cerc-io/telackey/type Include type in metadata output	2023-05-09 08:05:19 -07:00
Thomas E Lackey	5dafc1fbc0	Include type	2023-05-08 17:19:51 -05:00
Thomas E Lackey	fb040d7484	Run as root, but switch to the target UID/GID is supplied (fixes file permission issues with mounted volumes). (#130 )	2023-04-28 22:14:45 -05:00
Thomas E Lackey	60e951a172	tee logs (#129 )	2023-04-27 22:24:22 -05:00
Thomas E Lackey	9016be38a0	Merge pull request #128 from cerc-io/telackey/docker_prerun Output metadata on prerun.	2023-04-27 11:42:52 -05:00
Thomas E Lackey	a17ace0187	Output metadata on prerun.	2023-04-26 22:58:28 -05:00
Ian Norden	5a623fa8d3	Merge pull request #127 from cerc-io/ian/v5_dev fix builder unit tests	2023-04-10 01:37:49 -05:00
i-norden	75c2c4c348	don't skip	2023-04-10 01:27:53 -05:00
i-norden	834930a498	fix builder unit tests	2023-04-10 01:18:34 -05:00
Ian Norden	0a92769bc5	Merge pull request #126 from cerc-io/ian/v5_dev revert to using os.Interrupt	2023-04-09 14:33:19 -05:00
i-norden	8bcb2c312f	revert to using os.Interrupt	2023-04-09 14:26:21 -05:00
Ian Norden	2bb0b1aad6	Merge pull request #125 from cerc-io/ian/v5_dev Update to v5	2023-04-09 13:35:09 -05:00
i-norden	4ddd3d0e20	go mod tidy	2023-04-09 11:01:31 -05:00
i-norden	07bd875752	update tomls and README	2023-04-09 11:01:23 -05:00
i-norden	f3379e33dd	update cmd to v5	2023-04-09 11:01:12 -05:00
i-norden	2ff4e95040	update pkgs to v5	2023-04-09 11:01:05 -05:00
Ian Norden	18a7f23173	Merge pull request #124 from cerc-io/ian/v4_dev bump leveldb-ethdb-rpc version	2023-03-31 10:25:24 -05:00
i-norden	ef1846f58c	bump leveldb-ethdb-rpc version	2023-03-31 10:17:34 -05:00
Ian Norden	2eaa2c2262	Merge pull request #123 from cerc-io/ian/v4_dev [v4] bump to vdb v4 geth v1.11.5	2023-03-31 09:53:11 -05:00
i-norden	bfb0447710	use internal concurrent iterator pkg	2023-03-31 09:04:54 -05:00
i-norden	2c08f5594c	bump to vdb v4 geth v1.11.5	2023-03-31 09:02:54 -05:00
Ian Norden	253b1087bf	Merge pull request #112 from cerc-io/ian_test_3 Avoid quantization problems during segmentation and use worker chan to spread work across set # of workers	2023-03-20 07:53:09 -05:00
Michael	8a3b6bf2ac	Merge pull request #121 from cerc-io/rebase-1.11.4-wip version update for geth 1.11.4 statediff and container to go 1.19	2023-03-17 14:05:56 -04:00
Michael Shaw	b221bde694	version update for geth 1.11.4 statediff and container to go 1.19	2023-03-17 13:51:46 -04:00
Michael	2c41537636	Merge pull request #120 from cerc-io/rebase-v1.11.2-wip update interface for failing docker build step	2023-03-15 11:18:30 -04:00
Michael Shaw	d83b088c37	update interface for failing docker build step	2023-03-15 01:38:22 -04:00
Michael	aca78f89b7	Merge pull request #118 from cerc-io/rebase-v1.11.2-wip update for interface changes in 1.11.2 geth statediffing 4.3.7	2023-03-14 15:09:38 -04:00
Michael Shaw	1788b899a4	update for interface changes in 1.11.2 geth statediffing 4.3.7	2023-03-13 18:35:54 -04:00
Michael	1fe7a04af0	Merge pull request #116 from cerc-io/v4.1.6-wip dependency updates for geth 1.10.26 and leveldb cerc migraiton	2022-11-08 11:41:19 -05:00
Michael Shaw	68ebdca6f9	dependency updates for geth 1.10.26 and leveldb cerc migraiton	2022-11-08 11:19:27 -05:00
i-norden	5f7915649d	avoid quantization and use worker channel to ensure expected number of workers	2022-10-12 18:39:28 -05:00
Ian Norden	f6df15cb38	Merge pull request #110 from cerc-io/ian_util latest block height util	2022-10-10 18:31:14 -05:00
i-norden	03517a0eb4	add short description for stats command in the readme	2022-10-10 18:24:24 -05:00
i-norden	54205d8787	conform capitalization in logWithCommand logs	2022-10-10 13:44:46 -05:00
i-norden	8d8ff99d19	cmd for checking latest header height/hash; update serve command to report the latest header height/hash	2022-10-10 13:39:06 -05:00
i-norden	3054063942	add method to reader interface for accessing latest header	2022-10-10 13:38:33 -05:00
Michael	54e181ca68	Update manual_publish.yml publish SHA as latest	2022-09-28 17:50:24 -04:00
Michael	269333bb17	Update manual_publish.yml	2022-09-28 17:47:42 -04:00
Michael	db532467cc	Update manual_publish.yml	2022-09-28 17:45:25 -04:00
Michael	ccdf9d91fc	Update manual_publish.yml	2022-09-28 17:43:19 -04:00
Michael	aea3decebf	Update manual_publish.yml	2022-09-28 17:40:14 -04:00
Michael	2db16d69da	Create manual_publish.yml	2022-09-28 17:37:08 -04:00
Michael	e3d694e63c	Merge pull request #106 from cerc-io/geth-1.10.25-wip dependency updates for geth 1.10.25 rebase	2022-09-23 14:25:02 -04:00
Michael Shaw	2c0f3456f5	dependency updates for geth 1.10.25 rebase	2022-09-23 14:17:51 -04:00