one more reference changed to git.vdb.to

missed Makefile update for cerc-io
indentation problem in yml
2022-09-21 13:27:05 -04:00 · 2022-09-21 11:55:17 -04:00 · 2022-09-21 11:24:43 -04:00 · 2022-09-21 11:13:23 -04:00 · 2022-09-20 14:00:10 -04:00 · 2022-09-02 16:16:34 -04:00
32 changed files with 2499 additions and 1835 deletions
@@ -1,25 +0,0 @@
-name: Docker Compose Build
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag vulcanize/eth-statediff-service docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Push
-        run: docker push docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-
@@ -1,12 +0,0 @@
-name: Docker Build
-
-on: [pull_request]
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run docker build
-        run: make docker-build
@@ -0,0 +1,73 @@
+name: Publish Docker image
+on:
+  release:
+    types: [published]
+  pull_request:
+jobs:
+  pre_job:
+    # continue-on-error: true # Uncomment once integration is finished
+    runs-on: ubuntu-latest
+    # Map a step output to a job output
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@v4
+        with:
+          # All of these options are optional, so you can remove them if you are happy with the defaults
+          concurrent_skipping: "never"
+          skip_after_successful_duplicate: "true"
+          do_not_skip: '["workflow_dispatch", "schedule"]'
+  run-tests:
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    needs: pre_job
+    uses: ./.github/workflows/tests.yml
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    if: |
+      always() &&
+      (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') &&
+      github.event_name == 'release'
+    needs: run-tests
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the version
+        id: vars
+        run: |
+          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
+      - name: Run docker build
+        run: make docker-build
+      - name: Tag docker image
+        run: docker tag cerc-io/eth-statediff-service git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+      - name: Tag docker image TAG
+        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
+      - name: Docker Login
+        run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+      - name: Docker Push
+        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+#  push_to_registries:
+#    name: Push Docker image to Docker Hub
+#    runs-on: ubuntu-latest
+#    if: |
+#      always() &&
+#      (needs.build.result == 'success') &&
+#      github.event_name == 'release'
+#    needs: build
+#    steps:
+#      - name: Get the version
+#        id: vars
+#        run: |
+#          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+#          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
+#      - name: Docker Login to Github Registry
+#        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+#      - name: Docker Pull
+#        run: docker pull docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+#      - name: Docker Login to Docker Registry
+#        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
+#      - name: Tag docker image
+#        run: docker tag docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
+#      - name: Docker Push to Docker Hub
+#        run: docker push cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
@@ -1,25 +0,0 @@
-name: Publish Docker image
-on:
-  release:
-    types: [published]
-jobs:
-  push_to_registries:
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: |
-          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-          echo ::set-output name=branch::$(echo ${GITHUB_REF##*/})
-      - name: Docker Login
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          tags: vulcanize/eth-statediff-service:${{ steps.vars.outputs.tag }}
-          build-args: |
-            BRANCH=${{ steps.vars.outputs.branch }}
@@ -0,0 +1,37 @@
+name: Tests for Geth that are used in multiple jobs.
+
+on:
+  workflow_call:
+
+env:
+  GOPATH: /tmp/go
+
+jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run docker build
+        run: make docker-build
+
+  statediff-unit-test:
+    name: Run statediff unit tests
+    runs-on: ubuntu-latest
+    env:
+      GO111MODULE: on
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Run unit tests
+        run: |
+          make test
@@ -1,2 +1,3 @@
 .idea/
 eth-statediff-service
+.vscode
@@ -1,14 +1,14 @@
-FROM golang:1.13-alpine as builder
+FROM golang:1.18-alpine as builder

 RUN apk --update --no-cache add make git g++ linux-headers
 # DEBUG
 RUN apk add busybox-extras

 # Get and build ipfs-blockchain-watcher
-ADD . /go/src/github.com/vulcanize/eth-statediff-service
-#RUN git clone https://github.com/vulcanize/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
+ADD . /go/src/github.com/cerc-io/eth-statediff-service
+#RUN git clone https://github.com/cerc-io/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service

-WORKDIR /go/src/github.com/vulcanize/eth-statediff-service
+WORKDIR /go/src/github.com/cerc-io/eth-statediff-service
 RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .

 # app container
@@ -27,12 +27,12 @@ USER $USER

 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/$CONFIG_FILE config.toml
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/startup_script.sh .
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/environments environments
+COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/$CONFIG_FILE config.toml
+COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/startup_script.sh .
+COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/environments environments

 # keep binaries immutable
-COPY --from=builder /go/src/github.com/vulcanize/eth-statediff-service/eth-statediff-service eth-statediff-service
+COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/eth-statediff-service eth-statediff-service

 EXPOSE $EXPOSE_PORT

@@ -1,4 +1,12 @@
 ## Build docker image
 .PHONY: docker-build
 docker-build:
-	docker build -t vulcanize/eth-statediff-service .
+	docker build -t cerc-io/eth-statediff-service .
+
+.PHONY: test
+test: | $(GOOSE)
+	go test -p 1 ./pkg/... -v
+
+build:
+	go fmt ./...
+	go build
@@ -2,35 +2,21 @@

 [![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/eth-statediff-service)](https://goreportcard.com/report/github.com/vulcanize/eth-statediff-service)

->> standalone statediffing service ontop of leveldb
+>> standalone statediffing service on top of LevelDB

 Purpose:

-Stand up a statediffing service directly on top of a go-ethereum leveldb instance.
+Stand up a statediffing service directly on top of a go-ethereum LevelDB instance.
 This service can serve historical state data over the same rpc interface as
-[statediffing geth](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.5) without needing to run a full node
+[statediffing geth](https://github.com/cerc-io/go-ethereum) without needing to run a full node.

-## Usage
+## Setup

-### `serve`
+Build the binary:

-To serve state diffs over RPC:
-
-`eth-statediff-service serve --config=<config path>`
-
-Available RPC methods are:
-  * `statediff_stateTrieAt()`
-  * `statediff_streamCodeAndCodeHash()`
-  * `statediff_stateDiffAt()`
-  * `statediff_writeStateDiffAt()`
-
-### `write`
-
-To write state diffs directly to a database:
-
-`eth-statediff-service write --config=<config path>`
-
-This depends on the `database` settings being properly configured.
+```bash
+make build
+```

 ## Configuration

@@ -38,31 +24,304 @@ An example config file:

 ```toml
 [leveldb]
-    path = "/Users/user/Library/Ethereum/geth/chaindata"
-    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
+    # LevelDB access mode <local | remote>
+    mode = "local"  # LVLDB_MODE
+
+    # in local mode
+    # LevelDB paths
+    path    = "/Users/user/Library/Ethereum/geth/chaindata"         # LVLDB_PATH
+    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LVLDB_ANCIENT
+
+    # in remote mode
+    # URL for leveldb-ethdb-rpc endpoint
+    url = "http://127.0.0.1:8082/"  # LVLDB_URL

 [server]
-    ipcPath = "~/.vulcanize/vulcanize.ipc"
-    httpPath = "127.0.0.1:8545"
-
-[write]
-    ranges = [[1, 2], [3, 4]]
-[write.params]
-    IntermediateStateNodes   = true
-    IntermediateStorageNodes = false
-    IncludeBlock             = true
-    IncludeReceipts          = true
-    IncludeTD                = true
-    IncludeCode              = false
+    ipcPath  = ".ipc"           # SERVICE_IPC_PATH
+    httpPath = "127.0.0.1:8545" # SERVICE_HTTP_PATH

 [statediff]
-    workers = 4
+    prerun          = true  # STATEDIFF_PRERUN
+    serviceWorkers  = 1     # STATEDIFF_SERVICE_WORKERS
+    workerQueueSize = 1024  # STATEDIFF_WORKER_QUEUE_SIZE
+    trieWorkers     = 4     # STATEDIFF_TRIE_WORKERS
+
+[prerun]
+    only = false    # PRERUN_ONLY
+
+    # to perform prerun in a specific range (optional)
+    start = 0   # PRERUN_RANGE_START
+    stop  = 100 # PRERUN_RANGE_STOP
+
+    # to perform prerun over multiple ranges (optional)
+    ranges = [
+        [101, 1000]
+    ]
+
+    # statediffing params for prerun
+    [prerun.params]
+        intermediateStateNodes   = true # PRERUN_INTERMEDIATE_STATE_NODES
+        intermediateStorageNodes = true # PRERUN_INTERMEDIATE_STORAGE_NODES
+        includeBlock             = true # PRERUN_INCLUDE_BLOCK
+        includeReceipts          = true # PRERUN_INCLUDE_RECEIPTS
+        includeTD                = true # PRERUN_INCLUDE_TD
+        includeCode              = true # PRERUN_INCLUDE_CODE
+        watchedAddresses         = []

 [log]
-    file = "~/.vulcanize/statediff.log"
-    level = "info"
+    file  = ""      # LOG_FILE_PATH
+    level = "info"  # LOG_LEVEL

-[eth]
-    chainID = 1
+[database]
+    # output type <postgres | file | dump>
+    type = "postgres"

+    # with postgres type
+    # db credentials
+    name     = "vulcanize_test" # DATABASE_NAME
+    hostname = "localhost"      # DATABASE_HOSTNAME
+    port     = 5432             # DATABASE_PORT
+    user     = "vulcanize"      # DATABASE_USER
+    password = "..."            # DATABASE_PASSWORD
+    driver   = "sqlx"           # DATABASE_DRIVER_TYPE <sqlx | pgx>
+
+    # with file type
+    # file mode <sql | csv>
+    fileMode = "csv"    # DATABASE_FILE_MODE
+
+    # with SQL file mode
+    filePath = ""   # DATABASE_FILE_PATH
+
+    # with CSV file mode
+    fileCsvDir = "output_dir" # DATABASE_FILE_CSV_DIR
+
+    # with dump type
+    # <stdout | stderr | discard>
+    dumpDestination = ""    # DATABASE_DUMP_DST
+
+[cache]
+    database = 1024 # DB_CACHE_SIZE_MB
+    trie     = 1024 # TRIE_CACHE_SIZE_MB
+
+[prom]
+    # prometheus metrics
+    metrics  = true         # PROM_METRICS
+    http     = true         # PROM_HTTP
+    httpAddr = "localhost"  # PROM_HTTP_ADDR
+    httpPort = "8889"       # PROM_HTTP_PORT
+    dbStats = true          # PROM_DB_STATS
+
+[ethereum]
+    # node info
+    nodeID       = ""                       # ETH_NODE_ID
+    clientName   = "eth-statediff-service"  # ETH_CLIENT_NAME
+    networkID    = 1                        # ETH_NETWORK_ID
+    chainID      = 1                        # ETH_CHAIN_ID
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # ETH_GENESIS_BLOCK
+
+    # path to custom chain config file (optional)
+    # keep chainID same as that in chain config file
+    chainConfig  = "./chain.json"           # ETH_CHAIN_CONFIG
 ```
+
+### Local Setup
+
+* Create a chain config file `chain.json` according to chain config in genesis json file used by local geth.
+
+  Example:
+  ```json
+  {
+    "chainId": 41337,
+    "homesteadBlock": 0,
+    "eip150Block": 0,
+    "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+    "eip155Block": 0,
+    "eip158Block": 0,
+    "byzantiumBlock": 0,
+    "constantinopleBlock": 0,
+    "petersburgBlock": 0,
+    "istanbulBlock": 0,
+    "clique": {
+      "period": 5,
+      "epoch": 30000
+    }
+  }
+  ```
+
+  Provide the path to the above file in the config.
+
+## Usage
+
+* Create / update the config file (refer to example config above).
+
+### `serve`
+
+* To serve the statediff RPC API:
+
+    ```bash
+    ./eth-statediff-service serve --config=<config path>
+    ```
+
+    Example:
+
+    ```bash
+    ./eth-statediff-service serve --config environments/config.toml
+    ```
+
+* Available RPC methods:
+    * `statediff_stateTrieAt()`
+    * `statediff_streamCodeAndCodeHash()`
+    * `statediff_stateDiffAt()`
+    * `statediff_writeStateDiffAt()`
+    * `statediff_writeStateDiffsInRange()`
+
+    Example:
+
+    ```bash
+    curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"intermediateStateNodes":true,"intermediateStorageNodes":true,"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT"
+    ```
+
+* Prerun:
+    * The process can be configured locally with sets of ranges to process as a "prerun" to processing directed by the server endpoints.
+    * This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining ranges and params in the
+`prerun` section of the config.
+    * Set the range using `prerun.start` and `prerun.stop`. Use `prerun.ranges` if prerun on more than one range is required.
+
+* NOTE: Currently, `params.includeTD` must be set to / passed as `true`.
+
+## Monitoring
+
+* Enable metrics using config parameters `prom.metrics` and `prom.http`.
+* `eth-statediff-service` exposes following prometheus metrics at `/metrics` endpoint:
+    * `ranges_queued`: Number of range requests currently queued.
+    * `loaded_height`: The last block that was loaded for processing.
+    * `processed_height`: The last block that was processed.
+    * `stats.t_block_load`: Block loading time.
+    * `stats.t_block_processing`: Block (header, uncles, txs, rcts, tx trie, rct trie) processing time.
+    * `stats.t_state_processing`: State (state trie, storage tries, and code) processing time.
+    * `stats.t_postgres_tx_commit`: Postgres tx commit time.
+    * `http.count`: HTTP request count.
+    * `http.duration`: HTTP request duration.
+    * `ipc.count`: Unix socket connection count.
+
+## Tests
+
+* Run unit tests:
+
+    ```bash
+    make test
+    ```
+
+## Import output data in file mode into a database
+
+* When `eth-statediff-service` is run in file mode (`database.type`) the output is in form of a SQL file or multiple CSV files.
+
+* Assuming the output files are located in host's `./output_dir` directory.
+
+* Create a directory to store post-processed output:
+
+    ```bash
+    mkdir -p output_dir/processed_output
+    ```
+
+### SQL
+
+* De-duplicate data:
+
+    ```bash
+    sort -u output_dir/statediff.sql -o output_dir/processed_output/deduped-statediff.sql
+    ```
+
+* Copy over the post-processed output files to the DB server (say in `/output_dir`).
+
+* Run the following to import data:
+
+    ```bash
+    psql -U <DATABASE_USER> -h <DATABASE_HOSTNAME> -p <DATABASE_PORT> <DATABASE_NAME> --set ON_ERROR_STOP=on -f /output_dir/processed_output/deduped-statediff.sql
+    ```
+
+### CSV
+
+* De-duplicate data and copy to post-processed output directory:
+
+    ```bash
+    # public.blocks
+    sort -u output_dir/public.blocks.csv -o output_dir/processed_output/deduped-public.blocks.csv
+
+    # eth.header_cids
+    sort -u output_dir/eth.header_cids.csv -o output_dir/processed_output/deduped-eth.header_cids.csv
+
+    # eth.uncle_cids
+    sort -u output_dir/eth.uncle_cids.csv -o output_dir/processed_output/deduped-eth.uncle_cids.csv
+
+    # eth.transaction_cids
+    sort -u output_dir/eth.transaction_cids.csv -o output_dir/processed_output/deduped-eth.transaction_cids.csv
+
+    # eth.access_list_elements
+    sort -u output_dir/eth.access_list_elements.csv -o output_dir/processed_output/deduped-eth.access_list_elements.csv
+
+    # eth.receipt_cids
+    sort -u output_dir/eth.receipt_cids.csv -o output_dir/processed_output/deduped-eth.receipt_cids.csv
+
+    # eth.log_cids
+    sort -u output_dir/eth.log_cids.csv -o output_dir/processed_output/deduped-eth.log_cids.csv
+
+    # eth.state_cids
+    sort -u output_dir/eth.state_cids.csv -o output_dir/processed_output/deduped-eth.state_cids.csv
+
+    # eth.storage_cids
+    sort -u output_dir/eth.storage_cids.csv -o output_dir/processed_output/deduped-eth.storage_cids.csv
+
+    # eth.state_accounts
+    sort -u output_dir/eth.state_accounts.csv -o output_dir/processed_output/deduped-eth.state_accounts.csv
+
+    # public.nodes
+    cp output_dir/public.nodes.csv output_dir/processed_output/public.nodes.csv
+    ```
+
+* Copy over the post-processed output files to the DB server (say in `/output_dir`).
+
+* Start `psql` to run the import commands:
+
+    ```bash
+    psql -U <DATABASE_USER> -h <DATABASE_HOSTNAME> -p <DATABASE_PORT> <DATABASE_NAME>
+    ```
+
+* Run the following to import data:
+
+    ```bash
+    # public.nodes
+    COPY public.nodes FROM '/output_dir/processed_output/public.nodes.csv' CSV;
+
+    # public.nodes
+    COPY public.blocks FROM '/output_dir/processed_output/deduped-public.blocks.csv' CSV;
+
+    # eth.header_cids
+    COPY eth.header_cids FROM '/output_dir/processed_output/deduped-eth.header_cids.csv' CSV;
+
+    # eth.uncle_cids
+    COPY eth.uncle_cids FROM '/output_dir/processed_output/deduped-eth.uncle_cids.csv' CSV;
+
+    # eth.transaction_cids
+    COPY eth.transaction_cids FROM '/output_dir/processed_output/deduped-eth.transaction_cids.csv' CSV FORCE NOT NULL dst;
+
+    # eth.access_list_elements
+    COPY eth.access_list_elements FROM '/output_dir/processed_output/deduped-eth.access_list_elements.csv' CSV;
+
+    # eth.receipt_cids
+    COPY eth.receipt_cids FROM '/output_dir/processed_output/deduped-eth.receipt_cids.csv' CSV FORCE NOT NULL post_state, contract, contract_hash;
+
+    # eth.log_cids
+    COPY eth.log_cids FROM '/output_dir/processed_output/deduped-eth.log_cids.csv' CSV FORCE NOT NULL topic0, topic1, topic2, topic3;
+
+    # eth.state_cids
+    COPY eth.state_cids FROM '/output_dir/processed_output/deduped-eth.state_cids.csv' CSV FORCE NOT NULL state_leaf_key;
+
+    # eth.storage_cids
+    COPY eth.storage_cids FROM '/output_dir/processed_output/deduped-eth.storage_cids.csv' CSV FORCE NOT NULL storage_leaf_key;
+
+    # eth.state_accounts
+    COPY eth.state_accounts FROM '/output_dir/processed_output/deduped-eth.state_accounts.csv' CSV;
+    ```
+
+* NOTE: `COPY` command on CSVs inserts empty strings as `NULL` in the DB. Passing `FORCE_NOT_NULL <COLUMN_NAME>` forces it to insert empty strings instead. This is required to maintain compatibility of the imported statediff data with the data generated in `postgres` mode. Reference: https://www.postgresql.org/docs/14/sql-copy.html
@@ -18,8 +18,6 @@ package cmd

 import (
 	"github.com/spf13/viper"
-
-	pg "github.com/ethereum/go-ethereum/statediff/indexer/postgres"
 )

 const (
@@ -28,11 +26,14 @@ const (
 	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
 	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
 	ETH_CHAIN_ID      = "ETH_CHAIN_ID"
+	ETH_CHAIN_CONFIG  = "ETH_CHAIN_CONFIG"

 	DB_CACHE_SIZE_MB   = "DB_CACHE_SIZE_MB"
 	TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
+	LVLDB_MODE         = "LVLDB_MODE"
 	LVLDB_PATH         = "LVLDB_PATH"
 	LVLDB_ANCIENT      = "LVLDB_ANCIENT"
+	LVLDB_URL          = "LVLDB_URL"

 	STATEDIFF_PRERUN            = "STATEDIFF_PRERUN"
 	STATEDIFF_TRIE_WORKERS      = "STATEDIFF_TRIE_WORKERS"
@@ -48,12 +49,38 @@ const (
 	PROM_HTTP_PORT = "PROM_HTTP_PORT"
 	PROM_DB_STATS  = "PROM_DB_STATS"

-	PRERUN_ONLY        = "PRERUN_ONLY"
-	PRERUN_RANGE_START = "PRERUN_RANGE_START"
-	PRERUN_RANGE_STOP  = "PRERUN_RANGE_STOP"
+	PRERUN_ONLY                       = "PRERUN_ONLY"
+	PRERUN_RANGE_START                = "PRERUN_RANGE_START"
+	PRERUN_RANGE_STOP                 = "PRERUN_RANGE_STOP"
+	PRERUN_INTERMEDIATE_STATE_NODES   = "PRERUN_INTERMEDIATE_STATE_NODES"
+	PRERUN_INTERMEDIATE_STORAGE_NODES = "PRERUN_INTERMEDIATE_STORAGE_NODES"
+	PRERUN_INCLUDE_BLOCK              = "PRERUN_INCLUDE_BLOCK"
+	PRERUN_INCLUDE_RECEIPTS           = "PRERUN_INCLUDE_RECEIPTS"
+	PRERUN_INCLUDE_TD                 = "PRERUN_INCLUDE_TD"
+	PRERUN_INCLUDE_CODE               = "PRERUN_INCLUDE_CODE"

 	LOG_LEVEL     = "LOG_LEVEL"
 	LOG_FILE_PATH = "LOG_FILE_PATH"
+
+	DATABASE_NAME     = "DATABASE_NAME"
+	DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
+	DATABASE_PORT     = "DATABASE_PORT"
+	DATABASE_USER     = "DATABASE_USER"
+	DATABASE_PASSWORD = "DATABASE_PASSWORD"
+
+	DATABASE_TYPE         = "DATABASE_TYPE"
+	DATABASE_DRIVER_TYPE  = "DATABASE_DRIVER_TYPE"
+	DATABASE_DUMP_DST     = "DATABASE_DUMP_DST"
+	DATABASE_FILE_PATH    = "DATABASE_FILE_PATH"
+	DATABASE_FILE_MODE    = "DATABASE_FILE_MODE"
+	DATABASE_FILE_CSV_DIR = "DATABASE_FILE_CSV_DIR"
+
+	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
+	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
+	DATABASE_MIN_OPEN_CONNS       = "DATABASE_MIN_OPEN_CONNS"
+	DATABASE_MAX_CONN_LIFETIME    = "DATABASE_MAX_CONN_LIFETIME"
+	DATABASE_CONN_TIMEOUT         = "DATABSE_CONN_TIMEOUT"
+	DATABASE_MAX_CONN_IDLE_TIME   = "DATABASE_MAX_CONN_IDLE_TIME"
 )

 // Bind env vars for eth node and DB configuration
@@ -66,21 +93,35 @@ func init() {
 	viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
 	viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
 	viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
+	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)

-	viper.BindEnv("database.name", pg.DATABASE_NAME)
-	viper.BindEnv("database.hostname", pg.DATABASE_HOSTNAME)
-	viper.BindEnv("database.port", pg.DATABASE_PORT)
-	viper.BindEnv("database.user", pg.DATABASE_USER)
-	viper.BindEnv("database.password", pg.DATABASE_PASSWORD)
-	viper.BindEnv("database.maxIdle", pg.DATABASE_MAX_IDLE_CONNECTIONS)
-	viper.BindEnv("database.maxOpen", pg.DATABASE_MAX_OPEN_CONNECTIONS)
-	viper.BindEnv("database.maxLifetime", pg.DATABASE_MAX_CONN_LIFETIME)
+	viper.BindEnv("database.name", DATABASE_NAME)
+	viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
+	viper.BindEnv("database.port", DATABASE_PORT)
+	viper.BindEnv("database.user", DATABASE_USER)
+	viper.BindEnv("database.password", DATABASE_PASSWORD)
+
+	viper.BindEnv("database.maxIdle", DATABASE_MAX_IDLE_CONNECTIONS)
+	viper.BindEnv("database.maxOpen", DATABASE_MAX_OPEN_CONNECTIONS)
+	viper.BindEnv("database.minOpen", DATABASE_MIN_OPEN_CONNS)
+	viper.BindEnv("database.maxConnLifetime", DATABASE_MAX_CONN_LIFETIME)
+	viper.BindEnv("database.connTimeout", DATABASE_CONN_TIMEOUT)
+	viper.BindEnv("database.maxIdleTime", DATABASE_MAX_CONN_IDLE_TIME)
+
+	viper.BindEnv("database.type", DATABASE_TYPE)
+	viper.BindEnv("database.driver", DATABASE_DRIVER_TYPE)
+	viper.BindEnv("database.dumpDestination", DATABASE_DUMP_DST)
+	viper.BindEnv("database.fileMode", DATABASE_FILE_MODE)
+	viper.BindEnv("database.filePath", DATABASE_FILE_PATH)
+	viper.BindEnv("database.fileCsvDir", DATABASE_FILE_CSV_DIR)

 	viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
 	viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)

+	viper.BindEnv("leveldb.mode", LVLDB_MODE)
 	viper.BindEnv("leveldb.path", LVLDB_PATH)
 	viper.BindEnv("leveldb.ancient", LVLDB_ANCIENT)
+	viper.BindEnv("leveldb.url", LVLDB_URL)

 	viper.BindEnv("prom.metrics", PROM_METRICS)
 	viper.BindEnv("prom.http", PROM_HTTP)
@@ -96,6 +137,12 @@ func init() {
 	viper.BindEnv("prerun.only", PRERUN_ONLY)
 	viper.BindEnv("prerun.start", PRERUN_RANGE_START)
 	viper.BindEnv("prerun.stop", PRERUN_RANGE_STOP)
+	viper.BindEnv("prerun.params.intermediateStateNodes", PRERUN_INTERMEDIATE_STATE_NODES)
+	viper.BindEnv("prerun.params.intermediateStorageNodes", PRERUN_INTERMEDIATE_STORAGE_NODES)
+	viper.BindEnv("prerun.params.includeBlock", PRERUN_INCLUDE_BLOCK)
+	viper.BindEnv("prerun.params.includeReceipts", PRERUN_INCLUDE_RECEIPTS)
+	viper.BindEnv("prerun.params.includeTD", PRERUN_INCLUDE_TD)
+	viper.BindEnv("prerun.params.includeCode", PRERUN_INCLUDE_CODE)

 	viper.BindEnv("log.level", LOG_LEVEL)
 	viper.BindEnv("log.file", LOG_FILE_PATH)
@@ -23,13 +23,18 @@ import (
 	"strings"
 	"time"

+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/dump"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
 	"github.com/ethereum/go-ethereum/statediff/indexer/node"
-	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"

-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 var (
@@ -106,12 +111,15 @@ func init() {
 	rootCmd.PersistentFlags().String("http-path", "", "vdb server http path")
 	rootCmd.PersistentFlags().String("ipc-path", "", "vdb server ipc path")
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
+
 	rootCmd.PersistentFlags().String("log-file", "", "file path for logging")
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(),
 		"log level (trace, debug, info, warn, error, fatal, panic")

+	rootCmd.PersistentFlags().String("leveldb-mode", "local", "LevelDB access mode (local, remote)")
 	rootCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
 	rootCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
+	rootCmd.PersistentFlags().String("leveldb-url", "", "url to primary leveldb-ethdb-rpc server")

 	rootCmd.PersistentFlags().Bool("prerun", false, "turn on prerun of toml configured ranges")
 	rootCmd.PersistentFlags().Int("service-workers", 0, "number of range requests to process concurrently")
@@ -123,9 +131,21 @@ func init() {
 	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
 	rootCmd.PersistentFlags().String("database-user", "", "database user")
 	rootCmd.PersistentFlags().String("database-password", "", "database password")
+	rootCmd.PersistentFlags().Int("database-max-idle", 0, "database max number of idle connections")
+	rootCmd.PersistentFlags().Int("database-max-open", 0, "database max number of open connections")
+	rootCmd.PersistentFlags().Int("database-min-open", 0, "database min number of open connections")
+	rootCmd.PersistentFlags().Duration("database-max-conn-lifetime", 0, "database max connection lifetime")
+	rootCmd.PersistentFlags().Duration("database-conn-timeout", 0, "database connection timeout")
+	rootCmd.PersistentFlags().Duration("database-max-idle-time", 0, "database max connection idle time")
+	rootCmd.PersistentFlags().String("database-type", "postgres", "database type (currently supported: postgres, dump)")
+	rootCmd.PersistentFlags().String("database-driver", "sqlx", "database driver type (currently supported: sqlx, pgx)")
+	rootCmd.PersistentFlags().String("database-dump-dst", "stdout", "dump destination (for database-type=dump; options: stdout, stderr, discard)")
+	rootCmd.PersistentFlags().String("database-file-mode", "csv", "mode for writing file (for database-type=file; options: csv, sql)")
+	rootCmd.PersistentFlags().String("database-file-csv-dir", "", "full directory path (for database-file-mode=csv)")
+	rootCmd.PersistentFlags().String("database-file-path", "", "full file path (for database-file-mode=sql)")

 	rootCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
-	rootCmd.PersistentFlags().String("eth-client-name", "Geth", "eth client name")
+	rootCmd.PersistentFlags().String("eth-client-name", "eth-statediff-service", "eth client name")
 	rootCmd.PersistentFlags().String("eth-genesis-block",
 		"0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", "eth genesis block hash")
 	rootCmd.PersistentFlags().String("eth-network-id", "1", "eth network id")
@@ -143,37 +163,72 @@ func init() {
 	rootCmd.PersistentFlags().Bool("prerun-only", false, "only process pre-configured ranges; exit afterwards")
 	rootCmd.PersistentFlags().Int("prerun-start", 0, "start height for a prerun range")
 	rootCmd.PersistentFlags().Int("prerun-stop", 0, "stop height for a prerun range")
+	rootCmd.PersistentFlags().Bool("prerun-intermediate-state-nodes", true, "include intermediate state nodes in state diff")
+	rootCmd.PersistentFlags().Bool("prerun-intermediate-storage-nodes", true, "include intermediate storage nodes in state diff")
+	rootCmd.PersistentFlags().Bool("prerun-include-block", true, "include block data in the statediff payload")
+	rootCmd.PersistentFlags().Bool("prerun-include-receipts", true, "include receipts in the statediff payload")
+	rootCmd.PersistentFlags().Bool("prerun-include-td", true, "include td in the statediff payload")
+	rootCmd.PersistentFlags().Bool("prerun-include-code", true, "include code and codehash mappings in statediff payload")

 	viper.BindPFlag("server.httpPath", rootCmd.PersistentFlags().Lookup("http-path"))
 	viper.BindPFlag("server.ipcPath", rootCmd.PersistentFlags().Lookup("ipc-path"))
+
 	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
+
 	viper.BindPFlag("statediff.prerun", rootCmd.PersistentFlags().Lookup("prerun"))
 	viper.BindPFlag("statediff.serviceWorkers", rootCmd.PersistentFlags().Lookup("service-workers"))
 	viper.BindPFlag("statediff.trieWorkers", rootCmd.PersistentFlags().Lookup("trie-workers"))
 	viper.BindPFlag("statediff.workerQueueSize", rootCmd.PersistentFlags().Lookup("worker-queue-size"))
+
+	viper.BindPFlag("leveldb.mode", rootCmd.PersistentFlags().Lookup("leveldb-mode"))
 	viper.BindPFlag("leveldb.path", rootCmd.PersistentFlags().Lookup("leveldb-path"))
 	viper.BindPFlag("leveldb.ancient", rootCmd.PersistentFlags().Lookup("ancient-path"))
+	viper.BindPFlag("leveldb.url", rootCmd.PersistentFlags().Lookup("leveldb-url"))
+
 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
 	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
 	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
 	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
+	viper.BindPFlag("database.maxIdle", rootCmd.PersistentFlags().Lookup("database-max-idle"))
+	viper.BindPFlag("database.maxOpen", rootCmd.PersistentFlags().Lookup("database-max-open"))
+	viper.BindPFlag("database.minOpen", rootCmd.PersistentFlags().Lookup("database-min-open"))
+	viper.BindPFlag("database.maxConnLifetime", rootCmd.PersistentFlags().Lookup("database-max-conn-lifetime"))
+	viper.BindPFlag("database.connTimeout", rootCmd.PersistentFlags().Lookup("database-conn-timeout"))
+	viper.BindPFlag("database.maxIdleTime", rootCmd.PersistentFlags().Lookup("database-max-idle-time"))
+	viper.BindPFlag("database.type", rootCmd.PersistentFlags().Lookup("database-type"))
+	viper.BindPFlag("database.driver", rootCmd.PersistentFlags().Lookup("database-driver"))
+	viper.BindPFlag("database.dumpDestination", rootCmd.PersistentFlags().Lookup("database-dump-dst"))
+	viper.BindPFlag("database.fileMode", rootCmd.PersistentFlags().Lookup("database-file-mode"))
+	viper.BindPFlag("database.fileCsvDir", rootCmd.PersistentFlags().Lookup("database-file-csv-dir"))
+	viper.BindPFlag("database.filePath", rootCmd.PersistentFlags().Lookup("database-file-path"))
+
 	viper.BindPFlag("ethereum.nodeID", rootCmd.PersistentFlags().Lookup("eth-node-id"))
 	viper.BindPFlag("ethereum.clientName", rootCmd.PersistentFlags().Lookup("eth-client-name"))
 	viper.BindPFlag("ethereum.genesisBlock", rootCmd.PersistentFlags().Lookup("eth-genesis-block"))
 	viper.BindPFlag("ethereum.networkID", rootCmd.PersistentFlags().Lookup("eth-network-id"))
 	viper.BindPFlag("ethereum.chainID", rootCmd.PersistentFlags().Lookup("eth-chain-id"))
+	viper.BindPFlag("ethereum.chainConfig", rootCmd.PersistentFlags().Lookup("eth-chain-config"))
+
 	viper.BindPFlag("cache.database", rootCmd.PersistentFlags().Lookup("cache-db"))
 	viper.BindPFlag("cache.trie", rootCmd.PersistentFlags().Lookup("cache-trie"))
+
 	viper.BindPFlag("prom.http", rootCmd.PersistentFlags().Lookup("prom-http"))
 	viper.BindPFlag("prom.httpAddr", rootCmd.PersistentFlags().Lookup("prom-http-addr"))
 	viper.BindPFlag("prom.httpPort", rootCmd.PersistentFlags().Lookup("prom-http-port"))
 	viper.BindPFlag("prom.dbStats", rootCmd.PersistentFlags().Lookup("prom-db-stats"))
 	viper.BindPFlag("prom.metrics", rootCmd.PersistentFlags().Lookup("prom-metrics"))
+
 	viper.BindPFlag("prerun.only", rootCmd.PersistentFlags().Lookup("prerun-only"))
 	viper.BindPFlag("prerun.start", rootCmd.PersistentFlags().Lookup("prerun-start"))
 	viper.BindPFlag("prerun.stop", rootCmd.PersistentFlags().Lookup("prerun-stop"))
+	viper.BindPFlag("prerun.params.intermediateStateNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-state-nodes"))
+	viper.BindPFlag("prerun.params.intermediateStorageNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-storage-nodes"))
+	viper.BindPFlag("prerun.params.includeBlock", rootCmd.PersistentFlags().Lookup("prerun-include-block"))
+	viper.BindPFlag("prerun.params.includeReceipts", rootCmd.PersistentFlags().Lookup("prerun-include-receipts"))
+	viper.BindPFlag("prerun.params.includeTD", rootCmd.PersistentFlags().Lookup("prerun-include-td"))
+	viper.BindPFlag("prerun.params.includeCode", rootCmd.PersistentFlags().Lookup("prerun-include-code"))

 	rand.Seed(time.Now().UnixNano())
 }
@@ -191,7 +246,7 @@ func initConfig() {
 	}
 }

-func GetEthNodeInfo() node.Info {
+func getEthNodeInfo() node.Info {
 	var nodeID, genesisBlock, networkID, clientName string
 	var chainID uint64
 	if !viper.IsSet("ethereum.nodeID") {
@@ -199,6 +254,7 @@ func GetEthNodeInfo() node.Info {
 	} else {
 		nodeID = viper.GetString("ethereum.nodeID")
 	}
+	genesisBlock = viper.GetString("ethereum.genesisBlock")
 	if !viper.IsSet("ethereum.genesisBlock") {
 		genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
 	} else {
@@ -238,20 +294,95 @@ func randSeq(n int) string {
 	return string(b)
 }

-func GetDBParams() postgres.ConnectionParams {
-	return postgres.ConnectionParams{
-		Name:     viper.GetString("database.name"),
-		Hostname: viper.GetString("database.hostname"),
-		Port:     viper.GetInt("database.port"),
-		User:     viper.GetString("database.user"),
-		Password: viper.GetString("database.password"),
+// getConfig constructs and returns the appropriate config from viper params
+func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
+	dbTypeStr := viper.GetString("database.type")
+	dbType, err := shared.ResolveDBType(dbTypeStr)
+	if err != nil {
+		return nil, err
 	}
-}
+	logWithCommand.Infof("configuring service for database type: %s", dbType)
+	var indexerConfig interfaces.Config
+	switch dbType {
+	case shared.FILE:
+		logWithCommand.Info("starting in sql file writing mode")

-func GetDBConfig() postgres.ConnectionConfig {
-	return postgres.ConnectionConfig{
-		MaxIdle:     viper.GetInt("database.maxIdle"),
-		MaxOpen:     viper.GetInt("database.maxOpen"),
-		MaxLifetime: viper.GetInt("database.maxLifetime"),
+		fileModeStr := viper.GetString("database.fileMode")
+		fileMode, err := file.ResolveFileMode(fileModeStr)
+		if err != nil {
+			utils.Fatalf("%v", err)
+		}
+
+		filePathStr := viper.GetString("database.filePath")
+		if fileMode == file.SQL && filePathStr == "" {
+			logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided")
+		}
+
+		fileCsvDirStr := viper.GetString("database.fileCsvDir")
+		if fileMode == file.CSV && fileCsvDirStr == "" {
+			logWithCommand.Fatal("when operating in csv file writing mode a directory path must be provided")
+		}
+
+		indexerConfig = file.Config{
+			Mode:      fileMode,
+			OutputDir: fileCsvDirStr,
+			FilePath:  filePathStr,
+		}
+	case shared.DUMP:
+		logWithCommand.Info("starting in data dump mode")
+		dumpDstStr := viper.GetString("database.dumpDestination")
+		dumpDst, err := dump.ResolveDumpType(dumpDstStr)
+		if err != nil {
+			return nil, err
+		}
+		switch dumpDst {
+		case dump.STDERR:
+			indexerConfig = dump.Config{Dump: os.Stdout}
+		case dump.STDOUT:
+			indexerConfig = dump.Config{Dump: os.Stderr}
+		case dump.DISCARD:
+			indexerConfig = dump.Config{Dump: dump.NewDiscardWriterCloser()}
+		default:
+			return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst)
+		}
+	case shared.POSTGRES:
+		logWithCommand.Info("starting in postgres mode")
+		driverTypeStr := viper.GetString("database.driver")
+		driverType, err := postgres.ResolveDriverType(driverTypeStr)
+		if err != nil {
+			utils.Fatalf("%v", err)
+		}
+		pgConfig := postgres.Config{
+			Hostname:     viper.GetString("database.hostname"),
+			Port:         viper.GetInt("database.port"),
+			DatabaseName: viper.GetString("database.name"),
+			Username:     viper.GetString("database.user"),
+			Password:     viper.GetString("database.password"),
+			ID:           nodeInfo.ID,
+			ClientName:   nodeInfo.ClientName,
+			Driver:       driverType,
+		}
+		if viper.IsSet("database.maxIdle") {
+			pgConfig.MaxIdle = viper.GetInt("database.maxIdle")
+		}
+		if viper.IsSet("database.maxOpen") {
+			pgConfig.MaxConns = viper.GetInt("database.maxOpen")
+		}
+		if viper.IsSet("database.minOpen") {
+			pgConfig.MinConns = viper.GetInt("database.minOpen")
+		}
+		if viper.IsSet("database.maxConnLifetime") {
+			pgConfig.MaxConnLifetime = viper.GetDuration("database.maxConnLifetime")
+		}
+		if viper.IsSet("database.connTimeout") {
+			pgConfig.ConnTimeout = viper.GetDuration("database.connTimeout")
+		}
+		if viper.IsSet("database.maxIdleTime") {
+			pgConfig.MaxConnIdleTime = viper.GetDuration("database.maxIdleTime")
+		}
+		indexerConfig = pgConfig
+	default:
+		return nil, fmt.Errorf("unrecognized db type: %s", dbType)
 	}
+	return indexerConfig, nil
 }
@@ -25,14 +25,14 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"

-	sd "github.com/vulcanize/eth-statediff-service/pkg"
-	srpc "github.com/vulcanize/eth-statediff-service/pkg/rpc"
+	sd "github.com/cerc-io/eth-statediff-service/pkg"
+	srpc "github.com/cerc-io/eth-statediff-service/pkg/rpc"
 )

 // serveCmd represents the serve command
 var serveCmd = &cobra.Command{
 	Use:   "serve",
-	Short: "Stand up a standalone statediffing RPC service on top of leveldb",
+	Short: "Stand up a standalone statediffing RPC service on top of LevelDB",
 	Long: `Usage

 ./eth-statediff-service serve --config={path to toml config file}`,
@@ -1,19 +1,18 @@
 package cmd

 import (
-	"fmt"
+	"context"

 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/params"
-	gethsd "github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff"
 	ind "github.com/ethereum/go-ethereum/statediff/indexer"
-	"github.com/ethereum/go-ethereum/statediff/indexer/node"
-	"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/spf13/viper"

-	sd "github.com/vulcanize/eth-statediff-service/pkg"
-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	sd "github.com/cerc-io/eth-statediff-service/pkg"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 type blockRange [2]uint64
@@ -21,20 +20,41 @@ type blockRange [2]uint64
 func createStateDiffService() (sd.StateDiffService, error) {
 	// load some necessary params
 	logWithCommand.Info("Loading statediff service parameters")
+	mode := viper.GetString("leveldb.mode")
 	path := viper.GetString("leveldb.path")
 	ancientPath := viper.GetString("leveldb.ancient")
-	if path == "" || ancientPath == "" {
-		logWithCommand.Fatal("require a valid eth leveldb primary datastore path and ancient datastore path")
+	url := viper.GetString("leveldb.url")
+
+	if mode == "local" {
+		if path == "" || ancientPath == "" {
+			logWithCommand.Fatal("Require a valid eth LevelDB primary datastore path and ancient datastore path")
+		}
+	} else if mode == "remote" {
+		if url == "" {
+			logWithCommand.Fatal("Require a valid RPC url for accessing LevelDB")
+		}
+	} else {
+		logWithCommand.Fatal("Invalid mode provided for LevelDB access")
+	}
+
+	nodeInfo := getEthNodeInfo()
+
+	var chainConf *params.ChainConfig
+	var err error
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+
+	if chainConfigPath != "" {
+		chainConf, err = statediff.LoadConfig(chainConfigPath)
+	} else {
+		chainConf, err = statediff.ChainConfig(nodeInfo.ChainID)
 	}

-	nodeInfo := GetEthNodeInfo()
-	chainConf, err := chainConfig(nodeInfo.ChainID)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}

-	// create leveldb reader
-	logWithCommand.Info("Creating leveldb reader")
+	// create LevelDB reader
+	logWithCommand.Info("Creating LevelDB reader")
 	readerConf := sd.LvLDBReaderConfig{
 		TrieConfig: &trie.Config{
 			Cache:     viper.GetInt("cache.trie"),
@@ -42,8 +62,10 @@ func createStateDiffService() (sd.StateDiffService, error) {
 			Preimages: false,
 		},
 		ChainConfig: chainConf,
+		Mode:        mode,
 		Path:        path,
 		AncientPath: ancientPath,
+		Url:         url,
 		DBCacheSize: viper.GetInt("cache.database"),
 	}
 	lvlDBReader, err := sd.NewLvlDBReader(readerConf)
@@ -52,16 +74,21 @@ func createStateDiffService() (sd.StateDiffService, error) {
 	}

 	// create statediff service
-	logWithCommand.Info("Setting up Postgres DB")
-	db, err := setupPostgres(nodeInfo)
+	logWithCommand.Info("Setting up database")
+	conf, err := getConfig(nodeInfo)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
+
 	logWithCommand.Info("Creating statediff indexer")
-	indexer, err := ind.NewStateDiffIndexer(chainConf, db)
+	db, indexer, err := ind.NewStateDiffIndexer(context.Background(), chainConf, nodeInfo, conf)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
+	if conf.Type() == shared.POSTGRES && viper.GetBool("prom.dbStats") {
+		prom.RegisterDBCollector(viper.GetString("database.name"), db)
+	}
+
 	logWithCommand.Info("Creating statediff service")
 	sdConf := sd.Config{
 		ServiceWorkers:  viper.GetUint("statediff.serviceWorkers"),
@@ -72,25 +99,11 @@ func createStateDiffService() (sd.StateDiffService, error) {
 	return sd.NewStateDiffService(lvlDBReader, indexer, sdConf)
 }

-func setupPostgres(nodeInfo node.Info) (*postgres.DB, error) {
-	p := GetDBParams()
-	logWithCommand.Info("initializing DB connection pool")
-	db, err := postgres.NewDB(postgres.DbConnectionString(p), GetDBConfig(), nodeInfo)
-	if err != nil {
-		return nil, err
-	}
-	if viper.GetBool("prom.dbStats") {
-		logWithCommand.Info("registering DB collector")
-		prom.RegisterDBCollector(p.Name, db.DB)
-	}
-	return db, nil
-}
-
 func setupPreRunRanges() []sd.RangeRequest {
 	if !viper.GetBool("statediff.prerun") {
 		return nil
 	}
-	preRunParams := gethsd.Params{
+	preRunParams := statediff.Params{
 		IntermediateStateNodes:   viper.GetBool("prerun.params.intermediateStateNodes"),
 		IntermediateStorageNodes: viper.GetBool("prerun.params.intermediateStorageNodes"),
 		IncludeBlock:             viper.GetBool("prerun.params.includeBlock"),
@@ -105,13 +118,6 @@ func setupPreRunRanges() []sd.RangeRequest {
 		addrs[i] = common.HexToAddress(addrStr)
 	}
 	preRunParams.WatchedAddresses = addrs
-	var storageKeyStrs []string
-	viper.UnmarshalKey("prerun.params.watchedStorageKeys", &storageKeyStrs)
-	keys := make([]common.Hash, len(storageKeyStrs))
-	for i, keyStr := range storageKeyStrs {
-		keys[i] = common.HexToHash(keyStr)
-	}
-	preRunParams.WatchedStorageSlots = keys
 	var rawRanges []blockRange
 	viper.UnmarshalKey("prerun.ranges", &rawRanges)
 	blockRanges := make([]sd.RangeRequest, len(rawRanges))
@@ -134,18 +140,3 @@ func setupPreRunRanges() []sd.RangeRequest {

 	return blockRanges
 }
-
-func chainConfig(chainID uint64) (*params.ChainConfig, error) {
-	switch chainID {
-	case 1:
-		return params.MainnetChainConfig, nil
-	case 3:
-		return params.RopstenChainConfig, nil // Ropsten
-	case 4:
-		return params.RinkebyChainConfig, nil
-	case 5:
-		return params.GoerliChainConfig, nil
-	default:
-		return nil, fmt.Errorf("chain config for chainid %d not available", chainID)
-	}
-}
@@ -0,0 +1,42 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	v "github.com/cerc-io/eth-statediff-service/version"
+)
+
+// versionCmd represents the version command
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Prints the version of eth-statediff-service",
+	Long: `Use this command to fetch the version of eth-statediff-service
+
+Usage: ./eth-statediff-service  version`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *log.WithField("SubCommand", subCommand)
+		logWithCommand.Infof("eth-statediff-service version: %s", v.Version)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(versionCmd)
+}
@@ -6,7 +6,7 @@ services:
      context: ./
      cache_from:
      - alpine:latest
-      - golang:1.13
+      - golang:1.16
      dockerfile: ./Dockerfile
      args:
        USER: "vdbm"
@@ -1,4 +1,5 @@
 [leveldb]
+    mode = "local"
    path = "/app/geth-rw/chaindata"
    ancient = "/app/geth-rw/chaindata/ancient"

@@ -23,29 +24,35 @@
        includeTD = true
        includeCode = true
        watchedAddresses = []
-        watchedStorageKeys = []

 [log]
    file = ""
    level = "info"

-[eth]
-    chainID = 1
-
 [database]
+    type = "postgres"
    name     = ""
    hostname = ""
    port     = 5432
    user     = ""
    password = ""
+    driver = "sqlx"

 [cache]
    database = 1024
    trie = 4096

 [prom]
-    dbStats = false
    metrics = true
    http = true
    httpAddr = "0.0.0.0"
    httpPort = 9100
+    dbStats = false
+
+[ethereum]
+    nodeID = ""
+    clientName = "eth-statediff-service"
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
+    networkID = 1
+    chainID = 1
+    chainConfig = ""
@@ -1,6 +1,8 @@
 [leveldb]
+    mode = "local"
    path = "/Users/user/Library/Ethereum/geth/chaindata"
    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
+    url = "http://127.0.0.1:8082/"

 [server]
    ipcPath = ".ipc"
@@ -25,21 +27,21 @@
        includeTD = true
        includeCode = true
        watchedAddresses = []
-        watchedStorageKeys = []

 [log]
    file = ""
    level = "info"

-[eth]
-    chainID = 1
-
 [database]
    name     = "vulcanize_test"
    hostname = "localhost"
    port     = 5432
    user     = "vulcanize"
    password = "..."
+    type = "postgres"
+    driver = "sqlx"
+    dumpDestination = ""
+    filePath = ""

 [cache]
    database = 1024
@@ -51,3 +53,11 @@
    http = true
    httpAddr = "localhost"
    httpPort = "8889"
+
+[ethereum]
+    chainConfig = ""
+    nodeID = ""
+    clientName = "eth-statediff-service"
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
+    networkID = 1
+    chainID = 1
@@ -1,15 +1,138 @@
-module github.com/vulcanize/eth-statediff-service
+module github.com/cerc-io/eth-statediff-service

-go 1.13
+go 1.18

 require (
-	github.com/ethereum/go-ethereum v1.10.13
-	github.com/jmoiron/sqlx v1.2.0
-	github.com/prometheus/client_golang v1.0.0
-	github.com/sirupsen/logrus v1.7.0
-	github.com/spf13/cobra v1.1.1
-	github.com/spf13/viper v1.7.1
-	github.com/vulcanize/go-eth-state-node-iterator v0.0.1-alpha.0.20211014064906-d23d01ed8191
+	github.com/cerc-io/go-eth-state-node-iterator v1.1.7
+	github.com/ethereum/go-ethereum v1.10.23
+	github.com/jmoiron/sqlx v1.2.0 // indirect
+	github.com/prometheus/client_golang v1.4.0
+	github.com/sirupsen/logrus v1.9.0
+	github.com/spf13/cobra v1.3.0
+	github.com/spf13/viper v1.10.1
+	github.com/vulcanize/leveldb-ethdb-rpc v0.1.7
 )

-replace github.com/ethereum/go-ethereum v1.10.13 => github.com/vulcanize/go-ethereum v1.10.13-statediff-0.0.29
+require (
+	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deckarep/golang-set v1.8.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/deepmap/oapi-codegen v1.8.2 // indirect
+	github.com/edsrzf/mmap-go v1.0.0 // indirect
+	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
+	github.com/georgysavva/scany v0.2.9 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/graph-gophers/graphql-go v1.3.0 // indirect
+	github.com/hashicorp/go-bexpr v0.1.10 // indirect
+	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
+	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/huin/goupnp v1.0.3 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/influxdata/influxdb v1.8.3 // indirect
+	github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
+	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-block-format v0.0.3 // indirect
+	github.com/ipfs/go-cid v0.2.0 // indirect
+	github.com/ipfs/go-datastore v0.5.1 // indirect
+	github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
+	github.com/ipfs/go-ipld-format v0.4.0 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.1.3 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.10.0 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.1.1 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgtype v1.8.1 // indirect
+	github.com/jackc/pgx/v4 v4.13.0 // indirect
+	github.com/jackc/puddle v1.1.3 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/lib/pq v1.10.6 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/mitchellh/pointerstructure v1.2.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.0.3 // indirect
+	github.com/multiformats/go-base36 v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.0.3 // indirect
+	github.com/multiformats/go-multihash v0.1.0 // indirect
+	github.com/multiformats/go-varint v0.0.6 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
+	github.com/pganalyze/pg_query_go/v2 v2.1.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.9.1 // indirect
+	github.com/prometheus/procfs v0.0.8 // indirect
+	github.com/prometheus/tsdb v0.10.0 // indirect
+	github.com/rjeczalik/notify v0.9.1 // indirect
+	github.com/rs/cors v1.7.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.7.2 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
+	github.com/thoas/go-funk v0.9.2 // indirect
+	github.com/tklauser/go-sysconf v0.3.6 // indirect
+	github.com/tklauser/numcpus v0.2.2 // indirect
+	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
+	github.com/urfave/cli/v2 v2.10.2 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	go.uber.org/zap v1.17.0 // indirect
+	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
+	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/ini.v1 v1.66.2 // indirect
+	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	lukechampine.com/blake3 v1.1.6 // indirect
+)
+
+replace github.com/ethereum/go-ethereum v1.10.23 => github.com/cerc-io/go-ethereum v1.10.23-statediff-4.2.0-alpha
@@ -15,7 +15,7 @@

 package main

-import "github.com/vulcanize/eth-statediff-service/cmd"
+import "github.com/cerc-io/eth-statediff-service/cmd"

 func main() {
 	cmd.Execute()
@@ -28,7 +28,7 @@ const APIName = "statediff"
 const APIVersion = "0.0.1"

 // PublicStateDiffAPI provides an RPC interface
-// that can be used to fetch historical diffs from leveldb directly
+// that can be used to fetch historical diffs from LevelDB directly
 type PublicStateDiffAPI struct {
 	sds StateDiffService
 }
@@ -20,93 +20,25 @@
 package statediff

 import (
-	"bytes"
 	"fmt"
 	"math/bits"
 	"sync"

-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/rlp"
 	sd "github.com/ethereum/go-ethereum/statediff"
-	sdtrie "github.com/ethereum/go-ethereum/statediff/trie"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
-	"github.com/ethereum/go-ethereum/trie"
 	"github.com/sirupsen/logrus"

-	iter "github.com/vulcanize/go-eth-state-node-iterator"
+	iter "github.com/cerc-io/go-eth-state-node-iterator"
 )

-var (
-	nullHashBytes     = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
-	emptyNode, _      = rlp.EncodeToBytes([]byte{})
-	emptyContractRoot = crypto.Keccak256Hash(emptyNode)
-	nullCodeHash      = crypto.Keccak256Hash([]byte{}).Bytes()
-)
-
-// Builder interface exposes the method for building a state diff between two blocks
-type Builder interface {
-	BuildStateDiffObject(args sd.Args, params sd.Params) (sd.StateObject, error)
-	BuildStateTrieObject(current *types.Block) (sd.StateObject, error)
-	WriteStateDiffObject(args sd.StateRoots, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error
-}
-
 type builder struct {
-	stateCache state.Database
+	sd.StateDiffBuilder
 	numWorkers uint
 }

-type iterPair struct {
-	older, newer trie.NodeIterator
-}
-
-func resolveNode(it trie.NodeIterator, trieDB *trie.Database) (sdtypes.StateNode, []interface{}, error) {
-	nodePath := make([]byte, len(it.Path()))
-	copy(nodePath, it.Path())
-	node, err := trieDB.Node(it.Hash())
-	if err != nil {
-		return sdtypes.StateNode{}, nil, err
-	}
-	var nodeElements []interface{}
-	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
-		return sdtypes.StateNode{}, nil, err
-	}
-	ty, err := sdtrie.CheckKeyType(nodeElements)
-	if err != nil {
-		return sdtypes.StateNode{}, nil, err
-	}
-	return sdtypes.StateNode{
-		NodeType:  ty,
-		Path:      nodePath,
-		NodeValue: node,
-	}, nodeElements, nil
-}
-
-// convenience
-func stateNodeAppender(nodes *[]sdtypes.StateNode) sdtypes.StateNodeSink {
-	return func(node sdtypes.StateNode) error {
-		*nodes = append(*nodes, node)
-		return nil
-	}
-}
-func storageNodeAppender(nodes *[]sdtypes.StorageNode) sdtypes.StorageNodeSink {
-	return func(node sdtypes.StorageNode) error {
-		*nodes = append(*nodes, node)
-		return nil
-	}
-}
-func codeMappingAppender(data *[]sdtypes.CodeAndCodeHash) sdtypes.CodeSink {
-	return func(c sdtypes.CodeAndCodeHash) error {
-		*data = append(*data, c)
-		return nil
-	}
-}
-
 // NewBuilder is used to create a statediff builder
-func NewBuilder(stateCache state.Database, workers uint) (Builder, error) {
+func NewBuilder(stateCache state.Database, workers uint) (sd.Builder, error) {
 	if workers == 0 {
 		workers = 1
 	}
@@ -114,92 +46,24 @@ func NewBuilder(stateCache state.Database, workers uint) (Builder, error) {
 		return nil, fmt.Errorf("workers must be a power of 2")
 	}
 	return &builder{
-		stateCache: stateCache, // state cache is safe for concurrent reads
+		StateDiffBuilder: sd.StateDiffBuilder{
+			StateCache: stateCache,
+		},
 		numWorkers: workers,
 	}, nil
 }

-// BuildStateTrieObject builds a state trie object from the provided block
-func (sdb *builder) BuildStateTrieObject(current *types.Block) (sd.StateObject, error) {
-	currentTrie, err := sdb.stateCache.OpenTrie(current.Root())
-	if err != nil {
-		return sd.StateObject{}, fmt.Errorf("error creating trie for block %d: %v", current.Number(), err)
-	}
-	it := currentTrie.NodeIterator([]byte{})
-	stateNodes, codeAndCodeHashes, err := sdb.buildStateTrie(it)
-	if err != nil {
-		return sd.StateObject{}, fmt.Errorf("error collecting state nodes for block %d: %v", current.Number(), err)
-	}
-	return sd.StateObject{
-		BlockNumber:       current.Number(),
-		BlockHash:         current.Hash(),
-		Nodes:             stateNodes,
-		CodeAndCodeHashes: codeAndCodeHashes,
-	}, nil
-}
-
-func (sdb *builder) buildStateTrie(it trie.NodeIterator) ([]sdtypes.StateNode, []sdtypes.CodeAndCodeHash, error) {
-	stateNodes := make([]sdtypes.StateNode, 0)
-	codeAndCodeHashes := make([]sdtypes.CodeAndCodeHash, 0)
-	for it.Next(true) {
-		// skip value nodes
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return nil, nil, err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			var account types.StateAccount
-			if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
-				return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
-			}
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			node.LeafKey = leafKey
-			if !bytes.Equal(account.CodeHash, nullCodeHash) {
-				var storageNodes []sdtypes.StorageNode
-				err := sdb.buildStorageNodesEventual(account.Root, nil, true, storageNodeAppender(&storageNodes))
-				if err != nil {
-					return nil, nil, fmt.Errorf("failed building eventual storage diffs for account %+v\r\nerror: %v", account, err)
-				}
-				node.StorageNodes = storageNodes
-				// emit codehash => code mappings for cod
-				codeHash := common.BytesToHash(account.CodeHash)
-				code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash)
-				if err != nil {
-					return nil, nil, fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
-				}
-				codeAndCodeHashes = append(codeAndCodeHashes, sdtypes.CodeAndCodeHash{
-					Hash: codeHash,
-					Code: code,
-				})
-			}
-			stateNodes = append(stateNodes, node)
-		case sdtypes.Extension, sdtypes.Branch:
-			stateNodes = append(stateNodes, node)
-		default:
-			return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-	}
-	return stateNodes, codeAndCodeHashes, it.Error()
-}
-
 // BuildStateDiffObject builds a statediff object from two blocks and the provided parameters
-func (sdb *builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sd.StateObject, error) {
+func (sdb *builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sdtypes.StateObject, error) {
 	var stateNodes []sdtypes.StateNode
 	var codeAndCodeHashes []sdtypes.CodeAndCodeHash
 	err := sdb.WriteStateDiffObject(
-		sd.StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot},
-		params, stateNodeAppender(&stateNodes), codeMappingAppender(&codeAndCodeHashes))
+		sdtypes.StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot},
+		params, sd.StateNodeAppender(&stateNodes), sd.CodeMappingAppender(&codeAndCodeHashes))
 	if err != nil {
-		return sd.StateObject{}, err
+		return sdtypes.StateObject{}, err
 	}
-	return sd.StateObject{
+	return sdtypes.StateObject{
 		BlockHash:         args.BlockHash,
 		BlockNumber:       args.BlockNumber,
 		Nodes:             stateNodes,
@@ -208,35 +72,31 @@ func (sdb *builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sd.Sta
 }

 // WriteStateDiffObject writes a statediff object to output callback
-func (sdb *builder) WriteStateDiffObject(args sd.StateRoots, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
-	if len(params.WatchedAddresses) > 0 {
-		// if we are watching only specific accounts then we are only diffing leaf nodes
-		log.Info("Ignoring intermediate state nodes because WatchedAddresses was passed")
-		params.IntermediateStateNodes = false
-	}
-
+func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
 	// Load tries for old and new states
-	oldTrie, err := sdb.stateCache.OpenTrie(args.OldStateRoot)
+	oldTrie, err := sdb.StateCache.OpenTrie(args.OldStateRoot)
 	if err != nil {
-		return fmt.Errorf("error creating trie for old state root: %v", err)
+		return fmt.Errorf("error creating trie for oldStateRoot: %v", err)
 	}
-	newTrie, err := sdb.stateCache.OpenTrie(args.NewStateRoot)
+	newTrie, err := sdb.StateCache.OpenTrie(args.NewStateRoot)
 	if err != nil {
-		return fmt.Errorf("error creating trie for new state root: %v", err)
+		return fmt.Errorf("error creating trie for newStateRoot: %v", err)
 	}

 	// Split old and new tries into corresponding subtrie iterators
-	oldIterFac := iter.NewSubtrieIteratorFactory(oldTrie, sdb.numWorkers)
-	newIterFac := iter.NewSubtrieIteratorFactory(newTrie, sdb.numWorkers)
+	oldIters1 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
+	oldIters2 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
+	newIters1 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
+	newIters2 := iter.SubtrieIterators(newTrie, sdb.numWorkers)

 	// Create iterators ahead of time to avoid race condition in state.Trie access
 	// We do two state iterations per subtrie: one for new/updated nodes,
 	// one for deleted/updated nodes; prepare 2 iterator instances for each task
-	var iterPairs [][]iterPair
+	var iterPairs [][]sd.IterPair
 	for i := uint(0); i < sdb.numWorkers; i++ {
-		iterPairs = append(iterPairs, []iterPair{
-			{older: oldIterFac.IteratorAt(i), newer: newIterFac.IteratorAt(i)},
-			{older: oldIterFac.IteratorAt(i), newer: newIterFac.IteratorAt(i)},
+		iterPairs = append(iterPairs, []sd.IterPair{
+			{Older: oldIters1[i], Newer: newIters1[i]},
+			{Older: oldIters2[i], Newer: newIters2[i]},
 		})
 	}

@@ -253,8 +113,15 @@ func (sdb *builder) WriteStateDiffObject(args sd.StateRoots, params sd.Params, o
 			wg.Add(1)
 			go func(worker uint) {
 				defer wg.Done()
-				if err := sdb.buildStateDiff(iterPairs[worker], params, nodeSender, codeSender); err != nil {
-					logrus.Errorf("buildStateDiff error for worker %d, pparams %+v", worker, params)
+				var err error
+				if !params.IntermediateStateNodes {
+					err = sdb.BuildStateDiffWithoutIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender)
+				} else {
+					err = sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender)
+				}
+
+				if err != nil {
+					logrus.Errorf("buildStateDiff error for worker %d, params %+v", worker, params)
 				}
 			}(w)
 		}
@@ -286,501 +153,3 @@ func (sdb *builder) WriteStateDiffObject(args sd.StateRoots, params sd.Params, o

 	return nil
 }
-
-func (sdb *builder) buildStateDiff(args []iterPair, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
-	// collect a slice of all the intermediate nodes that were touched and exist at B
-	// a map of their leafkey to all the accounts that were touched and exist at B
-	// and a slice of all the paths for the nodes in both of the above sets
-	var diffAccountsAtB AccountMap
-	var diffPathsAtB map[string]bool
-	var err error
-	if params.IntermediateStateNodes {
-		diffAccountsAtB, diffPathsAtB, err = sdb.createdAndUpdatedStateWithIntermediateNodes(args[0], output)
-	} else {
-		diffAccountsAtB, diffPathsAtB, err = sdb.createdAndUpdatedState(args[0], params.WatchedAddresses)
-	}
-
-	if err != nil {
-		return fmt.Errorf("error collecting createdAndUpdatedNodes: %v", err)
-	}
-
-	// collect a slice of all the nodes that existed at a path in A that doesn't exist in B
-	// a map of their leafkey to all the accounts that were touched and exist at A
-	diffAccountsAtA, err := sdb.deletedOrUpdatedState(args[1], diffPathsAtB, output)
-	if err != nil {
-		return fmt.Errorf("error collecting deletedOrUpdatedNodes: %v", err)
-	}
-
-	// collect and sort the leafkey keys for both account mappings into a slice
-	createKeys := sortKeys(diffAccountsAtB)
-	deleteKeys := sortKeys(diffAccountsAtA)
-
-	// and then find the intersection of these keys
-	// these are the leafkeys for the accounts which exist at both A and B but are different
-	// this also mutates the passed in createKeys and deleteKeys, removing the intersection keys
-	// and leaving the truly created or deleted keys in place
-	updatedKeys := findIntersection(createKeys, deleteKeys)
-
-	// build the diff nodes for the updated accounts using the mappings at both A and B as directed by the keys found as the intersection of the two
-	err = sdb.buildAccountUpdates(
-		diffAccountsAtB, diffAccountsAtA, updatedKeys,
-		params.WatchedStorageSlots, params.IntermediateStorageNodes, output)
-	if err != nil {
-		return fmt.Errorf("error building diff for updated accounts: %v", err)
-	}
-	// build the diff nodes for created accounts
-	err = sdb.buildAccountCreations(diffAccountsAtB, params.WatchedStorageSlots, params.IntermediateStorageNodes, output, codeOutput)
-	if err != nil {
-		return fmt.Errorf("error building diff for created accounts: %v", err)
-	}
-	return nil
-}
-
-// createdAndUpdatedState returns
-// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
-// and a slice of the paths for all of the nodes included in both
-func (sdb *builder) createdAndUpdatedState(iters iterPair, watchedAddresses []common.Address) (AccountMap, map[string]bool, error) {
-	diffPathsAtB := make(map[string]bool)
-	diffAcountsAtB := make(AccountMap)
-	it, _ := trie.NewDifferenceIterator(iters.older, iters.newer)
-	for it.Next(true) {
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return nil, nil, err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			// created vs updated is important for leaf nodes since we need to diff their storage
-			// so we need to map all changed accounts at B to their leafkey, since account can change pathes but not leafkey
-			var account types.StateAccount
-			if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
-				return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
-			}
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			if isWatchedAddress(watchedAddresses, leafKey) {
-				diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
-					NodeType:  node.NodeType,
-					Path:      node.Path,
-					NodeValue: node.NodeValue,
-					LeafKey:   leafKey,
-					Account:   &account,
-				}
-			}
-		}
-		// add both intermediate and leaf node paths to the list of diffPathsAtB
-		diffPathsAtB[common.Bytes2Hex(node.Path)] = true
-	}
-	return diffAcountsAtB, diffPathsAtB, it.Error()
-}
-
-// createdAndUpdatedStateWithIntermediateNodes returns
-// a slice of all the intermediate nodes that exist in a different state at B than A
-// a mapping of their leafkeys to all the accounts that exist in a different state at B than A
-// and a slice of the paths for all of the nodes included in both
-func (sdb *builder) createdAndUpdatedStateWithIntermediateNodes(iters iterPair, output sdtypes.StateNodeSink) (AccountMap, map[string]bool, error) {
-	diffPathsAtB := make(map[string]bool)
-	diffAcountsAtB := make(AccountMap)
-	it, _ := trie.NewDifferenceIterator(iters.older, iters.newer)
-	for it.Next(true) {
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return nil, nil, err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			// created vs updated is important for leaf nodes since we need to diff their storage
-			// so we need to map all changed accounts at B to their leafkey, since account can change paths but not leafkey
-			var account types.StateAccount
-			if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
-				return nil, nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
-			}
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			diffAcountsAtB[common.Bytes2Hex(leafKey)] = accountWrapper{
-				NodeType:  node.NodeType,
-				Path:      node.Path,
-				NodeValue: node.NodeValue,
-				LeafKey:   leafKey,
-				Account:   &account,
-			}
-		case sdtypes.Extension, sdtypes.Branch:
-			// create a diff for any intermediate node that has changed at b
-			// created vs updated makes no difference for intermediate nodes since we do not need to diff storage
-			if err := output(sdtypes.StateNode{
-				NodeType:  node.NodeType,
-				Path:      node.Path,
-				NodeValue: node.NodeValue,
-			}); err != nil {
-				return nil, nil, err
-			}
-		default:
-			return nil, nil, fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-		// add both intermediate and leaf node paths to the list of diffPathsAtB
-		diffPathsAtB[common.Bytes2Hex(node.Path)] = true
-	}
-	return diffAcountsAtB, diffPathsAtB, it.Error()
-}
-
-// deletedOrUpdatedState returns a slice of all the paths that are emptied at B
-// and a mapping of their leafkeys to all the accounts that exist in a different state at A than B
-func (sdb *builder) deletedOrUpdatedState(iters iterPair, diffPathsAtB map[string]bool, output sdtypes.StateNodeSink) (AccountMap, error) {
-	diffAccountAtA := make(AccountMap)
-	it, _ := trie.NewDifferenceIterator(iters.newer, iters.older)
-	for it.Next(true) {
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return nil, err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			// map all different accounts at A to their leafkey
-			var account types.StateAccount
-			if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
-				return nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
-			}
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			diffAccountAtA[common.Bytes2Hex(leafKey)] = accountWrapper{
-				NodeType:  node.NodeType,
-				Path:      node.Path,
-				NodeValue: node.NodeValue,
-				LeafKey:   leafKey,
-				Account:   &account,
-			}
-			// if this node's path did not show up in diffPathsAtB
-			// that means the node at this path was deleted (or moved) in B
-			// emit an empty "removed" diff to signify as such
-			if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
-				if err := output(sdtypes.StateNode{
-					Path:      node.Path,
-					NodeValue: []byte{},
-					NodeType:  sdtypes.Removed,
-					LeafKey:   leafKey,
-				}); err != nil {
-					return nil, err
-				}
-			}
-		case sdtypes.Extension, sdtypes.Branch:
-			// if this node's path did not show up in diffPathsAtB
-			// that means the node at this path was deleted (or moved) in B
-			// emit an empty "removed" diff to signify as such
-			if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; !ok {
-				if err := output(sdtypes.StateNode{
-					Path:      node.Path,
-					NodeValue: []byte{},
-					NodeType:  sdtypes.Removed,
-				}); err != nil {
-					return nil, err
-				}
-			}
-			// fall through, we did everything we need to do with these node types
-		default:
-			return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-	}
-	return diffAccountAtA, it.Error()
-}
-
-// buildAccountUpdates uses the account diffs maps for A => B and B => A and the known intersection of their leafkeys
-// to generate the statediff node objects for all of the accounts that existed at both A and B but in different states
-// needs to be called before building account creations and deletions as this mutates
-// those account maps to remove the accounts which were updated
-func (sdb *builder) buildAccountUpdates(creations, deletions AccountMap, updatedKeys []string, watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output sdtypes.StateNodeSink) error {
-	var err error
-	for _, key := range updatedKeys {
-		createdAcc := creations[key]
-		deletedAcc := deletions[key]
-		var storageDiffs []sdtypes.StorageNode
-		if deletedAcc.Account != nil && createdAcc.Account != nil {
-			oldSR := deletedAcc.Account.Root
-			newSR := createdAcc.Account.Root
-			err = sdb.buildStorageNodesIncremental(oldSR, newSR, watchedStorageKeys, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
-			if err != nil {
-				return fmt.Errorf("failed building incremental storage diffs for account with leafkey %s\r\nerror: %v", key, err)
-			}
-		}
-		if err = output(sdtypes.StateNode{
-			NodeType:     createdAcc.NodeType,
-			Path:         createdAcc.Path,
-			NodeValue:    createdAcc.NodeValue,
-			LeafKey:      createdAcc.LeafKey,
-			StorageNodes: storageDiffs,
-		}); err != nil {
-			return err
-		}
-		delete(creations, key)
-		delete(deletions, key)
-	}
-
-	return nil
-}
-
-// buildAccountCreations returns the statediff node objects for all the accounts that exist at B but not at A
-// it also returns the code and codehash for created contract accounts
-func (sdb *builder) buildAccountCreations(accounts AccountMap, watchedStorageKeys []common.Hash, intermediateStorageNodes bool, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
-	for _, val := range accounts {
-		diff := sdtypes.StateNode{
-			NodeType:  val.NodeType,
-			Path:      val.Path,
-			LeafKey:   val.LeafKey,
-			NodeValue: val.NodeValue,
-		}
-		if !bytes.Equal(val.Account.CodeHash, nullCodeHash) {
-			// For contract creations, any storage node contained is a diff
-			var storageDiffs []sdtypes.StorageNode
-			err := sdb.buildStorageNodesEventual(val.Account.Root, watchedStorageKeys, intermediateStorageNodes, storageNodeAppender(&storageDiffs))
-			if err != nil {
-				return fmt.Errorf("failed building eventual storage diffs for node %x\r\nerror: %v", val.Path, err)
-			}
-			diff.StorageNodes = storageDiffs
-			// emit codehash => code mappings for code
-			codeHash := common.BytesToHash(val.Account.CodeHash)
-			code, err := sdb.stateCache.ContractCode(common.Hash{}, codeHash)
-			if err != nil {
-				return fmt.Errorf("failed to retrieve code for codehash %s\r\n error: %v", codeHash.String(), err)
-			}
-			if err := codeOutput(sdtypes.CodeAndCodeHash{
-				Hash: codeHash,
-				Code: code,
-			}); err != nil {
-				return err
-			}
-		}
-		if err := output(diff); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// buildStorageNodesEventual builds the storage diff node objects for a created account
-// i.e. it returns all the storage nodes at this state, since there is no previous state
-func (sdb *builder) buildStorageNodesEventual(sr common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
-	if bytes.Equal(sr.Bytes(), emptyContractRoot.Bytes()) {
-		return nil
-	}
-	log.Debug("Storage Root For Eventual Diff", "root", sr.Hex())
-	sTrie, err := sdb.stateCache.OpenTrie(sr)
-	if err != nil {
-		log.Info("error in build storage diff eventual", "error", err)
-		return err
-	}
-	it := sTrie.NodeIterator(make([]byte, 0))
-	err = sdb.buildStorageNodesFromTrie(it, watchedStorageKeys, intermediateNodes, output)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// buildStorageNodesFromTrie returns all the storage diff node objects in the provided node iterator
-// if any storage keys are provided it will only return those leaf nodes
-// including intermediate nodes can be turned on or off
-func (sdb *builder) buildStorageNodesFromTrie(it trie.NodeIterator, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
-	for it.Next(true) {
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			if isWatchedStorageKey(watchedStorageKeys, leafKey) {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  node.NodeType,
-					Path:      node.Path,
-					NodeValue: node.NodeValue,
-					LeafKey:   leafKey,
-				}); err != nil {
-					return err
-				}
-			}
-		case sdtypes.Extension, sdtypes.Branch:
-			if intermediateNodes {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  node.NodeType,
-					Path:      node.Path,
-					NodeValue: node.NodeValue,
-				}); err != nil {
-					return err
-				}
-			}
-		default:
-			return fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-	}
-	return it.Error()
-}
-
-// buildStorageNodesIncremental builds the storage diff node objects for all nodes that exist in a different state at B than A
-func (sdb *builder) buildStorageNodesIncremental(oldSR common.Hash, newSR common.Hash, watchedStorageKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
-	if bytes.Equal(newSR.Bytes(), oldSR.Bytes()) {
-		return nil
-	}
-	log.Debug("Storage Roots for Incremental Diff", "old", oldSR.Hex(), "new", newSR.Hex())
-	oldTrie, err := sdb.stateCache.OpenTrie(oldSR)
-	if err != nil {
-		return err
-	}
-	newTrie, err := sdb.stateCache.OpenTrie(newSR)
-	if err != nil {
-		return err
-	}
-
-	diffPathsAtB, err := sdb.createdAndUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), watchedStorageKeys, intermediateNodes, output)
-	if err != nil {
-		return err
-	}
-	err = sdb.deletedOrUpdatedStorage(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{}), diffPathsAtB, watchedStorageKeys, intermediateNodes, output)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (sdb *builder) createdAndUpdatedStorage(a, b trie.NodeIterator, watchedKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) (map[string]bool, error) {
-	diffPathsAtB := make(map[string]bool)
-	it, _ := trie.NewDifferenceIterator(a, b)
-	for it.Next(true) {
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return nil, err
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			if isWatchedStorageKey(watchedKeys, leafKey) {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  node.NodeType,
-					Path:      node.Path,
-					NodeValue: node.NodeValue,
-					LeafKey:   leafKey,
-				}); err != nil {
-					return nil, err
-				}
-			}
-		case sdtypes.Extension, sdtypes.Branch:
-			if intermediateNodes {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  node.NodeType,
-					Path:      node.Path,
-					NodeValue: node.NodeValue,
-				}); err != nil {
-					return nil, err
-				}
-			}
-		default:
-			return nil, fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-		diffPathsAtB[common.Bytes2Hex(node.Path)] = true
-	}
-	return diffPathsAtB, it.Error()
-}
-
-func (sdb *builder) deletedOrUpdatedStorage(a, b trie.NodeIterator, diffPathsAtB map[string]bool, watchedKeys []common.Hash, intermediateNodes bool, output sdtypes.StorageNodeSink) error {
-	it, _ := trie.NewDifferenceIterator(b, a)
-	for it.Next(true) {
-		if it.Leaf() || bytes.Equal(nullHashBytes, it.Hash().Bytes()) {
-			continue
-		}
-		node, nodeElements, err := resolveNode(it, sdb.stateCache.TrieDB())
-		if err != nil {
-			return err
-		}
-		// if this node path showed up in diffPathsAtB
-		// that means this node was updated at B and we already have the updated diff for it
-		// otherwise that means this node was deleted in B and we need to add a "removed" diff to represent that event
-		if _, ok := diffPathsAtB[common.Bytes2Hex(node.Path)]; ok {
-			continue
-		}
-		switch node.NodeType {
-		case sdtypes.Leaf:
-			partialPath := trie.CompactToHex(nodeElements[0].([]byte))
-			valueNodePath := append(node.Path, partialPath...)
-			encodedPath := trie.HexToCompact(valueNodePath)
-			leafKey := encodedPath[1:]
-			if isWatchedStorageKey(watchedKeys, leafKey) {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  sdtypes.Removed,
-					Path:      node.Path,
-					NodeValue: []byte{},
-					LeafKey:   leafKey,
-				}); err != nil {
-					return err
-				}
-			}
-		case sdtypes.Extension, sdtypes.Branch:
-			if intermediateNodes {
-				if err := output(sdtypes.StorageNode{
-					NodeType:  sdtypes.Removed,
-					Path:      node.Path,
-					NodeValue: []byte{},
-				}); err != nil {
-					return err
-				}
-			}
-		default:
-			return fmt.Errorf("unexpected node type %s", node.NodeType)
-		}
-	}
-	return it.Error()
-}
-
-// isWatchedAddress is used to check if a state account corresponds to one of the addresses the builder is configured to watch
-func isWatchedAddress(watchedAddresses []common.Address, stateLeafKey []byte) bool {
-	// If we aren't watching any specific addresses, we are watching everything
-	if len(watchedAddresses) == 0 {
-		return true
-	}
-	for _, addr := range watchedAddresses {
-		addrHashKey := crypto.Keccak256(addr.Bytes())
-		if bytes.Equal(addrHashKey, stateLeafKey) {
-			return true
-		}
-	}
-	return false
-}
-
-// isWatchedStorageKey is used to check if a storage leaf corresponds to one of the storage slots the builder is configured to watch
-func isWatchedStorageKey(watchedKeys []common.Hash, storageLeafKey []byte) bool {
-	// If we aren't watching any specific addresses, we are watching everything
-	if len(watchedKeys) == 0 {
-		return true
-	}
-	for _, hashKey := range watchedKeys {
-		if bytes.Equal(hashKey.Bytes(), storageLeafKey) {
-			return true
-		}
-	}
-	return false
-}
@@ -17,9 +17,9 @@
 package prom

 import (
-	"database/sql"
-
 	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
 )

 const (
@@ -29,7 +29,7 @@ const (

 // DBStatsGetter is an interface that gets sql.DBStats.
 type DBStatsGetter interface {
-	Stats() sql.DBStats
+	Stats() sql.Stats
 }

 // DBStatsCollector implements the prometheus.Collector interface.
@@ -122,41 +122,41 @@ func (c DBStatsCollector) Collect(ch chan<- prometheus.Metric) {
 	ch <- prometheus.MustNewConstMetric(
 		c.maxOpenDesc,
 		prometheus.GaugeValue,
-		float64(stats.MaxOpenConnections),
+		float64(stats.MaxOpen()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.openDesc,
 		prometheus.GaugeValue,
-		float64(stats.OpenConnections),
+		float64(stats.Open()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.inUseDesc,
 		prometheus.GaugeValue,
-		float64(stats.InUse),
+		float64(stats.InUse()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.idleDesc,
 		prometheus.GaugeValue,
-		float64(stats.Idle),
+		float64(stats.Idle()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.waitedForDesc,
 		prometheus.CounterValue,
-		float64(stats.WaitCount),
+		float64(stats.WaitCount()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.blockedSecondsDesc,
 		prometheus.CounterValue,
-		stats.WaitDuration.Seconds(),
+		stats.WaitDuration().Seconds(),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.closedMaxIdleDesc,
 		prometheus.CounterValue,
-		float64(stats.MaxIdleClosed),
+		float64(stats.MaxIdleClosed()),
 	)
 	ch <- prometheus.MustNewConstMetric(
 		c.closedMaxLifetimeDesc,
 		prometheus.CounterValue,
-		float64(stats.MaxLifetimeClosed),
+		float64(stats.MaxLifetimeClosed()),
 	)
 }
@@ -19,7 +19,6 @@ package prom
 import (
 	"time"

-	"github.com/jmoiron/sqlx"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )
@@ -123,7 +122,7 @@ func Init() {
 }

 // RegisterDBCollector create metric collector for given connection
-func RegisterDBCollector(name string, db *sqlx.DB) {
+func RegisterDBCollector(name string, db DBStatsGetter) {
 	if metrics {
 		prometheus.Register(NewDBStatsCollector(name, db))
 	}
@@ -26,6 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/vulcanize/leveldb-ethdb-rpc/pkg/client"
 )

 // Reader interface required by the statediffing service
@@ -46,15 +47,26 @@ type LvlDBReader struct {

 // LvLDBReaderConfig struct for initializing a LvlDBReader
 type LvLDBReaderConfig struct {
-	TrieConfig        *trie.Config
-	ChainConfig       *params.ChainConfig
-	Path, AncientPath string
-	DBCacheSize       int
+	TrieConfig             *trie.Config
+	ChainConfig            *params.ChainConfig
+	Mode                   string
+	Path, AncientPath, Url string
+	DBCacheSize            int
 }

 // NewLvlDBReader creates a new Read using LevelDB
 func NewLvlDBReader(conf LvLDBReaderConfig) (*LvlDBReader, error) {
-	edb, err := rawdb.NewLevelDBDatabaseWithFreezer(conf.Path, conf.DBCacheSize, 256, conf.AncientPath, "eth-statediff-service", true)
+	var edb ethdb.Database
+	var err error
+
+	if conf.Mode == "local" {
+		edb, err = rawdb.NewLevelDBDatabaseWithFreezer(conf.Path, conf.DBCacheSize, 256, conf.AncientPath, "eth-statediff-service", true)
+	}
+
+	if conf.Mode == "remote" {
+		edb, err = client.NewDatabaseClient(conf.Url)
+	}
+
 	if err != nil {
 		return nil, err
 	}
@@ -24,18 +24,18 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"

-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 // StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules.
 func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (*rpc.Server, error) {

 	srv := rpc.NewServer()
-	err := node.RegisterApis(apis, modules, srv, false)
+	err := node.RegisterApis(apis, modules, srv)
 	if err != nil {
 		utils.Fatalf("Could not register HTTP API: %w", err)
 	}
-	handler := node.NewHTTPHandlerStack(srv, cors, vhosts)
+	handler := node.NewHTTPHandlerStack(srv, cors, vhosts, nil)

 	// start http server
 	_, addr, err := node.StartHTTPEndpoint(endpoint, rpc.DefaultHTTPTimeouts, prom.HTTPMiddleware(handler))
@@ -26,7 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"

-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 var (
@@ -29,11 +29,11 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	sd "github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/eth-statediff-service/pkg/prom"

-	ind "github.com/ethereum/go-ethereum/statediff/indexer"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 const defaultQueueSize = 1024
@@ -66,13 +66,13 @@ type StateDiffService interface {
 // Service is the underlying struct for the state diffing service
 type Service struct {
 	// Used to build the state diff objects
-	Builder Builder
-	// Used to read data from leveldb
+	Builder sd.Builder
+	// Used to read data from LevelDB
 	lvlDBReader Reader
 	// Used to signal shutdown of the service
 	quitChan chan struct{}
 	// Interface for publishing statediffs as PG-IPLD objects
-	indexer ind.Indexer
+	indexer interfaces.StateDiffIndexer
 	// range queue
 	queue chan RangeRequest
 	// number of ranges we can work over concurrently
@@ -82,8 +82,8 @@ type Service struct {
 }

 // NewStateDiffService creates a new Service
-func NewStateDiffService(lvlDBReader Reader, indexer ind.Indexer, conf Config) (*Service, error) {
-	builder, err := NewBuilder(lvlDBReader.StateDB(), conf.TrieWorkers)
+func NewStateDiffService(lvlDBReader Reader, indexer interfaces.StateDiffIndexer, conf Config) (*Service, error) {
+	b, err := NewBuilder(lvlDBReader.StateDB(), conf.TrieWorkers)
 	if err != nil {
 		return nil, err
 	}
@@ -92,7 +92,7 @@ func NewStateDiffService(lvlDBReader Reader, indexer ind.Indexer, conf Config) (
 	}
 	return &Service{
 		lvlDBReader: lvlDBReader,
-		Builder:     builder,
+		Builder:     b,
 		indexer:     indexer,
 		workers:     conf.ServiceWorkers,
 		queue:       make(chan RangeRequest, conf.WorkerQueueSize),
@@ -193,7 +193,11 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Paylo
 	if err != nil {
 		return nil, err
 	}
-	logrus.Info(fmt.Sprintf("sending state diff at block %d", blockNumber))
+	logrus.Infof("sending state diff at block %d", blockNumber)
+
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
+
 	if blockNumber == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
 	}
@@ -211,7 +215,11 @@ func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.P
 	if err != nil {
 		return nil, err
 	}
-	logrus.Info(fmt.Sprintf("sending state diff at block %s", blockHash.Hex()))
+	logrus.Infof("sending state diff at block %s", blockHash.Hex())
+
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
+
 	if currentBlock.NumberU64() == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
 	}
@@ -233,7 +241,7 @@ func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot commo
 	if err != nil {
 		return nil, err
 	}
-	stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
+	stateDiffRlp, err := rlp.EncodeToBytes(&stateDiff)
 	if err != nil {
 		return nil, err
 	}
@@ -280,7 +288,11 @@ func (sds *Service) StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Paylo
 	if err != nil {
 		return nil, err
 	}
-	logrus.Info(fmt.Sprintf("sending state trie at block %d", blockNumber))
+	logrus.Infof("sending state trie at block %d", blockNumber)
+
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
+
 	return sds.processStateTrie(currentBlock, params)
 }

@@ -289,7 +301,7 @@ func (sds *Service) processStateTrie(block *types.Block, params sd.Params) (*sd.
 	if err != nil {
 		return nil, err
 	}
-	stateTrieRlp, err := rlp.EncodeToBytes(stateNodes)
+	stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes)
 	if err != nil {
 		return nil, err
 	}
@@ -314,12 +326,16 @@ func (sds *Service) Stop() error {
 // This operation cannot be performed back past the point of db pruning; it requires an archival node
 // for historical data
 func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error {
-	logrus.Info(fmt.Sprintf("Writing state diff at block %d", blockNumber))
+	logrus.Infof("Writing state diff at block %d", blockNumber)
 	t := time.Now()
 	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
 	if err != nil {
 		return err
 	}
+
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
+
 	parentRoot := common.Hash{}
 	if blockNumber != 0 {
 		parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
@@ -335,12 +351,16 @@ func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error
 // This operation cannot be performed back past the point of db pruning; it requires an archival node
 // for historical data
 func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) error {
-	logrus.Info(fmt.Sprintf("Writing state diff for block %s", blockHash.Hex()))
+	logrus.Infof("Writing state diff for block %s", blockHash.Hex())
 	t := time.Now()
 	currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
 	if err != nil {
 		return err
 	}
+
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
+
 	parentRoot := common.Hash{}
 	if currentBlock.NumberU64() != 0 {
 		parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
@@ -379,20 +399,20 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
 	}
 	// defer handling of commit/rollback for any return case
 	output := func(node sdtypes.StateNode) error {
-		return sds.indexer.PushStateNode(tx, node)
+		return sds.indexer.PushStateNode(tx, node, block.Hash().String())
 	}
 	codeOutput := func(c sdtypes.CodeAndCodeHash) error {
 		return sds.indexer.PushCodeAndCodeHash(tx, c)
 	}
 	prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t))
 	t = time.Now()
-	err = sds.Builder.WriteStateDiffObject(sd.StateRoots{
+	err = sds.Builder.WriteStateDiffObject(sdtypes.StateRoots{
 		NewStateRoot: block.Root(),
 		OldStateRoot: parentRoot,
 	}, params, output, codeOutput)
 	prom.SetTimeMetric(prom.T_STATE_PROCESSING, time.Now().Sub(t))
 	t = time.Now()
-	err = tx.Close(err)
+	err = tx.Submit(err)
 	prom.SetLastProcessedHeight(height)
 	prom.SetTimeMetric(prom.T_POSTGRES_TX_COMMIT, time.Now().Sub(t))
 	return err
@@ -1,125 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package testhelpers
-
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/consensus/ethash"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/params"
-)
-
-// MakeChain creates a chain of n blocks starting at and including parent.
-// the returned hash chain is ordered head->parent.
-func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, *core.BlockChain) {
-	config := params.TestChainConfig
-	config.LondonBlock.Set(big.NewInt(10))
-	blocks, _ := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
-	chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
-	return blocks, chain
-}
-
-func TestSelfDestructChainGen(i int, block *core.BlockGen) {
-	signer := types.HomesteadSigner{}
-	switch i {
-	case 0:
-		// Block 1 is mined by Account1Addr
-		// Account1Addr creates a new contract
-		block.SetCoinbase(TestBankAddress)
-		tx, _ := types.SignTx(types.NewContractCreation(0, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, TestBankKey)
-		ContractAddr = crypto.CreateAddress(TestBankAddress, 0)
-		block.AddTx(tx)
-	case 1:
-		// Block 2 is mined by Account1Addr
-		// Account1Addr self-destructs the contract
-		block.SetCoinbase(TestBankAddress)
-		data := common.Hex2Bytes("43D726D6")
-		tx, _ := types.SignTx(types.NewTransaction(1, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
-		block.AddTx(tx)
-	}
-}
-
-func TestChainGen(i int, block *core.BlockGen) {
-	signer := types.HomesteadSigner{}
-	switch i {
-	case 0:
-		// In block 1, the test bank sends account #1 some ether.
-		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey)
-		block.AddTx(tx)
-	case 1:
-		// In block 2, the test bank sends some more ether to account #1.
-		// Account1Addr passes it on to account #2.
-		// Account1Addr creates a test contract.
-		tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey)
-		nonce := block.TxNonce(Account1Addr)
-		tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key)
-		nonce++
-		tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, Account1Key)
-		ContractAddr = crypto.CreateAddress(Account1Addr, nonce)
-		block.AddTx(tx1)
-		block.AddTx(tx2)
-		block.AddTx(tx3)
-	case 2:
-		// Block 3 has a single tx from the bankAccount to the contract, that transfers no value
-		// Block 3 is mined by Account2Addr
-		block.SetCoinbase(Account2Addr)
-		//put function: c16431b9
-		//close function: 43d726d6
-		data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003")
-		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
-		block.AddTx(tx)
-	case 3:
-		// Block 4 has three txs from bankAccount to the contract, that transfer no value
-		// Two set the two original slot positions to 0 and one sets another position to a new value
-		// Block 4 is mined by Account2Addr
-		block.SetCoinbase(Account2Addr)
-		data1 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
-		data2 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000")
-		data3 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000009")
-
-		nonce := block.TxNonce(TestBankAddress)
-		tx1, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data1), signer, TestBankKey)
-		nonce++
-		tx2, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data2), signer, TestBankKey)
-		nonce++
-		tx3, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data3), signer, TestBankKey)
-		block.AddTx(tx1)
-		block.AddTx(tx2)
-		block.AddTx(tx3)
-	case 4:
-		// Block 5 has one tx from bankAccount to the contract, that transfers no value
-		// It sets the remaining storage value to zero
-		// Block 5 is mined by Account1Addr
-		block.SetCoinbase(Account1Addr)
-		data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000")
-		nonce := block.TxNonce(TestBankAddress)
-		tx, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
-		block.AddTx(tx)
-	case 5:
-		// Block 6 has a tx from Account1Key which self-destructs the contract, it transfers no value
-		// Block 6 is mined by Account2Addr
-		block.SetCoinbase(Account2Addr)
-		data := common.Hex2Bytes("43D726D6")
-		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(Account1Addr), ContractAddr, big.NewInt(0), 100000, nil, data), signer, Account1Key)
-		block.AddTx(tx)
-	}
-}
@@ -1,67 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package testhelpers
-
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/rlp"
-)
-
-// AddressToLeafKey hashes an returns an address
-func AddressToLeafKey(address common.Address) []byte {
-	return crypto.Keccak256(address[:])
-}
-
-// AddressToEncodedPath hashes an address and appends the even-number leaf flag to it
-func AddressToEncodedPath(address common.Address) []byte {
-	addrHash := crypto.Keccak256(address[:])
-	decodedPath := append(EvenLeafFlag, addrHash...)
-	return decodedPath
-}
-
-// Test variables
-var (
-	EvenLeafFlag = []byte{byte(2) << 4}
-	NullCodeHash = crypto.Keccak256Hash([]byte{})
-	NullHash     = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
-
-	Testdb          = rawdb.NewMemoryDatabase()
-	TestBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
-	TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
-	BankLeafKey     = AddressToLeafKey(TestBankAddress)
-	TestBankFunds   = big.NewInt(100000000)
-	Genesis         = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds)
-
-	Account1Key, _          = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
-	Account2Key, _          = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
-	Account1Addr            = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
-	Account2Addr            = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e
-	Account1LeafKey         = AddressToLeafKey(Account1Addr)
-	Account2LeafKey         = AddressToLeafKey(Account2Addr)
-	ContractCode            = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055506040518060200160405280600160ff16815250600190600161007492919061007a565b506100e4565b82606481019282156100ae579160200282015b828111156100ad578251829060ff1690559160200191906001019061008d565b5b5090506100bb91906100bf565b5090565b6100e191905b808211156100dd5760008160009055506001016100c5565b5090565b90565b6101ca806100f36000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c806343d726d61461003b578063c16431b914610045575b600080fd5b61004361007d565b005b61007b6004803603604081101561005b57600080fd5b81019080803590602001909291908035906020019092919050505061015c565b005b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610122576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260228152602001806101746022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b806001836064811061016a57fe5b0181905550505056fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a72305820e3747183708fb6bff3f6f7a80fb57dcc1c19f83f9cb25457a3ed5c0424bde66864736f6c634300050a0032")
-	ByteCodeAfterDeployment = common.Hex2Bytes("608060405234801561001057600080fd5b50600436106100365760003560e01c806343d726d61461003b578063c16431b914610045575b600080fd5b61004361007d565b005b61007b6004803603604081101561005b57600080fd5b81019080803590602001909291908035906020019092919050505061015c565b005b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610122576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260228152602001806101746022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b806001836064811061016a57fe5b0181905550505056fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a72305820e3747183708fb6bff3f6f7a80fb57dcc1c19f83f9cb25457a3ed5c0424bde66864736f6c634300050a0032")
-	CodeHash                = common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127")
-	ContractAddr            common.Address
-
-	EmptyRootNode, _  = rlp.EncodeToBytes([]byte{})
-	EmptyContractRoot = crypto.Keccak256Hash(EmptyRootNode)
-)
@@ -0,0 +1,30 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package version
+
+import "fmt"
+
+const (
+	major = 3 // major version component of the current release
+	minor = 0 // minor version component of the current release
+	patch = 0 // patch version component of the current release
+)
+
+// Version holds the textual version string.
+var Version = func() string {
+	return fmt.Sprintf("%d.%d.%d", major, minor, patch)
+}()
Author	SHA1	Message	Date
Michael Shaw	67d8bced4f	one more reference changed to git.vdb.to	2022-09-21 13:27:05 -04:00
Michael Shaw	12f4810ced	missed Makefile update for cerc-io	2022-09-21 11:55:17 -04:00
Michael Shaw	7a8d38c955	indentation problem in yml	2022-09-21 11:24:43 -04:00
Michael Shaw	d09cd0afe6	update tagging and push to git.vdb.to registry	2022-09-21 11:13:23 -04:00
Michael	bcca82eaa3	Cerc refactor (#104 ) * cerc refactor waiting on unpublished dependencies * updated go.mod for published dependencies in cerc-io refactoring	2022-09-20 14:00:10 -04:00
Michael	dad77b561d	Merge pull request #102 from vulcanize/rebase-1.10.23-wip update for geth 1.10.23	2022-09-02 16:16:34 -04:00
Michael Shaw	c939822a95	update for geth 1.10.23	2022-09-02 15:56:56 -04:00
Michael	1fdb8763ac	Merge pull request #101 from vulcanize/geth_1_10_21_wip updates for geth 1.10.21	2022-08-03 10:21:22 -04:00
Michael Shaw	cd5aee30c7	missed go mod tidy	2022-08-03 10:08:01 -04:00
Michael Shaw	040638ca05	updates for geth 1.10.21	2022-08-03 10:03:57 -04:00
Ashwin Phatak	681e656034	Merge pull request #98 from deep-stack/pm-import-instructions Add instructions to import data output in file mode	2022-07-19 15:10:25 +05:30
prathamesh	85896f91b7	Expose database stats in metrics	2022-07-19 15:00:01 +05:30
prathamesh	52c7f84432	Update instructions to import data output	2022-07-18 19:14:41 +05:30
prathamesh	aa6ee578f4	Add instructions to import data output in file mode	2022-07-13 19:33:47 +05:30
prathamesh0	1f898f60c3	Upgrade geth after schema changes (#97 ) * Include intermediate nodes in builder tests for watched addresses * Upgrade geth	2022-07-12 16:38:05 +05:30
nabarun	0c56037e1f	Upgrade geth and add params for CSV file mode (#96 ) * Upgrade geth and add params for CSV file mode * Add options for file mode flag * index intermediate nodes when watching addresses	2022-07-01 13:29:48 +05:30
prathamesh0	9aa683442e	Reuse builder from vulcanize-geth (#92 ) * Reuse builder code from vulcanize-geth * Upgrade geth dependency	2022-06-24 14:12:58 +05:30
Abdul Rabbani	adf3dd4b6f	Update CICD pipeline to skip duplicate tests (#95 ) * Update CICD pipeline to skip duplicate tests * fix dependencies * Fix other dep	2022-06-17 09:45:53 -04:00
Abdul Rabbani	5ea4b6766a	Merge pull request #94 from vulcanize/feature/update-geth-v1.10.19 Update geth and check for duplicate jobs in CI	2022-06-17 09:00:19 -04:00
Abdul Rabbani	2c4fd6f099	Update geth and check for duplicate jobs in CI	2022-06-17 08:56:25 -04:00
Ashwin Phatak	f964b53fe3	Merge pull request #91 from vulcanize/ng-update-iterator Update go-eth-state-node-iterator version	2022-06-10 15:27:31 +05:30
nabarun	b8dad6a09b	Update go-eth-state-node-iterator version	2022-06-10 15:02:42 +05:30
Abdul Rabbani	1dc90d0417	Merge pull request #89 from vulcanize/feature/update-go-geth-sharding Feature/update go geth sharding	2022-05-31 13:41:15 -04:00
Abdul Rabbani	784ffb8726	Update geth version	2022-05-31 13:37:35 -04:00
Abdul Rabbani	1ddffe65be	Merge branch 'master' into feature/update-go-geth-sharding	2022-05-31 13:35:11 -04:00
Abdul Rabbani	fdeece4b5d	Merge pull request #88 from vulcanize/release-v3.0.3 Update go and geth versions	2022-05-31 07:30:38 -04:00
Abdul Rabbani	a58b16b0a1	Update go and dependencies	2022-05-27 15:10:57 -04:00
Abdul Rabbani	a99dcb4ca9	Update the PR and release pipeline	2022-05-26 15:32:48 -04:00
Abdul Rabbani	d3fd7ef532	Update geth version using 1.10.18	2022-05-26 08:30:55 -04:00
Abdul Rabbani	3e8f5d34fb	Merge pull request #86 from vulcanize/feature/cicd-docker-test Combine build and publish	2022-05-18 15:21:38 -04:00
Abdul Rabbani	1df0e8b840	Fix GHA issues	2022-05-18 15:16:42 -04:00
Abdul Rabbani	5bdec6bc41	Change pipeline to not rebuild the image.	2022-05-18 15:11:06 -04:00
Abdul Rabbani	9fbc07c836	Combine build and publish	2022-05-18 14:58:46 -04:00
nabarun	cdcc3df9f2	Upgrade geth version to v4 (#85 ) * Upgrade geth version to v4 * Upgrade go-eth-state-node-iterator version	2022-05-17 19:22:30 +05:30
nabarun	5f6aec35b2	Support remote RPC client to access LevelDB (#83 ) * Implement use of custom chain config to test locally * Use leveldb-ethdb-rpc instead of leveldb path * Implement flag for leveldb access mode * Remove custom chain config json * Move database client to leveldb-ethdb-rpc * Update leveldb-ethdb-rpc version	2022-05-17 14:35:11 +05:30
Ashwin Phatak	867f5c9971	Merge pull request #84 from deep-stack/ng-update-geth Update vulcanize geth version to 1.10.17	2022-05-16 10:10:58 +05:30
nabarun	257b16e184	Add missing tests for direct indexing mode	2022-05-13 18:28:58 +05:30
nabarun	b4a899cc8c	Remove WatchedStorageSlots param and local chain.json	2022-05-13 17:25:31 +05:30
nabarun	4363bf5c07	Implement use of custom chain config to test locally	2022-05-13 16:37:30 +05:30
nabarun	7c19e86698	Call method for computing watchedAddressesLeafKeys	2022-05-12 15:51:46 +05:30
nabarun	6ed1bc739d	Update builder tests to use geth statediff test helpers	2022-05-11 18:53:39 +05:30
nabarun	32724ff46c	Fix rlp.EncodeToBytes arg and creating node iterators	2022-05-11 15:21:23 +05:30
nabarun	9a7ecf9ef3	Add watched addresses direct indexing mode changes	2022-05-11 00:03:20 +05:30
nabarun	ea4fb6222b	Update geth version	2022-05-10 11:36:14 +05:30
Ian Norden	1d10009678	Merge pull request #78 from vulcanize/release-v3.0.0 Release v3.0.0	2022-02-01 12:15:32 -06:00
i-norden	feca7b67db	bump major version	2022-02-01 12:11:57 -06:00
i-norden	edef920ae4	bump iterator version	2022-02-01 12:11:21 -06:00
i-norden	ec0fdb5215	rebase	2022-02-01 12:08:25 -06:00
i-norden	040bc850f0	bump statediff geth version	2022-02-01 11:34:02 -06:00
Ian Norden	0c630b6e99	Merge pull request #77 from vulcanize/release-v2.0.0 update to use v1.10.15-statediff-2.0.0 geth	2022-01-26 09:27:35 -06:00
i-norden	b68bd29304	bump version	2022-01-26 09:26:18 -06:00
i-norden	c3b007de26	update to use v1.10.15-statediff-2.0.0 geth	2022-01-26 09:22:17 -06:00
Arijit Das	e9428ac208	Merge pull request #75 from vulcanize/docker-release-using-branch Release docker image using branch.	2022-01-13 21:21:38 +05:30
Ian Norden	9521bda9d0	Merge pull request #76 from vulcanize/impl-version-cmd Add version command.	2021-12-27 11:49:56 -06:00
i-norden	9f76540c3a	bump version, update license year	2021-12-27 11:49:10 -06:00
Arijit Das	747c5f5400	Use commit hash to build docker image and publish.	2021-12-23 19:53:53 +05:30
Arijit Das	3725006984	Add version command.	2021-12-23 16:40:23 +05:30
Arijit Das	e08d31e242	Use branch name.	2021-12-21 19:09:08 +05:30
Arijit Das	0ff60a69da	Small fix.	2021-12-21 18:12:22 +05:30
Ian Norden	0d60752b8a	Merge pull request #71 from vulcanize/bump bump go-ethereum version	2021-12-17 13:36:33 -06:00
i-norden	1126466d51	bump go-ethereum version	2021-12-17 12:19:44 -06:00
i-norden	6e604c3db6	bump go-ethereum version	2021-12-01 23:24:05 -06:00
Ian Norden	0c2a9bc171	Merge pull request #67 from vulcanize/rct_size_bug bump go-ethereum version	2021-12-01 22:47:37 -06:00
i-norden	f35c5cf71e	bump go-ethereum version	2021-12-01 22:46:45 -06:00
Ian Norden	fe94c5a33c	Merge pull request #66 from vulcanize/log_processing_fix bump go-ethereum version	2021-12-01 16:01:29 -06:00
i-norden	55a20ac15f	bump go-ethereum version	2021-12-01 16:00:53 -06:00
Ian Norden	eee4330bf4	Merge pull request #65 from vulcanize/log_processing_fix bump go-etheruem version	2021-11-30 18:21:12 -06:00
i-norden	5543a52472	bump go-etheruem version	2021-11-30 17:56:04 -06:00
Ian Norden	db786ae3b0	Merge pull request #62 from vulcanize/schema_updates Schema updates	2021-11-26 09:33:27 -06:00
i-norden	d8a3177a20	bump statediffing geth version	2021-11-26 09:29:22 -06:00
i-norden	fb54640823	add some missing ENVs	2021-11-26 09:25:01 -06:00
Ian Norden	731c986d75	Merge pull request #61 from vulcanize/schema_updates fix cli	2021-11-22 13:56:01 -06:00
i-norden	30b481db24	fix cli	2021-11-22 13:55:29 -06:00
Ian Norden	dcdfb9a05a	Merge pull request #60 from vulcanize/schema_updates update to new version of go-ethereum; minor README update	2021-11-20 14:50:19 -06:00
i-norden	d78a503ba4	update to new version of go-ethereum; minor README update	2021-11-20 14:49:36 -06:00
Ian Norden	ddbb690ee6	Merge pull request #59 from vulcanize/schema_updates reflect changes in integrated service	2021-11-18 19:25:36 -06:00
i-norden	bb5d5322d2	update README	2021-11-18 19:18:12 -06:00
i-norden	108a6f89f1	update to use new version of indexer; support file writing mode	2021-11-18 18:47:28 -06:00
i-norden	d9eb68b2d2	update to use refactored statediffing geth	2021-11-12 18:49:24 -06:00
i-norden	c5c230a2ea	update to use new version of statediffing geth pkgs with public.blocks insert batching	2021-11-08 19:08:36 -06:00
i-norden	dcf8afb96c	bump Go version to 1.16	2021-11-08 19:05:35 -06:00