one more reference changed to git.vdb.to

missed Makefile update for cerc-io
indentation problem in yml
2022-09-21 13:27:05 -04:00 · 2022-09-21 11:55:17 -04:00 · 2022-09-21 11:24:43 -04:00 · 2022-09-21 11:13:23 -04:00 · 2022-09-20 14:00:10 -04:00 · 2022-09-02 16:16:34 -04:00
36 changed files with 6723 additions and 353 deletions
@@ -0,0 +1,73 @@
+name: Publish Docker image
+on:
+  release:
+    types: [published]
+  pull_request:
+jobs:
+  pre_job:
+    # continue-on-error: true # Uncomment once integration is finished
+    runs-on: ubuntu-latest
+    # Map a step output to a job output
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@v4
+        with:
+          # All of these options are optional, so you can remove them if you are happy with the defaults
+          concurrent_skipping: "never"
+          skip_after_successful_duplicate: "true"
+          do_not_skip: '["workflow_dispatch", "schedule"]'
+  run-tests:
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    needs: pre_job
+    uses: ./.github/workflows/tests.yml
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    if: |
+      always() &&
+      (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') &&
+      github.event_name == 'release'
+    needs: run-tests
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the version
+        id: vars
+        run: |
+          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
+      - name: Run docker build
+        run: make docker-build
+      - name: Tag docker image
+        run: docker tag cerc-io/eth-statediff-service git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+      - name: Tag docker image TAG
+        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
+      - name: Docker Login
+        run: echo ${{ secrets.GITEA_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+      - name: Docker Push
+        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+#  push_to_registries:
+#    name: Push Docker image to Docker Hub
+#    runs-on: ubuntu-latest
+#    if: |
+#      always() &&
+#      (needs.build.result == 'success') &&
+#      github.event_name == 'release'
+#    needs: build
+#    steps:
+#      - name: Get the version
+#        id: vars
+#        run: |
+#          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+#          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
+#      - name: Docker Login to Github Registry
+#        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+#      - name: Docker Pull
+#        run: docker pull docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+#      - name: Docker Login to Docker Registry
+#        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
+#      - name: Tag docker image
+#        run: docker tag docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
+#      - name: Docker Push to Docker Hub
+#        run: docker push cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
@@ -0,0 +1,37 @@
+name: Tests for Geth that are used in multiple jobs.
+
+on:
+  workflow_call:
+
+env:
+  GOPATH: /tmp/go
+
+jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run docker build
+        run: make docker-build
+
+  statediff-unit-test:
+    name: Run statediff unit tests
+    runs-on: ubuntu-latest
+    env:
+      GO111MODULE: on
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Run unit tests
+        run: |
+          make test
@@ -1,2 +1,3 @@
 .idea/
 eth-statediff-service
+.vscode
@@ -1,33 +1,38 @@
-FROM golang:1.13-alpine as builder
+FROM golang:1.18-alpine as builder

 RUN apk --update --no-cache add make git g++ linux-headers
 # DEBUG
 RUN apk add busybox-extras

 # Get and build ipfs-blockchain-watcher
-ADD . /go/src/github.com/vulcanize/eth-statediff-service
-WORKDIR /go/src/github.com/vulcanize/eth-statediff-service
+ADD . /go/src/github.com/cerc-io/eth-statediff-service
+#RUN git clone https://github.com/cerc-io/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
+
+WORKDIR /go/src/github.com/cerc-io/eth-statediff-service
 RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .

 # app container
 FROM alpine

-ARG USER
-ARG CONFIG_FILE
-ARG EXPOSE_PORT
-RUN adduser -Du 5000 $USER
+ARG USER="vdm"
+ARG CONFIG_FILE="./environments/config.toml"
+ARG EXPOSE_PORT=8545
+
+RUN adduser -Du 5000 $USER adm
+RUN adduser $USER adm; apk --no-cache add sudo; echo '%adm ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
 WORKDIR /app
 RUN chown $USER /app
 USER $USER

 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/$CONFIG_FILE config.toml
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/startup_script.sh .
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/environments environments
+COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/$CONFIG_FILE config.toml
+COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/startup_script.sh .
+COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/eth-statediff-service/environments environments

 # keep binaries immutable
-COPY --from=builder /go/src/github.com/vulcanize/eth-statediff-service/eth-statediff-service eth-statediff-service
+COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/eth-statediff-service eth-statediff-service

 EXPOSE $EXPOSE_PORT

@@ -0,0 +1,12 @@
+## Build docker image
+.PHONY: docker-build
+docker-build:
+	docker build -t cerc-io/eth-statediff-service .
+
+.PHONY: test
+test: | $(GOOSE)
+	go test -p 1 ./pkg/... -v
+
+build:
+	go fmt ./...
+	go build
@@ -1,34 +1,327 @@
-## eth-statediff-service
+# eth-statediff-service

 [![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/eth-statediff-service)](https://goreportcard.com/report/github.com/vulcanize/eth-statediff-service)

->> standalone statediffing service ontop of leveldb
+>> standalone statediffing service on top of LevelDB

 Purpose:

-Stand up a statediffing service directly on top of a go-ethereum leveldb instance.
+Stand up a statediffing service directly on top of a go-ethereum LevelDB instance.
 This service can serve historical state data over the same rpc interface as
-[statediffing geth](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.5) without needing to run a full node
+[statediffing geth](https://github.com/cerc-io/go-ethereum) without needing to run a full node.

-Usage:
+## Setup

-`./eth-statediff-service serve --config={path to toml config file}`
+Build the binary:

-Config:
+```bash
+make build
+```
+
+## Configuration
+
+An example config file:

 ```toml
 [leveldb]
-    path = "/Users/user/Library/Ethereum/geth/chaindata"
-    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
+    # LevelDB access mode <local | remote>
+    mode = "local"  # LVLDB_MODE
+
+    # in local mode
+    # LevelDB paths
+    path    = "/Users/user/Library/Ethereum/geth/chaindata"         # LVLDB_PATH
+    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient" # LVLDB_ANCIENT
+
+    # in remote mode
+    # URL for leveldb-ethdb-rpc endpoint
+    url = "http://127.0.0.1:8082/"  # LVLDB_URL

 [server]
-    ipcPath = "~/.vulcanize/vulcanize.ipc"
-    httpPath = "127.0.0.1:8545"
+    ipcPath  = ".ipc"           # SERVICE_IPC_PATH
+    httpPath = "127.0.0.1:8545" # SERVICE_HTTP_PATH
+
+[statediff]
+    prerun          = true  # STATEDIFF_PRERUN
+    serviceWorkers  = 1     # STATEDIFF_SERVICE_WORKERS
+    workerQueueSize = 1024  # STATEDIFF_WORKER_QUEUE_SIZE
+    trieWorkers     = 4     # STATEDIFF_TRIE_WORKERS
+
+[prerun]
+    only = false    # PRERUN_ONLY
+
+    # to perform prerun in a specific range (optional)
+    start = 0   # PRERUN_RANGE_START
+    stop  = 100 # PRERUN_RANGE_STOP
+
+    # to perform prerun over multiple ranges (optional)
+    ranges = [
+        [101, 1000]
+    ]
+
+    # statediffing params for prerun
+    [prerun.params]
+        intermediateStateNodes   = true # PRERUN_INTERMEDIATE_STATE_NODES
+        intermediateStorageNodes = true # PRERUN_INTERMEDIATE_STORAGE_NODES
+        includeBlock             = true # PRERUN_INCLUDE_BLOCK
+        includeReceipts          = true # PRERUN_INCLUDE_RECEIPTS
+        includeTD                = true # PRERUN_INCLUDE_TD
+        includeCode              = true # PRERUN_INCLUDE_CODE
+        watchedAddresses         = []

 [log]
-    file = ""
-    level = "info"
+    file  = ""      # LOG_FILE_PATH
+    level = "info"  # LOG_LEVEL

-[eth]
-    chainID = 1
+[database]
+    # output type <postgres | file | dump>
+    type = "postgres"
+
+    # with postgres type
+    # db credentials
+    name     = "vulcanize_test" # DATABASE_NAME
+    hostname = "localhost"      # DATABASE_HOSTNAME
+    port     = 5432             # DATABASE_PORT
+    user     = "vulcanize"      # DATABASE_USER
+    password = "..."            # DATABASE_PASSWORD
+    driver   = "sqlx"           # DATABASE_DRIVER_TYPE <sqlx | pgx>
+
+    # with file type
+    # file mode <sql | csv>
+    fileMode = "csv"    # DATABASE_FILE_MODE
+
+    # with SQL file mode
+    filePath = ""   # DATABASE_FILE_PATH
+
+    # with CSV file mode
+    fileCsvDir = "output_dir" # DATABASE_FILE_CSV_DIR
+
+    # with dump type
+    # <stdout | stderr | discard>
+    dumpDestination = ""    # DATABASE_DUMP_DST
+
+[cache]
+    database = 1024 # DB_CACHE_SIZE_MB
+    trie     = 1024 # TRIE_CACHE_SIZE_MB
+
+[prom]
+    # prometheus metrics
+    metrics  = true         # PROM_METRICS
+    http     = true         # PROM_HTTP
+    httpAddr = "localhost"  # PROM_HTTP_ADDR
+    httpPort = "8889"       # PROM_HTTP_PORT
+    dbStats = true          # PROM_DB_STATS
+
+[ethereum]
+    # node info
+    nodeID       = ""                       # ETH_NODE_ID
+    clientName   = "eth-statediff-service"  # ETH_CLIENT_NAME
+    networkID    = 1                        # ETH_NETWORK_ID
+    chainID      = 1                        # ETH_CHAIN_ID
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # ETH_GENESIS_BLOCK
+
+    # path to custom chain config file (optional)
+    # keep chainID same as that in chain config file
+    chainConfig  = "./chain.json"           # ETH_CHAIN_CONFIG
 ```
+
+### Local Setup
+
+* Create a chain config file `chain.json` according to chain config in genesis json file used by local geth.
+
+  Example:
+  ```json
+  {
+    "chainId": 41337,
+    "homesteadBlock": 0,
+    "eip150Block": 0,
+    "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+    "eip155Block": 0,
+    "eip158Block": 0,
+    "byzantiumBlock": 0,
+    "constantinopleBlock": 0,
+    "petersburgBlock": 0,
+    "istanbulBlock": 0,
+    "clique": {
+      "period": 5,
+      "epoch": 30000
+    }
+  }
+  ```
+
+  Provide the path to the above file in the config.
+
+## Usage
+
+* Create / update the config file (refer to example config above).
+
+### `serve`
+
+* To serve the statediff RPC API:
+
+    ```bash
+    ./eth-statediff-service serve --config=<config path>
+    ```
+
+    Example:
+
+    ```bash
+    ./eth-statediff-service serve --config environments/config.toml
+    ```
+
+* Available RPC methods:
+    * `statediff_stateTrieAt()`
+    * `statediff_streamCodeAndCodeHash()`
+    * `statediff_stateDiffAt()`
+    * `statediff_writeStateDiffAt()`
+    * `statediff_writeStateDiffsInRange()`
+
+    Example:
+
+    ```bash
+    curl -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"statediff_writeStateDiffsInRange","params":['"$BEGIN"', '"$END"', {"intermediateStateNodes":true,"intermediateStorageNodes":true,"includeBlock":true,"includeReceipts":true,"includeTD":true,"includeCode":true}],"id":1}' "$HOST":"$PORT"
+    ```
+
+* Prerun:
+    * The process can be configured locally with sets of ranges to process as a "prerun" to processing directed by the server endpoints.
+    * This is done by turning "prerun" on in the config (`statediff.prerun = true`) and defining ranges and params in the
+`prerun` section of the config.
+    * Set the range using `prerun.start` and `prerun.stop`. Use `prerun.ranges` if prerun on more than one range is required.
+
+* NOTE: Currently, `params.includeTD` must be set to / passed as `true`.
+
+## Monitoring
+
+* Enable metrics using config parameters `prom.metrics` and `prom.http`.
+* `eth-statediff-service` exposes following prometheus metrics at `/metrics` endpoint:
+    * `ranges_queued`: Number of range requests currently queued.
+    * `loaded_height`: The last block that was loaded for processing.
+    * `processed_height`: The last block that was processed.
+    * `stats.t_block_load`: Block loading time.
+    * `stats.t_block_processing`: Block (header, uncles, txs, rcts, tx trie, rct trie) processing time.
+    * `stats.t_state_processing`: State (state trie, storage tries, and code) processing time.
+    * `stats.t_postgres_tx_commit`: Postgres tx commit time.
+    * `http.count`: HTTP request count.
+    * `http.duration`: HTTP request duration.
+    * `ipc.count`: Unix socket connection count.
+
+## Tests
+
+* Run unit tests:
+
+    ```bash
+    make test
+    ```
+
+## Import output data in file mode into a database
+
+* When `eth-statediff-service` is run in file mode (`database.type`) the output is in form of a SQL file or multiple CSV files.
+
+* Assuming the output files are located in host's `./output_dir` directory.
+
+* Create a directory to store post-processed output:
+
+    ```bash
+    mkdir -p output_dir/processed_output
+    ```
+
+### SQL
+
+* De-duplicate data:
+
+    ```bash
+    sort -u output_dir/statediff.sql -o output_dir/processed_output/deduped-statediff.sql
+    ```
+
+* Copy over the post-processed output files to the DB server (say in `/output_dir`).
+
+* Run the following to import data:
+
+    ```bash
+    psql -U <DATABASE_USER> -h <DATABASE_HOSTNAME> -p <DATABASE_PORT> <DATABASE_NAME> --set ON_ERROR_STOP=on -f /output_dir/processed_output/deduped-statediff.sql
+    ```
+
+### CSV
+
+* De-duplicate data and copy to post-processed output directory:
+
+    ```bash
+    # public.blocks
+    sort -u output_dir/public.blocks.csv -o output_dir/processed_output/deduped-public.blocks.csv
+
+    # eth.header_cids
+    sort -u output_dir/eth.header_cids.csv -o output_dir/processed_output/deduped-eth.header_cids.csv
+
+    # eth.uncle_cids
+    sort -u output_dir/eth.uncle_cids.csv -o output_dir/processed_output/deduped-eth.uncle_cids.csv
+
+    # eth.transaction_cids
+    sort -u output_dir/eth.transaction_cids.csv -o output_dir/processed_output/deduped-eth.transaction_cids.csv
+
+    # eth.access_list_elements
+    sort -u output_dir/eth.access_list_elements.csv -o output_dir/processed_output/deduped-eth.access_list_elements.csv
+
+    # eth.receipt_cids
+    sort -u output_dir/eth.receipt_cids.csv -o output_dir/processed_output/deduped-eth.receipt_cids.csv
+
+    # eth.log_cids
+    sort -u output_dir/eth.log_cids.csv -o output_dir/processed_output/deduped-eth.log_cids.csv
+
+    # eth.state_cids
+    sort -u output_dir/eth.state_cids.csv -o output_dir/processed_output/deduped-eth.state_cids.csv
+
+    # eth.storage_cids
+    sort -u output_dir/eth.storage_cids.csv -o output_dir/processed_output/deduped-eth.storage_cids.csv
+
+    # eth.state_accounts
+    sort -u output_dir/eth.state_accounts.csv -o output_dir/processed_output/deduped-eth.state_accounts.csv
+
+    # public.nodes
+    cp output_dir/public.nodes.csv output_dir/processed_output/public.nodes.csv
+    ```
+
+* Copy over the post-processed output files to the DB server (say in `/output_dir`).
+
+* Start `psql` to run the import commands:
+
+    ```bash
+    psql -U <DATABASE_USER> -h <DATABASE_HOSTNAME> -p <DATABASE_PORT> <DATABASE_NAME>
+    ```
+
+* Run the following to import data:
+
+    ```bash
+    # public.nodes
+    COPY public.nodes FROM '/output_dir/processed_output/public.nodes.csv' CSV;
+
+    # public.nodes
+    COPY public.blocks FROM '/output_dir/processed_output/deduped-public.blocks.csv' CSV;
+
+    # eth.header_cids
+    COPY eth.header_cids FROM '/output_dir/processed_output/deduped-eth.header_cids.csv' CSV;
+
+    # eth.uncle_cids
+    COPY eth.uncle_cids FROM '/output_dir/processed_output/deduped-eth.uncle_cids.csv' CSV;
+
+    # eth.transaction_cids
+    COPY eth.transaction_cids FROM '/output_dir/processed_output/deduped-eth.transaction_cids.csv' CSV FORCE NOT NULL dst;
+
+    # eth.access_list_elements
+    COPY eth.access_list_elements FROM '/output_dir/processed_output/deduped-eth.access_list_elements.csv' CSV;
+
+    # eth.receipt_cids
+    COPY eth.receipt_cids FROM '/output_dir/processed_output/deduped-eth.receipt_cids.csv' CSV FORCE NOT NULL post_state, contract, contract_hash;
+
+    # eth.log_cids
+    COPY eth.log_cids FROM '/output_dir/processed_output/deduped-eth.log_cids.csv' CSV FORCE NOT NULL topic0, topic1, topic2, topic3;
+
+    # eth.state_cids
+    COPY eth.state_cids FROM '/output_dir/processed_output/deduped-eth.state_cids.csv' CSV FORCE NOT NULL state_leaf_key;
+
+    # eth.storage_cids
+    COPY eth.storage_cids FROM '/output_dir/processed_output/deduped-eth.storage_cids.csv' CSV FORCE NOT NULL storage_leaf_key;
+
+    # eth.state_accounts
+    COPY eth.state_accounts FROM '/output_dir/processed_output/deduped-eth.state_accounts.csv' CSV;
+    ```
+
+* NOTE: `COPY` command on CSVs inserts empty strings as `NULL` in the DB. Passing `FORCE_NOT_NULL <COLUMN_NAME>` forces it to insert empty strings instead. This is required to maintain compatibility of the imported statediff data with the data generated in `postgres` mode. Reference: https://www.postgresql.org/docs/14/sql-copy.html
@@ -0,0 +1,149 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"github.com/spf13/viper"
+)
+
+const (
+	ETH_NODE_ID       = "ETH_NODE_ID"
+	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
+	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
+	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
+	ETH_CHAIN_ID      = "ETH_CHAIN_ID"
+	ETH_CHAIN_CONFIG  = "ETH_CHAIN_CONFIG"
+
+	DB_CACHE_SIZE_MB   = "DB_CACHE_SIZE_MB"
+	TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
+	LVLDB_MODE         = "LVLDB_MODE"
+	LVLDB_PATH         = "LVLDB_PATH"
+	LVLDB_ANCIENT      = "LVLDB_ANCIENT"
+	LVLDB_URL          = "LVLDB_URL"
+
+	STATEDIFF_PRERUN            = "STATEDIFF_PRERUN"
+	STATEDIFF_TRIE_WORKERS      = "STATEDIFF_TRIE_WORKERS"
+	STATEDIFF_SERVICE_WORKERS   = "STATEDIFF_SERVICE_WORKERS"
+	STATEDIFF_WORKER_QUEUE_SIZE = "STATEDIFF_WORKER_QUEUE_SIZE"
+
+	SERVICE_IPC_PATH  = "SERVICE_IPC_PATH"
+	SERVICE_HTTP_PATH = "SERVICE_HTTP_PATH"
+
+	PROM_METRICS   = "PROM_METRICS"
+	PROM_HTTP      = "PROM_HTTP"
+	PROM_HTTP_ADDR = "PROM_HTTP_ADDR"
+	PROM_HTTP_PORT = "PROM_HTTP_PORT"
+	PROM_DB_STATS  = "PROM_DB_STATS"
+
+	PRERUN_ONLY                       = "PRERUN_ONLY"
+	PRERUN_RANGE_START                = "PRERUN_RANGE_START"
+	PRERUN_RANGE_STOP                 = "PRERUN_RANGE_STOP"
+	PRERUN_INTERMEDIATE_STATE_NODES   = "PRERUN_INTERMEDIATE_STATE_NODES"
+	PRERUN_INTERMEDIATE_STORAGE_NODES = "PRERUN_INTERMEDIATE_STORAGE_NODES"
+	PRERUN_INCLUDE_BLOCK              = "PRERUN_INCLUDE_BLOCK"
+	PRERUN_INCLUDE_RECEIPTS           = "PRERUN_INCLUDE_RECEIPTS"
+	PRERUN_INCLUDE_TD                 = "PRERUN_INCLUDE_TD"
+	PRERUN_INCLUDE_CODE               = "PRERUN_INCLUDE_CODE"
+
+	LOG_LEVEL     = "LOG_LEVEL"
+	LOG_FILE_PATH = "LOG_FILE_PATH"
+
+	DATABASE_NAME     = "DATABASE_NAME"
+	DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
+	DATABASE_PORT     = "DATABASE_PORT"
+	DATABASE_USER     = "DATABASE_USER"
+	DATABASE_PASSWORD = "DATABASE_PASSWORD"
+
+	DATABASE_TYPE         = "DATABASE_TYPE"
+	DATABASE_DRIVER_TYPE  = "DATABASE_DRIVER_TYPE"
+	DATABASE_DUMP_DST     = "DATABASE_DUMP_DST"
+	DATABASE_FILE_PATH    = "DATABASE_FILE_PATH"
+	DATABASE_FILE_MODE    = "DATABASE_FILE_MODE"
+	DATABASE_FILE_CSV_DIR = "DATABASE_FILE_CSV_DIR"
+
+	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
+	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
+	DATABASE_MIN_OPEN_CONNS       = "DATABASE_MIN_OPEN_CONNS"
+	DATABASE_MAX_CONN_LIFETIME    = "DATABASE_MAX_CONN_LIFETIME"
+	DATABASE_CONN_TIMEOUT         = "DATABSE_CONN_TIMEOUT"
+	DATABASE_MAX_CONN_IDLE_TIME   = "DATABASE_MAX_CONN_IDLE_TIME"
+)
+
+// Bind env vars for eth node and DB configuration
+func init() {
+	viper.BindEnv("server.ipcPath", SERVICE_IPC_PATH)
+	viper.BindEnv("server.httpPath", SERVICE_HTTP_PATH)
+
+	viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
+	viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
+	viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
+	viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
+	viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
+	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
+
+	viper.BindEnv("database.name", DATABASE_NAME)
+	viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
+	viper.BindEnv("database.port", DATABASE_PORT)
+	viper.BindEnv("database.user", DATABASE_USER)
+	viper.BindEnv("database.password", DATABASE_PASSWORD)
+
+	viper.BindEnv("database.maxIdle", DATABASE_MAX_IDLE_CONNECTIONS)
+	viper.BindEnv("database.maxOpen", DATABASE_MAX_OPEN_CONNECTIONS)
+	viper.BindEnv("database.minOpen", DATABASE_MIN_OPEN_CONNS)
+	viper.BindEnv("database.maxConnLifetime", DATABASE_MAX_CONN_LIFETIME)
+	viper.BindEnv("database.connTimeout", DATABASE_CONN_TIMEOUT)
+	viper.BindEnv("database.maxIdleTime", DATABASE_MAX_CONN_IDLE_TIME)
+
+	viper.BindEnv("database.type", DATABASE_TYPE)
+	viper.BindEnv("database.driver", DATABASE_DRIVER_TYPE)
+	viper.BindEnv("database.dumpDestination", DATABASE_DUMP_DST)
+	viper.BindEnv("database.fileMode", DATABASE_FILE_MODE)
+	viper.BindEnv("database.filePath", DATABASE_FILE_PATH)
+	viper.BindEnv("database.fileCsvDir", DATABASE_FILE_CSV_DIR)
+
+	viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
+	viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)
+
+	viper.BindEnv("leveldb.mode", LVLDB_MODE)
+	viper.BindEnv("leveldb.path", LVLDB_PATH)
+	viper.BindEnv("leveldb.ancient", LVLDB_ANCIENT)
+	viper.BindEnv("leveldb.url", LVLDB_URL)
+
+	viper.BindEnv("prom.metrics", PROM_METRICS)
+	viper.BindEnv("prom.http", PROM_HTTP)
+	viper.BindEnv("prom.httpAddr", PROM_HTTP_ADDR)
+	viper.BindEnv("prom.httpPort", PROM_HTTP_PORT)
+	viper.BindEnv("prom.dbStats", PROM_DB_STATS)
+
+	viper.BindEnv("statediff.serviceWorkers", STATEDIFF_SERVICE_WORKERS)
+	viper.BindEnv("statediff.trieWorkers", STATEDIFF_TRIE_WORKERS)
+	viper.BindEnv("statediff.workerQueueSize", STATEDIFF_WORKER_QUEUE_SIZE)
+
+	viper.BindEnv("statediff.prerun", STATEDIFF_PRERUN)
+	viper.BindEnv("prerun.only", PRERUN_ONLY)
+	viper.BindEnv("prerun.start", PRERUN_RANGE_START)
+	viper.BindEnv("prerun.stop", PRERUN_RANGE_STOP)
+	viper.BindEnv("prerun.params.intermediateStateNodes", PRERUN_INTERMEDIATE_STATE_NODES)
+	viper.BindEnv("prerun.params.intermediateStorageNodes", PRERUN_INTERMEDIATE_STORAGE_NODES)
+	viper.BindEnv("prerun.params.includeBlock", PRERUN_INCLUDE_BLOCK)
+	viper.BindEnv("prerun.params.includeReceipts", PRERUN_INCLUDE_RECEIPTS)
+	viper.BindEnv("prerun.params.includeTD", PRERUN_INCLUDE_TD)
+	viper.BindEnv("prerun.params.includeCode", PRERUN_INCLUDE_CODE)
+
+	viper.BindEnv("log.level", LOG_LEVEL)
+	viper.BindEnv("log.file", LOG_FILE_PATH)
+}
@@ -18,12 +18,23 @@ package cmd

 import (
 	"fmt"
+	"math/rand"
 	"os"
 	"strings"
+	"time"

+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/dump"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 var (
@@ -45,7 +56,7 @@ func Execute() {
 }

 func initFuncs(cmd *cobra.Command, args []string) {
-	logfile := viper.GetString("logfile")
+	logfile := viper.GetString("log.file")
 	if logfile != "" {
 		file, err := os.OpenFile(logfile,
 			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
@@ -62,10 +73,24 @@ func initFuncs(cmd *cobra.Command, args []string) {
 	if err := logLevel(); err != nil {
 		log.Fatal("Could not set log level: ", err)
 	}
+
+	if viper.GetBool("prom.metrics") {
+		log.Info("initializing prometheus metrics")
+		prom.Init()
+	}
+
+	if viper.GetBool("prom.http") {
+		addr := fmt.Sprintf(
+			"%s:%s",
+			viper.GetString("prom.httpAddr"),
+			viper.GetString("prom.httpPort"),
+		)
+		log.Info("starting prometheus server")
+		prom.Listen(addr)
+	}
 }

 func logLevel() error {
-	viper.BindEnv("log.level", "LOGRUS_LEVEL")
 	lvl, err := log.ParseLevel(viper.GetString("log.level"))
 	if err != nil {
 		return err
@@ -83,12 +108,129 @@ func init() {
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	viper.AutomaticEnv()

+	rootCmd.PersistentFlags().String("http-path", "", "vdb server http path")
+	rootCmd.PersistentFlags().String("ipc-path", "", "vdb server ipc path")
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
+
 	rootCmd.PersistentFlags().String("log-file", "", "file path for logging")
-	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic")
+	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(),
+		"log level (trace, debug, info, warn, error, fatal, panic")
+
+	rootCmd.PersistentFlags().String("leveldb-mode", "local", "LevelDB access mode (local, remote)")
+	rootCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
+	rootCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
+	rootCmd.PersistentFlags().String("leveldb-url", "", "url to primary leveldb-ethdb-rpc server")
+
+	rootCmd.PersistentFlags().Bool("prerun", false, "turn on prerun of toml configured ranges")
+	rootCmd.PersistentFlags().Int("service-workers", 0, "number of range requests to process concurrently")
+	rootCmd.PersistentFlags().Int("trie-workers", 0, "number of workers to use for trie traversal and processing")
+	rootCmd.PersistentFlags().Int("worker-queue-size", 0, "size of the range request queue for service workers")
+
+	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
+	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
+	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
+	rootCmd.PersistentFlags().String("database-user", "", "database user")
+	rootCmd.PersistentFlags().String("database-password", "", "database password")
+	rootCmd.PersistentFlags().Int("database-max-idle", 0, "database max number of idle connections")
+	rootCmd.PersistentFlags().Int("database-max-open", 0, "database max number of open connections")
+	rootCmd.PersistentFlags().Int("database-min-open", 0, "database min number of open connections")
+	rootCmd.PersistentFlags().Duration("database-max-conn-lifetime", 0, "database max connection lifetime")
+	rootCmd.PersistentFlags().Duration("database-conn-timeout", 0, "database connection timeout")
+	rootCmd.PersistentFlags().Duration("database-max-idle-time", 0, "database max connection idle time")
+	rootCmd.PersistentFlags().String("database-type", "postgres", "database type (currently supported: postgres, dump)")
+	rootCmd.PersistentFlags().String("database-driver", "sqlx", "database driver type (currently supported: sqlx, pgx)")
+	rootCmd.PersistentFlags().String("database-dump-dst", "stdout", "dump destination (for database-type=dump; options: stdout, stderr, discard)")
+	rootCmd.PersistentFlags().String("database-file-mode", "csv", "mode for writing file (for database-type=file; options: csv, sql)")
+	rootCmd.PersistentFlags().String("database-file-csv-dir", "", "full directory path (for database-file-mode=csv)")
+	rootCmd.PersistentFlags().String("database-file-path", "", "full file path (for database-file-mode=sql)")
+
+	rootCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
+	rootCmd.PersistentFlags().String("eth-client-name", "eth-statediff-service", "eth client name")
+	rootCmd.PersistentFlags().String("eth-genesis-block",
+		"0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", "eth genesis block hash")
+	rootCmd.PersistentFlags().String("eth-network-id", "1", "eth network id")
+	rootCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
+
+	rootCmd.PersistentFlags().Int("cache-db", 1024, "megabytes of memory allocated to database cache")
+	rootCmd.PersistentFlags().Int("cache-trie", 1024, "Megabytes of memory allocated to trie cache")
+
+	rootCmd.PersistentFlags().Bool("prom-http", false, "enable prometheus http service")
+	rootCmd.PersistentFlags().String("prom-http-addr", "127.0.0.1", "prometheus http host")
+	rootCmd.PersistentFlags().String("prom-http-port", "8080", "prometheus http port")
+	rootCmd.PersistentFlags().Bool("prom-db-stats", false, "enables prometheus db stats")
+	rootCmd.PersistentFlags().Bool("prom-metrics", false, "enable prometheus metrics")
+
+	rootCmd.PersistentFlags().Bool("prerun-only", false, "only process pre-configured ranges; exit afterwards")
+	rootCmd.PersistentFlags().Int("prerun-start", 0, "start height for a prerun range")
+	rootCmd.PersistentFlags().Int("prerun-stop", 0, "stop height for a prerun range")
+	rootCmd.PersistentFlags().Bool("prerun-intermediate-state-nodes", true, "include intermediate state nodes in state diff")
+	rootCmd.PersistentFlags().Bool("prerun-intermediate-storage-nodes", true, "include intermediate storage nodes in state diff")
+	rootCmd.PersistentFlags().Bool("prerun-include-block", true, "include block data in the statediff payload")
+	rootCmd.PersistentFlags().Bool("prerun-include-receipts", true, "include receipts in the statediff payload")
+	rootCmd.PersistentFlags().Bool("prerun-include-td", true, "include td in the statediff payload")
+	rootCmd.PersistentFlags().Bool("prerun-include-code", true, "include code and codehash mappings in statediff payload")
+
+	viper.BindPFlag("server.httpPath", rootCmd.PersistentFlags().Lookup("http-path"))
+	viper.BindPFlag("server.ipcPath", rootCmd.PersistentFlags().Lookup("ipc-path"))

 	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
+
+	viper.BindPFlag("statediff.prerun", rootCmd.PersistentFlags().Lookup("prerun"))
+	viper.BindPFlag("statediff.serviceWorkers", rootCmd.PersistentFlags().Lookup("service-workers"))
+	viper.BindPFlag("statediff.trieWorkers", rootCmd.PersistentFlags().Lookup("trie-workers"))
+	viper.BindPFlag("statediff.workerQueueSize", rootCmd.PersistentFlags().Lookup("worker-queue-size"))
+
+	viper.BindPFlag("leveldb.mode", rootCmd.PersistentFlags().Lookup("leveldb-mode"))
+	viper.BindPFlag("leveldb.path", rootCmd.PersistentFlags().Lookup("leveldb-path"))
+	viper.BindPFlag("leveldb.ancient", rootCmd.PersistentFlags().Lookup("ancient-path"))
+	viper.BindPFlag("leveldb.url", rootCmd.PersistentFlags().Lookup("leveldb-url"))
+
+	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
+	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
+	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
+	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
+	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
+	viper.BindPFlag("database.maxIdle", rootCmd.PersistentFlags().Lookup("database-max-idle"))
+	viper.BindPFlag("database.maxOpen", rootCmd.PersistentFlags().Lookup("database-max-open"))
+	viper.BindPFlag("database.minOpen", rootCmd.PersistentFlags().Lookup("database-min-open"))
+	viper.BindPFlag("database.maxConnLifetime", rootCmd.PersistentFlags().Lookup("database-max-conn-lifetime"))
+	viper.BindPFlag("database.connTimeout", rootCmd.PersistentFlags().Lookup("database-conn-timeout"))
+	viper.BindPFlag("database.maxIdleTime", rootCmd.PersistentFlags().Lookup("database-max-idle-time"))
+	viper.BindPFlag("database.type", rootCmd.PersistentFlags().Lookup("database-type"))
+	viper.BindPFlag("database.driver", rootCmd.PersistentFlags().Lookup("database-driver"))
+	viper.BindPFlag("database.dumpDestination", rootCmd.PersistentFlags().Lookup("database-dump-dst"))
+	viper.BindPFlag("database.fileMode", rootCmd.PersistentFlags().Lookup("database-file-mode"))
+	viper.BindPFlag("database.fileCsvDir", rootCmd.PersistentFlags().Lookup("database-file-csv-dir"))
+	viper.BindPFlag("database.filePath", rootCmd.PersistentFlags().Lookup("database-file-path"))
+
+	viper.BindPFlag("ethereum.nodeID", rootCmd.PersistentFlags().Lookup("eth-node-id"))
+	viper.BindPFlag("ethereum.clientName", rootCmd.PersistentFlags().Lookup("eth-client-name"))
+	viper.BindPFlag("ethereum.genesisBlock", rootCmd.PersistentFlags().Lookup("eth-genesis-block"))
+	viper.BindPFlag("ethereum.networkID", rootCmd.PersistentFlags().Lookup("eth-network-id"))
+	viper.BindPFlag("ethereum.chainID", rootCmd.PersistentFlags().Lookup("eth-chain-id"))
+	viper.BindPFlag("ethereum.chainConfig", rootCmd.PersistentFlags().Lookup("eth-chain-config"))
+
+	viper.BindPFlag("cache.database", rootCmd.PersistentFlags().Lookup("cache-db"))
+	viper.BindPFlag("cache.trie", rootCmd.PersistentFlags().Lookup("cache-trie"))
+
+	viper.BindPFlag("prom.http", rootCmd.PersistentFlags().Lookup("prom-http"))
+	viper.BindPFlag("prom.httpAddr", rootCmd.PersistentFlags().Lookup("prom-http-addr"))
+	viper.BindPFlag("prom.httpPort", rootCmd.PersistentFlags().Lookup("prom-http-port"))
+	viper.BindPFlag("prom.dbStats", rootCmd.PersistentFlags().Lookup("prom-db-stats"))
+	viper.BindPFlag("prom.metrics", rootCmd.PersistentFlags().Lookup("prom-metrics"))
+
+	viper.BindPFlag("prerun.only", rootCmd.PersistentFlags().Lookup("prerun-only"))
+	viper.BindPFlag("prerun.start", rootCmd.PersistentFlags().Lookup("prerun-start"))
+	viper.BindPFlag("prerun.stop", rootCmd.PersistentFlags().Lookup("prerun-stop"))
+	viper.BindPFlag("prerun.params.intermediateStateNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-state-nodes"))
+	viper.BindPFlag("prerun.params.intermediateStorageNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-storage-nodes"))
+	viper.BindPFlag("prerun.params.includeBlock", rootCmd.PersistentFlags().Lookup("prerun-include-block"))
+	viper.BindPFlag("prerun.params.includeReceipts", rootCmd.PersistentFlags().Lookup("prerun-include-receipts"))
+	viper.BindPFlag("prerun.params.includeTD", rootCmd.PersistentFlags().Lookup("prerun-include-td"))
+	viper.BindPFlag("prerun.params.includeCode", rootCmd.PersistentFlags().Lookup("prerun-include-code"))
+
+	rand.Seed(time.Now().UnixNano())
 }

 func initConfig() {
@@ -103,3 +245,144 @@ func initConfig() {
 		log.Warn("No config file passed with --config flag")
 	}
 }
+
+func getEthNodeInfo() node.Info {
+	var nodeID, genesisBlock, networkID, clientName string
+	var chainID uint64
+	if !viper.IsSet("ethereum.nodeID") {
+		nodeID = randSeq(12)
+	} else {
+		nodeID = viper.GetString("ethereum.nodeID")
+	}
+	genesisBlock = viper.GetString("ethereum.genesisBlock")
+	if !viper.IsSet("ethereum.genesisBlock") {
+		genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
+	} else {
+		genesisBlock = viper.GetString("ethereum.genesisBlock")
+	}
+	if !viper.IsSet("ethereum.chainID") {
+		chainID = 1
+	} else {
+		chainID = viper.GetUint64("ethereum.chainID")
+	}
+	if !viper.IsSet("ethereum.networkID") {
+		networkID = "1"
+	} else {
+		networkID = viper.GetString("ethereum.networkID")
+	}
+	if !viper.IsSet("ethereum.clientName") {
+		clientName = "eth-statediff-service"
+	} else {
+		clientName = viper.GetString("ethereum.clientName")
+	}
+	return node.Info{
+		ID:           nodeID,
+		ClientName:   clientName,
+		GenesisBlock: genesisBlock,
+		NetworkID:    networkID,
+		ChainID:      chainID,
+	}
+}
+
+var characters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
+
+func randSeq(n int) string {
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = characters[rand.Intn(len(characters))]
+	}
+	return string(b)
+}
+
+// getConfig constructs and returns the appropriate config from viper params
+func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
+	dbTypeStr := viper.GetString("database.type")
+	dbType, err := shared.ResolveDBType(dbTypeStr)
+	if err != nil {
+		return nil, err
+	}
+	logWithCommand.Infof("configuring service for database type: %s", dbType)
+	var indexerConfig interfaces.Config
+	switch dbType {
+	case shared.FILE:
+		logWithCommand.Info("starting in sql file writing mode")
+
+		fileModeStr := viper.GetString("database.fileMode")
+		fileMode, err := file.ResolveFileMode(fileModeStr)
+		if err != nil {
+			utils.Fatalf("%v", err)
+		}
+
+		filePathStr := viper.GetString("database.filePath")
+		if fileMode == file.SQL && filePathStr == "" {
+			logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided")
+		}
+
+		fileCsvDirStr := viper.GetString("database.fileCsvDir")
+		if fileMode == file.CSV && fileCsvDirStr == "" {
+			logWithCommand.Fatal("when operating in csv file writing mode a directory path must be provided")
+		}
+
+		indexerConfig = file.Config{
+			Mode:      fileMode,
+			OutputDir: fileCsvDirStr,
+			FilePath:  filePathStr,
+		}
+	case shared.DUMP:
+		logWithCommand.Info("starting in data dump mode")
+		dumpDstStr := viper.GetString("database.dumpDestination")
+		dumpDst, err := dump.ResolveDumpType(dumpDstStr)
+		if err != nil {
+			return nil, err
+		}
+		switch dumpDst {
+		case dump.STDERR:
+			indexerConfig = dump.Config{Dump: os.Stdout}
+		case dump.STDOUT:
+			indexerConfig = dump.Config{Dump: os.Stderr}
+		case dump.DISCARD:
+			indexerConfig = dump.Config{Dump: dump.NewDiscardWriterCloser()}
+		default:
+			return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst)
+		}
+	case shared.POSTGRES:
+		logWithCommand.Info("starting in postgres mode")
+		driverTypeStr := viper.GetString("database.driver")
+		driverType, err := postgres.ResolveDriverType(driverTypeStr)
+		if err != nil {
+			utils.Fatalf("%v", err)
+		}
+		pgConfig := postgres.Config{
+			Hostname:     viper.GetString("database.hostname"),
+			Port:         viper.GetInt("database.port"),
+			DatabaseName: viper.GetString("database.name"),
+			Username:     viper.GetString("database.user"),
+			Password:     viper.GetString("database.password"),
+			ID:           nodeInfo.ID,
+			ClientName:   nodeInfo.ClientName,
+			Driver:       driverType,
+		}
+		if viper.IsSet("database.maxIdle") {
+			pgConfig.MaxIdle = viper.GetInt("database.maxIdle")
+		}
+		if viper.IsSet("database.maxOpen") {
+			pgConfig.MaxConns = viper.GetInt("database.maxOpen")
+		}
+		if viper.IsSet("database.minOpen") {
+			pgConfig.MinConns = viper.GetInt("database.minOpen")
+		}
+		if viper.IsSet("database.maxConnLifetime") {
+			pgConfig.MaxConnLifetime = viper.GetDuration("database.maxConnLifetime")
+		}
+		if viper.IsSet("database.connTimeout") {
+			pgConfig.ConnTimeout = viper.GetDuration("database.connTimeout")
+		}
+		if viper.IsSet("database.maxIdleTime") {
+			pgConfig.MaxConnIdleTime = viper.GetDuration("database.maxIdleTime")
+		}
+		indexerConfig = pgConfig
+	default:
+		return nil, fmt.Errorf("unrecognized db type: %s", dbType)
+	}
+	return indexerConfig, nil
+}
@@ -16,25 +16,23 @@
 package cmd

 import (
-	"fmt"
 	"os"
 	"os/signal"
 	"sync"

-	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/statediff"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"

-	sd "github.com/vulcanize/eth-statediff-service/pkg"
+	sd "github.com/cerc-io/eth-statediff-service/pkg"
+	srpc "github.com/cerc-io/eth-statediff-service/pkg/rpc"
 )

 // serveCmd represents the serve command
 var serveCmd = &cobra.Command{
 	Use:   "serve",
-	Short: "Standup a standalone statediffing RPC service on top of leveldb",
+	Short: "Stand up a standalone statediffing RPC service on top of LevelDB",
 	Long: `Usage

 ./eth-statediff-service serve --config={path to toml config file}`,
@@ -45,71 +43,48 @@ var serveCmd = &cobra.Command{
 	},
 }

+func init() {
+	rootCmd.AddCommand(serveCmd)
+}
+
 func serve() {
-	logWithCommand.Info("starting statediff RPC service")
+	logWithCommand.Info("Running eth-statediff-service serve command")

-	// load params
-	path := viper.GetString("leveldb.path")
-	ancientPath := viper.GetString("leveldb.ancient")
-	if path == "" || ancientPath == "" {
-		logWithCommand.Fatal("require a valid eth leveldb primary datastore path and ancient datastore path")
-	}
-	chainID := viper.GetUint64("eth.chainID")
-	config, err := chainConfig(chainID)
+	statediffService, err := createStateDiffService()
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}

-	// create leveldb reader
-	logWithCommand.Info("creating leveldb reader")
-	lvlDBReader, err := sd.NewLvlDBReader(path, ancientPath, config)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-
-	// create statediff service
-	logWithCommand.Info("creating statediff service")
-	statediffService, err := sd.NewStateDiffService(lvlDBReader)
-	if err != nil {
-		logWithCommand.Fatal(err)
+	// short circuit if we only want to perform prerun
+	if viper.GetBool("prerun.only") {
+		if err := statediffService.Run(nil); err != nil {
+			logWithCommand.Fatal("unable to perform prerun: %v", err)
+		}
+		return
 	}

 	// start service and servers
-	logWithCommand.Info("starting statediff service")
+	logWithCommand.Info("Starting statediff service")
 	wg := new(sync.WaitGroup)
-	go statediffService.Loop(wg)
-	logWithCommand.Info("starting rpc servers")
+	if err := statediffService.Loop(wg); err != nil {
+		logWithCommand.Fatalf("unable to start statediff service: %v", err)
+	}
+	logWithCommand.Info("Starting RPC servers")
 	if err := startServers(statediffService); err != nil {
 		logWithCommand.Fatal(err)
 	}
-	logWithCommand.Info("rpc servers successfully spun up; awaiting requests")
+	logWithCommand.Info("RPC servers successfully spun up; awaiting requests")

 	// clean shutdown
 	shutdown := make(chan os.Signal)
 	signal.Notify(shutdown, os.Interrupt)
 	<-shutdown
-	logWithCommand.Info("received interrupt signal, shutting down")
+	logWithCommand.Info("Received interrupt signal, shutting down")
 	statediffService.Stop()
 	wg.Wait()
 }

-func init() {
-	rootCmd.AddCommand(serveCmd)
-
-	serveCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
-	serveCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
-	serveCmd.PersistentFlags().Uint("chain-id", 1, "ethereum chain id (mainnet = 1)")
-	serveCmd.PersistentFlags().String("http-path", "", "vdb server http path")
-	serveCmd.PersistentFlags().String("ipc-path", "", "vdb server ipc path")
-
-	viper.BindPFlag("leveldb.path", serveCmd.PersistentFlags().Lookup("leveldb-path"))
-	viper.BindPFlag("leveldb.ancient", serveCmd.PersistentFlags().Lookup("ancient-path"))
-	viper.BindPFlag("eth.chainID", serveCmd.PersistentFlags().Lookup("chain-id"))
-	viper.BindPFlag("server.httpPath", serveCmd.PersistentFlags().Lookup("http-path"))
-	viper.BindPFlag("server.ipcPath", serveCmd.PersistentFlags().Lookup("ipc-path"))
-}
-
-func startServers(serv sd.IService) error {
+func startServers(serv sd.StateDiffService) error {
 	ipcPath := viper.GetString("server.ipcPath")
 	httpPath := viper.GetString("server.httpPath")
 	if ipcPath == "" && httpPath == "" {
@@ -117,31 +92,20 @@ func startServers(serv sd.IService) error {
 	}
 	if ipcPath != "" {
 		logWithCommand.Info("starting up IPC server")
-		_, _, err := rpc.StartIPCEndpoint(ipcPath, serv.APIs())
+		_, _, err := srpc.StartIPCEndpoint(ipcPath, serv.APIs())
 		if err != nil {
 			return err
 		}
 	}
 	if httpPath != "" {
 		logWithCommand.Info("starting up HTTP server")
-		if _, _, err := rpc.StartHTTPEndpoint(httpPath, serv.APIs(), []string{statediff.APIName}, nil, nil, rpc.HTTPTimeouts{}); err != nil {
+		_, err := srpc.StartHTTPEndpoint(httpPath, serv.APIs(), []string{"statediff"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
+		if err != nil {
 			return err
 		}
+	} else {
+		logWithCommand.Info("HTTP server is disabled")
 	}
+
 	return nil
 }
-
-func chainConfig(chainID uint64) (*params.ChainConfig, error) {
-	switch chainID {
-	case 1:
-		return params.MainnetChainConfig, nil
-	case 3:
-		return params.TestnetChainConfig, nil // Ropsten
-	case 4:
-		return params.RinkebyChainConfig, nil
-	case 5:
-		return params.GoerliChainConfig, nil
-	default:
-		return nil, fmt.Errorf("chain config for chainid %d not available", chainID)
-	}
-}
@@ -0,0 +1,142 @@
+package cmd
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff"
+	ind "github.com/ethereum/go-ethereum/statediff/indexer"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/spf13/viper"
+
+	sd "github.com/cerc-io/eth-statediff-service/pkg"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
+)
+
+type blockRange [2]uint64
+
+func createStateDiffService() (sd.StateDiffService, error) {
+	// load some necessary params
+	logWithCommand.Info("Loading statediff service parameters")
+	mode := viper.GetString("leveldb.mode")
+	path := viper.GetString("leveldb.path")
+	ancientPath := viper.GetString("leveldb.ancient")
+	url := viper.GetString("leveldb.url")
+
+	if mode == "local" {
+		if path == "" || ancientPath == "" {
+			logWithCommand.Fatal("Require a valid eth LevelDB primary datastore path and ancient datastore path")
+		}
+	} else if mode == "remote" {
+		if url == "" {
+			logWithCommand.Fatal("Require a valid RPC url for accessing LevelDB")
+		}
+	} else {
+		logWithCommand.Fatal("Invalid mode provided for LevelDB access")
+	}
+
+	nodeInfo := getEthNodeInfo()
+
+	var chainConf *params.ChainConfig
+	var err error
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+
+	if chainConfigPath != "" {
+		chainConf, err = statediff.LoadConfig(chainConfigPath)
+	} else {
+		chainConf, err = statediff.ChainConfig(nodeInfo.ChainID)
+	}
+
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	// create LevelDB reader
+	logWithCommand.Info("Creating LevelDB reader")
+	readerConf := sd.LvLDBReaderConfig{
+		TrieConfig: &trie.Config{
+			Cache:     viper.GetInt("cache.trie"),
+			Journal:   "",
+			Preimages: false,
+		},
+		ChainConfig: chainConf,
+		Mode:        mode,
+		Path:        path,
+		AncientPath: ancientPath,
+		Url:         url,
+		DBCacheSize: viper.GetInt("cache.database"),
+	}
+	lvlDBReader, err := sd.NewLvlDBReader(readerConf)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	// create statediff service
+	logWithCommand.Info("Setting up database")
+	conf, err := getConfig(nodeInfo)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	logWithCommand.Info("Creating statediff indexer")
+	db, indexer, err := ind.NewStateDiffIndexer(context.Background(), chainConf, nodeInfo, conf)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	if conf.Type() == shared.POSTGRES && viper.GetBool("prom.dbStats") {
+		prom.RegisterDBCollector(viper.GetString("database.name"), db)
+	}
+
+	logWithCommand.Info("Creating statediff service")
+	sdConf := sd.Config{
+		ServiceWorkers:  viper.GetUint("statediff.serviceWorkers"),
+		TrieWorkers:     viper.GetUint("statediff.trieWorkers"),
+		WorkerQueueSize: viper.GetUint("statediff.workerQueueSize"),
+		PreRuns:         setupPreRunRanges(),
+	}
+	return sd.NewStateDiffService(lvlDBReader, indexer, sdConf)
+}
+
+func setupPreRunRanges() []sd.RangeRequest {
+	if !viper.GetBool("statediff.prerun") {
+		return nil
+	}
+	preRunParams := statediff.Params{
+		IntermediateStateNodes:   viper.GetBool("prerun.params.intermediateStateNodes"),
+		IntermediateStorageNodes: viper.GetBool("prerun.params.intermediateStorageNodes"),
+		IncludeBlock:             viper.GetBool("prerun.params.includeBlock"),
+		IncludeReceipts:          viper.GetBool("prerun.params.includeReceipts"),
+		IncludeTD:                viper.GetBool("prerun.params.includeTD"),
+		IncludeCode:              viper.GetBool("prerun.params.includeCode"),
+	}
+	var addrStrs []string
+	viper.UnmarshalKey("prerun.params.watchedAddresses", &addrStrs)
+	addrs := make([]common.Address, len(addrStrs))
+	for i, addrStr := range addrStrs {
+		addrs[i] = common.HexToAddress(addrStr)
+	}
+	preRunParams.WatchedAddresses = addrs
+	var rawRanges []blockRange
+	viper.UnmarshalKey("prerun.ranges", &rawRanges)
+	blockRanges := make([]sd.RangeRequest, len(rawRanges))
+	for i, rawRange := range rawRanges {
+		blockRanges[i] = sd.RangeRequest{
+			Start:  rawRange[0],
+			Stop:   rawRange[1],
+			Params: preRunParams,
+		}
+	}
+	if viper.IsSet("prerun.start") && viper.IsSet("prerun.stop") {
+		hardStart := viper.GetInt("prerun.start")
+		hardStop := viper.GetInt("prerun.stop")
+		blockRanges = append(blockRanges, sd.RangeRequest{
+			Start:  uint64(hardStart),
+			Stop:   uint64(hardStop),
+			Params: preRunParams,
+		})
+	}
+
+	return blockRanges
+}
@@ -0,0 +1,42 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	v "github.com/cerc-io/eth-statediff-service/version"
+)
+
+// versionCmd represents the version command
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Prints the version of eth-statediff-service",
+	Long: `Use this command to fetch the version of eth-statediff-service
+
+Usage: ./eth-statediff-service  version`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *log.WithField("SubCommand", subCommand)
+		logWithCommand.Infof("eth-statediff-service version: %s", v.Version)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(versionCmd)
+}
@@ -6,7 +6,7 @@ services:
      context: ./
      cache_from:
      - alpine:latest
-      - golang:1.13
+      - golang:1.16
      dockerfile: ./Dockerfile
      args:
        USER: "vdbm"
@@ -0,0 +1,58 @@
+[leveldb]
+    mode = "local"
+    path = "/app/geth-rw/chaindata"
+    ancient = "/app/geth-rw/chaindata/ancient"
+
+[server]
+    ipcPath = ""
+    httpPath = "0.0.0.0:8545"
+
+[statediff]
+    prerun = true
+    serviceWorkers = 1
+    workerQueueSize = 1024
+    trieWorkers = 16
+
+[prerun]
+    only = true
+    ranges = []
+    [prerun.params]
+        intermediateStateNodes = true
+        intermediateStorageNodes = true
+        includeBlock = true
+        includeReceipts = true
+        includeTD = true
+        includeCode = true
+        watchedAddresses = []
+
+[log]
+    file = ""
+    level = "info"
+
+[database]
+    type = "postgres"
+    name     = ""
+    hostname = ""
+    port     = 5432
+    user     = ""
+    password = ""
+    driver = "sqlx"
+
+[cache]
+    database = 1024
+    trie = 4096
+
+[prom]
+    metrics = true
+    http = true
+    httpAddr = "0.0.0.0"
+    httpPort = 9100
+    dbStats = false
+
+[ethereum]
+    nodeID = ""
+    clientName = "eth-statediff-service"
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
+    networkID = 1
+    chainID = 1
+    chainConfig = ""
@@ -1,14 +1,63 @@
 [leveldb]
+    mode = "local"
    path = "/Users/user/Library/Ethereum/geth/chaindata"
    ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
+    url = "http://127.0.0.1:8082/"

 [server]
    ipcPath = ".ipc"
    httpPath = "127.0.0.1:8545"

+[statediff]
+    prerun = true
+    serviceWorkers = 1
+    workerQueueSize = 1024
+    trieWorkers = 4
+
+[prerun]
+    only = false
+    ranges = [
+        [0, 1000]
+    ]
+    [prerun.params]
+        intermediateStateNodes = true
+        intermediateStorageNodes = true
+        includeBlock = true
+        includeReceipts = true
+        includeTD = true
+        includeCode = true
+        watchedAddresses = []
+
 [log]
    file = ""
    level = "info"

-[eth]
-    chainID = 1
+[database]
+    name     = "vulcanize_test"
+    hostname = "localhost"
+    port     = 5432
+    user     = "vulcanize"
+    password = "..."
+    type = "postgres"
+    driver = "sqlx"
+    dumpDestination = ""
+    filePath = ""
+
+[cache]
+    database = 1024
+    trie = 1024
+
+[prom]
+    dbStats = false
+    metrics = true
+    http = true
+    httpAddr = "localhost"
+    httpPort = "8889"
+
+[ethereum]
+    chainConfig = ""
+    nodeID = ""
+    clientName = "eth-statediff-service"
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
+    networkID = 1
+    chainID = 1
@@ -1,12 +1,138 @@
-module github.com/vulcanize/eth-statediff-service
+module github.com/cerc-io/eth-statediff-service

-go 1.13
+go 1.18

 require (
-	github.com/ethereum/go-ethereum v1.9.11
-	github.com/sirupsen/logrus v1.6.0
-	github.com/spf13/cobra v1.0.0
-	github.com/spf13/viper v1.7.1
+	github.com/cerc-io/go-eth-state-node-iterator v1.1.7
+	github.com/ethereum/go-ethereum v1.10.23
+	github.com/jmoiron/sqlx v1.2.0 // indirect
+	github.com/prometheus/client_golang v1.4.0
+	github.com/sirupsen/logrus v1.9.0
+	github.com/spf13/cobra v1.3.0
+	github.com/spf13/viper v1.10.1
+	github.com/vulcanize/leveldb-ethdb-rpc v0.1.7
 )

-replace github.com/ethereum/go-ethereum v1.9.11 => github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5
+require (
+	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deckarep/golang-set v1.8.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/deepmap/oapi-codegen v1.8.2 // indirect
+	github.com/edsrzf/mmap-go v1.0.0 // indirect
+	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
+	github.com/georgysavva/scany v0.2.9 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/graph-gophers/graphql-go v1.3.0 // indirect
+	github.com/hashicorp/go-bexpr v0.1.10 // indirect
+	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
+	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/huin/goupnp v1.0.3 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/influxdata/influxdb v1.8.3 // indirect
+	github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
+	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-block-format v0.0.3 // indirect
+	github.com/ipfs/go-cid v0.2.0 // indirect
+	github.com/ipfs/go-datastore v0.5.1 // indirect
+	github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
+	github.com/ipfs/go-ipld-format v0.4.0 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.1.3 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.10.0 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.1.1 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgtype v1.8.1 // indirect
+	github.com/jackc/pgx/v4 v4.13.0 // indirect
+	github.com/jackc/puddle v1.1.3 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/lib/pq v1.10.6 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/mitchellh/pointerstructure v1.2.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.0.3 // indirect
+	github.com/multiformats/go-base36 v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.0.3 // indirect
+	github.com/multiformats/go-multihash v0.1.0 // indirect
+	github.com/multiformats/go-varint v0.0.6 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
+	github.com/pganalyze/pg_query_go/v2 v2.1.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.9.1 // indirect
+	github.com/prometheus/procfs v0.0.8 // indirect
+	github.com/prometheus/tsdb v0.10.0 // indirect
+	github.com/rjeczalik/notify v0.9.1 // indirect
+	github.com/rs/cors v1.7.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.7.2 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
+	github.com/thoas/go-funk v0.9.2 // indirect
+	github.com/tklauser/go-sysconf v0.3.6 // indirect
+	github.com/tklauser/numcpus v0.2.2 // indirect
+	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
+	github.com/urfave/cli/v2 v2.10.2 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	go.uber.org/zap v1.17.0 // indirect
+	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
+	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/ini.v1 v1.66.2 // indirect
+	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	lukechampine.com/blake3 v1.1.6 // indirect
+)
+
+replace github.com/ethereum/go-ethereum v1.10.23 => github.com/cerc-io/go-ethereum v1.10.23-statediff-4.2.0-alpha
@@ -15,7 +15,7 @@

 package main

-import "github.com/vulcanize/eth-statediff-service/cmd"
+import "github.com/cerc-io/eth-statediff-service/cmd"

 func main() {
 	cmd.Execute()
@@ -0,0 +1,449 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 4,
+  "iteration": 1635774999288,
+  "links": [
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Grafana Docs",
+      "tooltip": "",
+      "type": "link",
+      "url": "https://grafana.com/docs/grafana/latest/"
+    },
+    {
+      "icon": "info",
+      "tags": [],
+      "targetBlank": true,
+      "title": "Prometheus Docs",
+      "type": "link",
+      "url": "http://prometheus.io/docs/introduction/overview/"
+    }
+  ],
+  "panels": [
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "eth-statediff-service",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 18,
+        "x": 0,
+        "y": 0
+      },
+      "hiddenSeries": false,
+      "id": 3,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pluginVersion": "7.1.5",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "rate(eth_statediff_service_processed_height{instance=\"$instance\"}[1m])",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Instance block processed (rate-1m)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "content": "<span style=\"font-family: 'Open Sans', 'Helvetica Neue', Helvetica; font-size: 25px;vertical-align: text-top;color: #bbbfc2;margin-left: 10px;\">Prometheus</span>\n\n<p style=\"margin-top: 10px;\">You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the <a href=\"https://grafana.com/\">Grafana</a> and <a href=\"http://prometheus.io/\">Prometheus</a> projects.</p>",
+      "datasource": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 9,
+      "links": [],
+      "mode": "html",
+      "options": {
+        "content": "<span style=\"font-family: 'Open Sans', 'Helvetica Neue', Helvetica; font-size: 25px;vertical-align: text-top;color: #bbbfc2;margin-left: 10px;\">Prometheus</span>\n\n<p style=\"margin-top: 10px;\">You're using Prometheus, an open-source systems monitoring and alerting toolkit originally built at SoundCloud. For more information, check out the <a href=\"https://grafana.com/\">Grafana</a> and <a href=\"http://prometheus.io/\">Prometheus</a> projects.</p>",
+        "mode": "html"
+      },
+      "pluginVersion": "7.1.0",
+      "style": {},
+      "title": "",
+      "transparent": true,
+      "type": "text"
+    },
+    {
+      "content": "#### Samples Ingested\nThis graph displays the count of samples ingested by the Prometheus server, as measured over the last 5 minutes, per time series in the range vector. When troubleshooting an issue on IRC or Github, this is often the first stat requested by the Prometheus team. ",
+      "datasource": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 18,
+        "y": 5
+      },
+      "id": 8,
+      "links": [],
+      "mode": "markdown",
+      "options": {
+        "content": "#### Samples Ingested\nThis graph displays the count of samples ingested by the Prometheus server, as measured over the last 5 minutes, per time series in the range vector. When troubleshooting an issue on IRC or Github, this is often the first stat requested by the Prometheus team. ",
+        "mode": "markdown"
+      },
+      "pluginVersion": "7.1.0",
+      "style": {},
+      "title": "",
+      "transparent": true,
+      "type": "text"
+    },
+    {
+      "aliasColors": {
+        "Value": "red",
+        "aggr": "red"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "eth-statediff-service",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 18,
+        "x": 0,
+        "y": 6
+      },
+      "hiddenSeries": false,
+      "id": 17,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "percentage": false,
+      "pluginVersion": "7.1.5",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(increase(eth_statediff_service_processed_height{instance=~\".+\"}[1m]))",
+          "interval": "",
+          "legendFormat": "aggr",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Aggregate block processed (rate-1m)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "content": "#### Scrapes\nPrometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. Target scrapes will show how frequently targets are scraped, as measured over the last 5 minutes, per time series in the range vector. Scrape Duration will show how long the scrapes are taking, with percentiles available as series. ",
+      "datasource": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 6,
+        "x": 18,
+        "y": 11
+      },
+      "id": 11,
+      "links": [],
+      "mode": "markdown",
+      "options": {
+        "content": "#### Scrapes\nPrometheus scrapes metrics from instrumented jobs, either directly or via an intermediary push gateway for short-lived jobs. Target scrapes will show how frequently targets are scraped, as measured over the last 5 minutes, per time series in the range vector. Scrape Duration will show how long the scrapes are taking, with percentiles available as series. ",
+        "mode": "markdown"
+      },
+      "pluginVersion": "7.1.0",
+      "style": {},
+      "title": "",
+      "transparent": true,
+      "type": "text"
+    },
+    {
+      "content": "#### Rule Evaluation Duration\nThis graph panel plots the duration for all evaluations to execute. The 50th percentile, 90th percentile and 99th percentile are shown as three separate series to help identify outliers that may be skewing the data.",
+      "datasource": null,
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 6,
+        "x": 18,
+        "y": 18
+      },
+      "id": 15,
+      "links": [],
+      "mode": "markdown",
+      "options": {
+        "content": "#### Rule Evaluation Duration\nThis graph panel plots the duration for all evaluations to execute. The 50th percentile, 90th percentile and 99th percentile are shown as three separate series to help identify outliers that may be skewing the data.",
+        "mode": "markdown"
+      },
+      "pluginVersion": "7.1.0",
+      "style": {},
+      "title": "",
+      "transparent": true,
+      "type": "text"
+    }
+  ],
+  "refresh": false,
+  "revision": "1.0",
+  "schemaVersion": 26,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {
+          "selected": true,
+          "text": "localhost:8888",
+          "value": "localhost:8888"
+        },
+        "datasource": "eth-statediff-service",
+        "definition": "label_values(instance)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "SelectInstance",
+        "multi": false,
+        "name": "instance",
+        "options": [],
+        "query": "label_values(instance)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {
+    "now": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "browser",
+  "title": "Prometheus Block Process Stats",
+  "uid": "q1FUJHKnz",
+  "version": 14
+}
@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 10s
+
+scrape_configs:
+  - job_name: 'eth-statediff-service'
+    static_configs:
+      - targets: ['localhost:8100']
@@ -18,28 +18,44 @@ package statediff
 import (
 	"context"

-	"github.com/ethereum/go-ethereum/statediff"
+	sd "github.com/ethereum/go-ethereum/statediff"
 )

+// APIName is the namespace used for the state diffing service API
+const APIName = "statediff"
+
+// APIVersion is the version of the state diffing service API
+const APIVersion = "0.0.1"
+
 // PublicStateDiffAPI provides an RPC interface
-// that can be used to fetch historical diffs from leveldb directly
+// that can be used to fetch historical diffs from LevelDB directly
 type PublicStateDiffAPI struct {
-	sds IService
+	sds StateDiffService
 }

 // NewPublicStateDiffAPI creates an rpc interface for the underlying statediff service
-func NewPublicStateDiffAPI(sds IService) *PublicStateDiffAPI {
+func NewPublicStateDiffAPI(sds StateDiffService) *PublicStateDiffAPI {
 	return &PublicStateDiffAPI{
 		sds: sds,
 	}
 }

 // StateDiffAt returns a state diff payload at the specific blockheight
-func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
+func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint64, params sd.Params) (*sd.Payload, error) {
 	return api.sds.StateDiffAt(blockNumber, params)
 }

 // StateTrieAt returns a state trie payload at the specific blockheight
-func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
+func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params sd.Params) (*sd.Payload, error) {
 	return api.sds.StateTrieAt(blockNumber, params)
 }
+
+// WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight
+func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params sd.Params) error {
+	return api.sds.WriteStateDiffAt(blockNumber, params)
+}
+
+// WriteStateDiffsInRange writes the state diff objects for the provided block range, with the provided params
+func (api *PublicStateDiffAPI) WriteStateDiffsInRange(ctx context.Context, start, stop uint64, params sd.Params) error {
+	return api.sds.WriteStateDiffsInRange(start, stop, params)
+}
@@ -0,0 +1,155 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+	"fmt"
+	"math/bits"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/core/state"
+	sd "github.com/ethereum/go-ethereum/statediff"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/sirupsen/logrus"
+
+	iter "github.com/cerc-io/go-eth-state-node-iterator"
+)
+
+type builder struct {
+	sd.StateDiffBuilder
+	numWorkers uint
+}
+
+// NewBuilder is used to create a statediff builder
+func NewBuilder(stateCache state.Database, workers uint) (sd.Builder, error) {
+	if workers == 0 {
+		workers = 1
+	}
+	if bits.OnesCount(workers) != 1 {
+		return nil, fmt.Errorf("workers must be a power of 2")
+	}
+	return &builder{
+		StateDiffBuilder: sd.StateDiffBuilder{
+			StateCache: stateCache,
+		},
+		numWorkers: workers,
+	}, nil
+}
+
+// BuildStateDiffObject builds a statediff object from two blocks and the provided parameters
+func (sdb *builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sdtypes.StateObject, error) {
+	var stateNodes []sdtypes.StateNode
+	var codeAndCodeHashes []sdtypes.CodeAndCodeHash
+	err := sdb.WriteStateDiffObject(
+		sdtypes.StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot},
+		params, sd.StateNodeAppender(&stateNodes), sd.CodeMappingAppender(&codeAndCodeHashes))
+	if err != nil {
+		return sdtypes.StateObject{}, err
+	}
+	return sdtypes.StateObject{
+		BlockHash:         args.BlockHash,
+		BlockNumber:       args.BlockNumber,
+		Nodes:             stateNodes,
+		CodeAndCodeHashes: codeAndCodeHashes,
+	}, nil
+}
+
+// WriteStateDiffObject writes a statediff object to output callback
+func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
+	// Load tries for old and new states
+	oldTrie, err := sdb.StateCache.OpenTrie(args.OldStateRoot)
+	if err != nil {
+		return fmt.Errorf("error creating trie for oldStateRoot: %v", err)
+	}
+	newTrie, err := sdb.StateCache.OpenTrie(args.NewStateRoot)
+	if err != nil {
+		return fmt.Errorf("error creating trie for newStateRoot: %v", err)
+	}
+
+	// Split old and new tries into corresponding subtrie iterators
+	oldIters1 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
+	oldIters2 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
+	newIters1 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
+	newIters2 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
+
+	// Create iterators ahead of time to avoid race condition in state.Trie access
+	// We do two state iterations per subtrie: one for new/updated nodes,
+	// one for deleted/updated nodes; prepare 2 iterator instances for each task
+	var iterPairs [][]sd.IterPair
+	for i := uint(0); i < sdb.numWorkers; i++ {
+		iterPairs = append(iterPairs, []sd.IterPair{
+			{Older: oldIters1[i], Newer: newIters1[i]},
+			{Older: oldIters2[i], Newer: newIters2[i]},
+		})
+	}
+
+	// Dispatch workers to process trie data; sync and collect results here via channels
+	nodeChan := make(chan sdtypes.StateNode)
+	codeChan := make(chan sdtypes.CodeAndCodeHash)
+
+	go func() {
+		nodeSender := func(node sdtypes.StateNode) error { nodeChan <- node; return nil }
+		codeSender := func(code sdtypes.CodeAndCodeHash) error { codeChan <- code; return nil }
+		var wg sync.WaitGroup
+
+		for w := uint(0); w < sdb.numWorkers; w++ {
+			wg.Add(1)
+			go func(worker uint) {
+				defer wg.Done()
+				var err error
+				if !params.IntermediateStateNodes {
+					err = sdb.BuildStateDiffWithoutIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender)
+				} else {
+					err = sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender)
+				}
+
+				if err != nil {
+					logrus.Errorf("buildStateDiff error for worker %d, params %+v", worker, params)
+				}
+			}(w)
+		}
+		wg.Wait()
+		close(nodeChan)
+		close(codeChan)
+	}()
+
+	for nodeChan != nil || codeChan != nil {
+		select {
+		case node, more := <-nodeChan:
+			if more {
+				if err := output(node); err != nil {
+					return err
+				}
+			} else {
+				nodeChan = nil
+			}
+		case codeAndCodeHash, more := <-codeChan:
+			if more {
+				if err := codeOutput(codeAndCodeHash); err != nil {
+					return err
+				}
+			} else {
+				codeChan = nil
+			}
+		}
+	}
+
+	return nil
+}
@@ -0,0 +1,9 @@
+package statediff
+
+// Config holds config params for the statediffing service
+type Config struct {
+	ServiceWorkers  uint
+	TrieWorkers     uint
+	WorkerQueueSize uint
+	PreRuns         []RangeRequest
+}
@@ -0,0 +1,72 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+	"sort"
+	"strings"
+)
+
+func sortKeys(data AccountMap) []string {
+	keys := make([]string, 0, len(data))
+	for key := range data {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+
+	return keys
+}
+
+// findIntersection finds the set of strings from both arrays that are equivalent
+// a and b must first be sorted
+// this is used to find which keys have been both "deleted" and "created" i.e. they were updated
+func findIntersection(a, b []string) []string {
+	lenA := len(a)
+	lenB := len(b)
+	iOfA, iOfB := 0, 0
+	updates := make([]string, 0)
+	if iOfA >= lenA || iOfB >= lenB {
+		return updates
+	}
+	for {
+		switch strings.Compare(a[iOfA], b[iOfB]) {
+		// -1 when a[iOfA] < b[iOfB]
+		case -1:
+			iOfA++
+			if iOfA >= lenA {
+				return updates
+			}
+			// 0 when a[iOfA] == b[iOfB]
+		case 0:
+			updates = append(updates, a[iOfA])
+			iOfA++
+			iOfB++
+			if iOfA >= lenA || iOfB >= lenB {
+				return updates
+			}
+			// 1 when a[iOfA] > b[iOfB]
+		case 1:
+			iOfB++
+			if iOfB >= lenB {
+				return updates
+			}
+		}
+	}
+}
@@ -0,0 +1,162 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+)
+
+const (
+	namespace = "eth_statediff_service"
+	subsystem = "connections"
+)
+
+// DBStatsGetter is an interface that gets sql.DBStats.
+type DBStatsGetter interface {
+	Stats() sql.Stats
+}
+
+// DBStatsCollector implements the prometheus.Collector interface.
+type DBStatsCollector struct {
+	sg DBStatsGetter
+
+	// descriptions of exported metrics
+	maxOpenDesc           *prometheus.Desc
+	openDesc              *prometheus.Desc
+	inUseDesc             *prometheus.Desc
+	idleDesc              *prometheus.Desc
+	waitedForDesc         *prometheus.Desc
+	blockedSecondsDesc    *prometheus.Desc
+	closedMaxIdleDesc     *prometheus.Desc
+	closedMaxLifetimeDesc *prometheus.Desc
+}
+
+// NewDBStatsCollector creates a new DBStatsCollector.
+func NewDBStatsCollector(dbName string, sg DBStatsGetter) *DBStatsCollector {
+	labels := prometheus.Labels{"db_name": dbName}
+	return &DBStatsCollector{
+		sg: sg,
+		maxOpenDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "max_open"),
+			"Maximum number of open connections to the database.",
+			nil,
+			labels,
+		),
+		openDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "open"),
+			"The number of established connections both in use and idle.",
+			nil,
+			labels,
+		),
+		inUseDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "in_use"),
+			"The number of connections currently in use.",
+			nil,
+			labels,
+		),
+		idleDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "idle"),
+			"The number of idle connections.",
+			nil,
+			labels,
+		),
+		waitedForDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "waited_for"),
+			"The total number of connections waited for.",
+			nil,
+			labels,
+		),
+		blockedSecondsDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "blocked_seconds"),
+			"The total time blocked waiting for a new connection.",
+			nil,
+			labels,
+		),
+		closedMaxIdleDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "closed_max_idle"),
+			"The total number of connections closed due to SetMaxIdleConns.",
+			nil,
+			labels,
+		),
+		closedMaxLifetimeDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "closed_max_lifetime"),
+			"The total number of connections closed due to SetConnMaxLifetime.",
+			nil,
+			labels,
+		),
+	}
+}
+
+// Describe implements the prometheus.Collector interface.
+func (c DBStatsCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.maxOpenDesc
+	ch <- c.openDesc
+	ch <- c.inUseDesc
+	ch <- c.idleDesc
+	ch <- c.waitedForDesc
+	ch <- c.blockedSecondsDesc
+	ch <- c.closedMaxIdleDesc
+	ch <- c.closedMaxLifetimeDesc
+}
+
+// Collect implements the prometheus.Collector interface.
+func (c DBStatsCollector) Collect(ch chan<- prometheus.Metric) {
+	stats := c.sg.Stats()
+
+	ch <- prometheus.MustNewConstMetric(
+		c.maxOpenDesc,
+		prometheus.GaugeValue,
+		float64(stats.MaxOpen()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.openDesc,
+		prometheus.GaugeValue,
+		float64(stats.Open()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.inUseDesc,
+		prometheus.GaugeValue,
+		float64(stats.InUse()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.idleDesc,
+		prometheus.GaugeValue,
+		float64(stats.Idle()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.waitedForDesc,
+		prometheus.CounterValue,
+		float64(stats.WaitCount()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.blockedSecondsDesc,
+		prometheus.CounterValue,
+		stats.WaitDuration().Seconds(),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.closedMaxIdleDesc,
+		prometheus.CounterValue,
+		float64(stats.MaxIdleClosed()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.closedMaxLifetimeDesc,
+		prometheus.CounterValue,
+		float64(stats.MaxLifetimeClosed()),
+	)
+}
@@ -0,0 +1,51 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// HTTPMiddleware http connection metric reader
+func HTTPMiddleware(next http.Handler) http.Handler {
+	if !metrics {
+		return next
+	}
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		httpCount.Inc()
+
+		start := time.Now()
+		next.ServeHTTP(w, r)
+		duration := time.Now().Sub(start)
+		httpDuration.Observe(float64(duration.Seconds()))
+	})
+}
+
+// IPCMiddleware unix-socket connection counter
+func IPCMiddleware(server *rpc.Server, client rpc.Conn) {
+	if metrics {
+		ipcCount.Inc()
+	}
+	server.ServeCodec(rpc.NewCodec(client), 0)
+	if metrics {
+		ipcCount.Dec()
+	}
+}
@@ -0,0 +1,175 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const (
+	statsSubsystem = "stats"
+	subsystemHTTP  = "http"
+	subsystemIPC   = "ipc"
+)
+
+var (
+	metrics bool
+
+	queuedRanges        prometheus.Gauge
+	lastLoadedHeight    prometheus.Gauge
+	lastProcessedHeight prometheus.Gauge
+
+	tBlockLoad       prometheus.Histogram
+	tBlockProcessing prometheus.Histogram
+	tStateProcessing prometheus.Histogram
+	tTxCommit        prometheus.Histogram
+
+	httpCount    prometheus.Counter
+	httpDuration prometheus.Histogram
+	ipcCount     prometheus.Gauge
+)
+
+const (
+	RANGES_QUEUED        = "ranges_queued"
+	LOADED_HEIGHT        = "loaded_height"
+	PROCESSED_HEIGHT     = "processed_height"
+	T_BLOCK_LOAD         = "t_block_load"
+	T_BLOCK_PROCESSING   = "t_block_processing"
+	T_STATE_PROCESSING   = "t_state_processing"
+	T_POSTGRES_TX_COMMIT = "t_postgres_tx_commit"
+)
+
+// Init module initialization
+func Init() {
+	metrics = true
+
+	queuedRanges = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Name:      RANGES_QUEUED,
+		Help:      "Number of range requests currently queued",
+	})
+	lastLoadedHeight = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Name:      LOADED_HEIGHT,
+		Help:      "The last block that was loaded for processing",
+	})
+	lastProcessedHeight = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Name:      PROCESSED_HEIGHT,
+		Help:      "The last block that was processed",
+	})
+
+	tBlockLoad = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Subsystem: statsSubsystem,
+		Name:      T_BLOCK_LOAD,
+		Help:      "Block loading time",
+	})
+	tBlockProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Subsystem: statsSubsystem,
+		Name:      T_BLOCK_PROCESSING,
+		Help:      "Block (header, uncles, txs, rcts, tx trie, rct trie) processing time",
+	})
+	tStateProcessing = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Subsystem: statsSubsystem,
+		Name:      T_STATE_PROCESSING,
+		Help:      "State (state trie, storage tries, and code) processing time",
+	})
+	tTxCommit = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Subsystem: statsSubsystem,
+		Name:      T_POSTGRES_TX_COMMIT,
+		Help:      "Postgres tx commit time",
+	})
+
+	httpCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystemHTTP,
+		Name:      "count",
+		Help:      "http request count",
+	})
+	httpDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Subsystem: subsystemHTTP,
+		Name:      "duration",
+		Help:      "http request duration",
+	})
+	ipcCount = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemIPC,
+		Name:      "count",
+		Help:      "unix socket connection count",
+	})
+}
+
+// RegisterDBCollector create metric collector for given connection
+func RegisterDBCollector(name string, db DBStatsGetter) {
+	if metrics {
+		prometheus.Register(NewDBStatsCollector(name, db))
+	}
+}
+
+// IncQueuedRanges increments the number of queued range requests
+func IncQueuedRanges() {
+	if metrics {
+		queuedRanges.Inc()
+	}
+}
+
+// DecQueuedRanges decrements the number of queued range requests
+func DecQueuedRanges() {
+	if metrics {
+		queuedRanges.Dec()
+	}
+}
+
+// SetLastLoadedHeight sets last loaded height
+func SetLastLoadedHeight(height int64) {
+	if metrics {
+		lastLoadedHeight.Set(float64(height))
+	}
+}
+
+// SetLastProcessedHeight sets last processed height
+func SetLastProcessedHeight(height int64) {
+	if metrics {
+		lastProcessedHeight.Set(float64(height))
+	}
+}
+
+// SetTimeMetric time metric observation
+func SetTimeMetric(name string, t time.Duration) {
+	if !metrics {
+		return
+	}
+	tAsF64 := t.Seconds()
+	switch name {
+	case T_BLOCK_LOAD:
+		tBlockLoad.Observe(tAsF64)
+	case T_BLOCK_PROCESSING:
+		tBlockProcessing.Observe(tAsF64)
+	case T_STATE_PROCESSING:
+		tStateProcessing.Observe(tAsF64)
+	case T_POSTGRES_TX_COMMIT:
+		tTxCommit.Observe(tAsF64)
+	}
+}
@@ -0,0 +1,40 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/sirupsen/logrus"
+)
+
+// Listen start listening http
+func Listen(addr string) *http.Server {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	srv := http.Server{
+		Addr:    addr,
+		Handler: mux,
+	}
+	go func() {
+		if err := srv.ListenAndServe(); err != nil {
+			logrus.Fatal(err)
+		}
+	}()
+	return &srv
+}
@@ -25,25 +25,55 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/vulcanize/leveldb-ethdb-rpc/pkg/client"
 )

-// LvlDBReader exposes the necessary read functions on lvldb
+// Reader interface required by the statediffing service
+type Reader interface {
+	GetBlockByHash(hash common.Hash) (*types.Block, error)
+	GetBlockByNumber(number uint64) (*types.Block, error)
+	GetReceiptsByHash(hash common.Hash) (types.Receipts, error)
+	GetTdByHash(hash common.Hash) (*big.Int, error)
+	StateDB() state.Database
+}
+
+// LvlDBReader exposes the necessary Reader methods on lvldb
 type LvlDBReader struct {
 	ethDB       ethdb.Database
 	stateDB     state.Database
 	chainConfig *params.ChainConfig
 }

-// NewLvlDBReader creates a new LvlDBReader
-func NewLvlDBReader(path, ancient string, chainConfig *params.ChainConfig) (*LvlDBReader, error) {
-	edb, err := rawdb.NewLevelDBDatabaseWithFreezer(path, 1024, 256, ancient, "eth-statediff-service")
+// LvLDBReaderConfig struct for initializing a LvlDBReader
+type LvLDBReaderConfig struct {
+	TrieConfig             *trie.Config
+	ChainConfig            *params.ChainConfig
+	Mode                   string
+	Path, AncientPath, Url string
+	DBCacheSize            int
+}
+
+// NewLvlDBReader creates a new Read using LevelDB
+func NewLvlDBReader(conf LvLDBReaderConfig) (*LvlDBReader, error) {
+	var edb ethdb.Database
+	var err error
+
+	if conf.Mode == "local" {
+		edb, err = rawdb.NewLevelDBDatabaseWithFreezer(conf.Path, conf.DBCacheSize, 256, conf.AncientPath, "eth-statediff-service", true)
+	}
+
+	if conf.Mode == "remote" {
+		edb, err = client.NewDatabaseClient(conf.Url)
+	}
+
 	if err != nil {
 		return nil, err
 	}
 	return &LvlDBReader{
 		ethDB:       edb,
-		stateDB:     state.NewDatabase(edb),
-		chainConfig: chainConfig,
+		stateDB:     state.NewDatabaseWithConfig(edb, conf.TrieConfig),
+		chainConfig: conf.ChainConfig,
 	}, nil
 }

@@ -0,0 +1,37 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import "github.com/ethereum/go-ethereum/rpc"
+
+// checkModuleAvailability check that all names given in modules are actually
+// available API services.
+func checkModuleAvailability(modules []string, apis []rpc.API) (bad, available []string) {
+	availableSet := make(map[string]struct{})
+	for _, api := range apis {
+		if _, ok := availableSet[api.Namespace]; !ok {
+			availableSet[api.Namespace] = struct{}{}
+			available = append(available, api.Namespace)
+		}
+	}
+	for _, name := range modules {
+		if _, ok := availableSet[name]; !ok {
+			bad = append(bad, name)
+		}
+	}
+	return bad, available
+}
@@ -0,0 +1,49 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
+)
+
+// StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules.
+func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (*rpc.Server, error) {
+
+	srv := rpc.NewServer()
+	err := node.RegisterApis(apis, modules, srv)
+	if err != nil {
+		utils.Fatalf("Could not register HTTP API: %w", err)
+	}
+	handler := node.NewHTTPHandlerStack(srv, cors, vhosts, nil)
+
+	// start http server
+	_, addr, err := node.StartHTTPEndpoint(endpoint, rpc.DefaultHTTPTimeouts, prom.HTTPMiddleware(handler))
+	if err != nil {
+		utils.Fatalf("Could not start RPC api: %v", err)
+	}
+	extapiURL := fmt.Sprintf("http://%v/", addr)
+	log.Infof("HTTP endpoint opened %s", extapiURL)
+
+	return srv, err
+}
@@ -0,0 +1,92 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+
+	"github.com/ethereum/go-ethereum/p2p/netutil"
+	"github.com/ethereum/go-ethereum/rpc"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
+)
+
+var (
+	//  On Linux, sun_path is 108 bytes in size
+	// see http://man7.org/linux/man-pages/man7/unix.7.html
+	maxPathSize = 108
+)
+
+// ipcListen will create a Unix socket on the given endpoint.
+func ipcListen(endpoint string) (net.Listener, error) {
+	if len(endpoint) > int(maxPathSize) {
+		log.Warn(fmt.Sprintf("The ipc endpoint is longer than %d characters. ", maxPathSize),
+			"endpoint", endpoint)
+	}
+
+	// Ensure the IPC path exists and remove any previous leftover
+	if err := os.MkdirAll(filepath.Dir(endpoint), 0751); err != nil {
+		return nil, err
+	}
+	os.Remove(endpoint)
+	l, err := net.Listen("unix", endpoint)
+	if err != nil {
+		return nil, err
+	}
+	os.Chmod(endpoint, 0600)
+	return l, nil
+}
+
+func ipcServe(srv *rpc.Server, listener net.Listener) {
+	for {
+		conn, err := listener.Accept()
+		if netutil.IsTemporaryError(err) {
+			log.WithError(err).Warn("rpc accept error")
+			continue
+		}
+		if err != nil {
+			log.WithError(err).Warn("unknown error")
+			continue
+		}
+		log.WithField("addr", conn.RemoteAddr()).Trace("accepted ipc connection")
+		go prom.IPCMiddleware(srv, conn)
+	}
+}
+
+// StartIPCEndpoint starts an IPC endpoint.
+func StartIPCEndpoint(ipcEndpoint string, apis []rpc.API) (net.Listener, *rpc.Server, error) {
+	// Register all the APIs exposed by the services.
+	handler := rpc.NewServer()
+	for _, api := range apis {
+		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
+			return nil, nil, err
+		}
+		log.Debug("IPC registered", "namespace", api.Namespace)
+	}
+	// All APIs registered, start the IPC listener.
+	listener, err := ipcListen(ipcEndpoint)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	go ipcServe(handler, listener)
+	return listener, handler, nil
+}
@@ -20,55 +20,83 @@ import (
 	"fmt"
 	"math/big"
 	"sync"
+	"time"

 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/statediff"
+	sd "github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/sirupsen/logrus"
+
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

-// lvlDBReader are the db interfaces required by the statediffing service
-type lvlDBReader interface {
-	GetBlockByHash(hash common.Hash) (*types.Block, error)
-	GetBlockByNumber(number uint64) (*types.Block, error)
-	GetReceiptsByHash(hash common.Hash) (types.Receipts, error)
-	GetTdByHash(hash common.Hash) (*big.Int, error)
-	StateDB() state.Database
-}
+const defaultQueueSize = 1024

-// IService is the state-diffing service interface
-type IService interface {
-	// APIs(), Protocols(), Start() and Stop()
-	node.Service
-	// Main event loop for processing state diffs
-	Loop(wg *sync.WaitGroup)
-	// Method to get state diff object at specific block
-	StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error)
-	// Method to get state trie object at specific block
-	StateTrieAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error)
+// StateDiffService is the state-diffing service interface
+type StateDiffService interface {
+	// Lifecycle Start() and Stop()
+	node.Lifecycle
+	// APIs and Protocols() interface for node service registration
+	APIs() []rpc.API
+	Protocols() []p2p.Protocol
+	// Loop is the main event loop for processing state diffs
+	Loop(wg *sync.WaitGroup) error
+	// Run is a one-off command to run on a predefined set of ranges
+	Run(ranges []RangeRequest) error
+	// StateDiffAt method to get state diff object at specific block
+	StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
+	// StateDiffFor method to get state diff object at specific block
+	StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.Payload, error)
+	// StateTrieAt method to get state trie object at specific block
+	StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
+	// WriteStateDiffAt method to write state diff object directly to DB
+	WriteStateDiffAt(blockNumber uint64, params sd.Params) error
+	// WriteStateDiffFor method to get state trie object at specific block
+	WriteStateDiffFor(blockHash common.Hash, params sd.Params) error
+	// WriteStateDiffsInRange method to wrtie state diff objects within the range directly to the DB
+	WriteStateDiffsInRange(start, stop uint64, params sd.Params) error
 }

 // Service is the underlying struct for the state diffing service
 type Service struct {
 	// Used to build the state diff objects
-	Builder statediff.Builder
-	// Used to read data from leveldb
-	lvlDBReader lvlDBReader
+	Builder sd.Builder
+	// Used to read data from LevelDB
+	lvlDBReader Reader
 	// Used to signal shutdown of the service
-	QuitChan chan bool
+	quitChan chan struct{}
+	// Interface for publishing statediffs as PG-IPLD objects
+	indexer interfaces.StateDiffIndexer
+	// range queue
+	queue chan RangeRequest
+	// number of ranges we can work over concurrently
+	workers uint
+	// ranges configured locally
+	preruns []RangeRequest
 }

-// NewStateDiffService creates a new statediff.Service
-func NewStateDiffService(lvlDBReader lvlDBReader) (*Service, error) {
+// NewStateDiffService creates a new Service
+func NewStateDiffService(lvlDBReader Reader, indexer interfaces.StateDiffIndexer, conf Config) (*Service, error) {
+	b, err := NewBuilder(lvlDBReader.StateDB(), conf.TrieWorkers)
+	if err != nil {
+		return nil, err
+	}
+	if conf.WorkerQueueSize == 0 {
+		conf.WorkerQueueSize = defaultQueueSize
+	}
 	return &Service{
 		lvlDBReader: lvlDBReader,
-		Builder:     statediff.NewBuilder(lvlDBReader.StateDB()),
-		QuitChan:    make(chan bool),
+		Builder:     b,
+		indexer:     indexer,
+		workers:     conf.ServiceWorkers,
+		queue:       make(chan RangeRequest, conf.WorkerQueueSize),
+		preruns:     conf.PreRuns,
 	}, nil
 }

@@ -77,39 +105,99 @@ func (sds *Service) Protocols() []p2p.Protocol {
 	return []p2p.Protocol{}
 }

-// APIs returns the RPC descriptors the statediff.Service offers
+// APIs returns the RPC descriptors the Service offers
 func (sds *Service) APIs() []rpc.API {
 	return []rpc.API{
 		{
-			Namespace: statediff.APIName,
-			Version:   statediff.APIVersion,
+			Namespace: APIName,
+			Version:   APIVersion,
 			Service:   NewPublicStateDiffAPI(sds),
 			Public:    true,
 		},
 	}
 }

-// Loop is an empty service loop for awaiting rpc requests
-func (sds *Service) Loop(wg *sync.WaitGroup) {
-	wg.Add(1)
-	for {
-		select {
-		case <-sds.QuitChan:
-			logrus.Info("closing the statediff service loop")
-			wg.Done()
-			return
+// Run does a one-off processing run on the provided RangeRequests + any pre-runs, exiting afterwards
+func (sds *Service) Run(rngs []RangeRequest) error {
+	for _, preRun := range sds.preruns {
+		logrus.Infof("processing prerun range (%d, %d)", preRun.Start, preRun.Stop)
+		for i := preRun.Start; i <= preRun.Stop; i++ {
+			if err := sds.WriteStateDiffAt(i, preRun.Params); err != nil {
+				return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, preRun.Start, preRun.Stop, err)
+			}
 		}
 	}
+	sds.preruns = nil
+	for _, rng := range rngs {
+		logrus.Infof("processing prerun range (%d, %d)", rng.Start, rng.Stop)
+		for i := rng.Start; i <= rng.Stop; i++ {
+			if err := sds.WriteStateDiffAt(i, rng.Params); err != nil {
+				return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, rng.Start, rng.Stop, err)
+			}
+		}
+	}
+	return nil
+}
+
+// Loop is an empty service loop for awaiting rpc requests
+func (sds *Service) Loop(wg *sync.WaitGroup) error {
+	if sds.quitChan != nil {
+		return fmt.Errorf("service loop is already running")
+	}
+
+	sds.quitChan = make(chan struct{})
+	for i := 0; i < int(sds.workers); i++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+			for {
+				select {
+				case blockRange := <-sds.queue:
+					logrus.Infof("service worker %d received range (%d, %d) off of work queue, beginning processing", id, blockRange.Start, blockRange.Stop)
+					prom.DecQueuedRanges()
+					for j := blockRange.Start; j <= blockRange.Stop; j++ {
+						if err := sds.WriteStateDiffAt(j, blockRange.Params); err != nil {
+							logrus.Errorf("service worker %d error writing statediff at height %d in range (%d, %d) : %v", id, j, blockRange.Start, blockRange.Stop, err)
+						}
+						select {
+						case <-sds.quitChan:
+							logrus.Infof("closing service worker %d\n"+
+								"working in range (%d, %d)\n"+
+								"last processed height: %d", id, blockRange.Start, blockRange.Stop, j)
+							return
+						default:
+							logrus.Infof("service worker %d finished processing statediff height %d in range (%d, %d)", id, j, blockRange.Start, blockRange.Stop)
+						}
+					}
+					logrus.Infof("service worker %d finished processing range (%d, %d)", id, blockRange.Start, blockRange.Stop)
+				case <-sds.quitChan:
+					logrus.Infof("closing the statediff service loop worker %d", id)
+					return
+				}
+			}
+		}(i)
+	}
+	for _, preRun := range sds.preruns {
+		if err := sds.WriteStateDiffsInRange(preRun.Start, preRun.Stop, preRun.Params); err != nil {
+			close(sds.quitChan)
+			return err
+		}
+	}
+	return nil
 }

 // StateDiffAt returns a state diff object payload at the specific blockheight
 // This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
+func (sds *Service) StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error) {
 	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
 	if err != nil {
 		return nil, err
 	}
-	logrus.Info(fmt.Sprintf("sending state diff at block %d", blockNumber))
+	logrus.Infof("sending state diff at block %d", blockNumber)
+
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
+
 	if blockNumber == 0 {
 		return sds.processStateDiff(currentBlock, common.Hash{}, params)
 	}
@@ -120,18 +208,40 @@ func (sds *Service) StateDiffAt(blockNumber uint64, params statediff.Params) (*s
 	return sds.processStateDiff(currentBlock, parentBlock.Root(), params)
 }

+// StateDiffFor returns a state diff object payload for the specific blockhash
+// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
+func (sds *Service) StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.Payload, error) {
+	currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
+	if err != nil {
+		return nil, err
+	}
+	logrus.Infof("sending state diff at block %s", blockHash.Hex())
+
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
+
+	if currentBlock.NumberU64() == 0 {
+		return sds.processStateDiff(currentBlock, common.Hash{}, params)
+	}
+	parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
+	if err != nil {
+		return nil, err
+	}
+	return sds.processStateDiff(currentBlock, parentBlock.Root(), params)
+}
+
 // processStateDiff method builds the state diff payload from the current block, parent state root, and provided params
-func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params statediff.Params) (*statediff.Payload, error) {
-	stateDiff, err := sds.Builder.BuildStateDiffObject(statediff.Args{
-		NewStateRoot: currentBlock.Root(),
-		OldStateRoot: parentRoot,
+func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot common.Hash, params sd.Params) (*sd.Payload, error) {
+	stateDiff, err := sds.Builder.BuildStateDiffObject(sd.Args{
 		BlockHash:    currentBlock.Hash(),
 		BlockNumber:  currentBlock.Number(),
+		OldStateRoot: parentRoot,
+		NewStateRoot: currentBlock.Root(),
 	}, params)
 	if err != nil {
 		return nil, err
 	}
-	stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
+	stateDiffRlp, err := rlp.EncodeToBytes(&stateDiff)
 	if err != nil {
 		return nil, err
 	}
@@ -139,8 +249,8 @@ func (sds *Service) processStateDiff(currentBlock *types.Block, parentRoot commo
 	return sds.newPayload(stateDiffRlp, currentBlock, params)
 }

-func (sds *Service) newPayload(stateObject []byte, block *types.Block, params statediff.Params) (*statediff.Payload, error) {
-	payload := &statediff.Payload{
+func (sds *Service) newPayload(stateObject []byte, block *types.Block, params sd.Params) (*sd.Payload, error) {
+	payload := &sd.Payload{
 		StateObjectRlp: stateObject,
 	}
 	if params.IncludeBlock {
@@ -173,21 +283,25 @@ func (sds *Service) newPayload(stateObject []byte, block *types.Block, params st

 // StateTrieAt returns a state trie object payload at the specified blockheight
 // This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateTrieAt(blockNumber uint64, params statediff.Params) (*statediff.Payload, error) {
+func (sds *Service) StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error) {
 	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
 	if err != nil {
 		return nil, err
 	}
-	logrus.Info(fmt.Sprintf("sending state trie at block %d", blockNumber))
+	logrus.Infof("sending state trie at block %d", blockNumber)
+
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
+
 	return sds.processStateTrie(currentBlock, params)
 }

-func (sds *Service) processStateTrie(block *types.Block, params statediff.Params) (*statediff.Payload, error) {
+func (sds *Service) processStateTrie(block *types.Block, params sd.Params) (*sd.Payload, error) {
 	stateNodes, err := sds.Builder.BuildStateTrieObject(block)
 	if err != nil {
 		return nil, err
 	}
-	stateTrieRlp, err := rlp.EncodeToBytes(stateNodes)
+	stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes)
 	if err != nil {
 		return nil, err
 	}
@@ -196,15 +310,126 @@ func (sds *Service) processStateTrie(block *types.Block, params statediff.Params
 }

 // Start is used to begin the service
-func (sds *Service) Start(*p2p.Server) error {
+func (sds *Service) Start() error {
 	logrus.Info("starting statediff service")
-	go sds.Loop(new(sync.WaitGroup))
-	return nil
+	return sds.Loop(new(sync.WaitGroup))
 }

 // Stop is used to close down the service
 func (sds *Service) Stop() error {
 	logrus.Info("stopping statediff service")
-	close(sds.QuitChan)
+	close(sds.quitChan)
 	return nil
 }
+
+// WriteStateDiffAt writes a state diff at the specific blockheight directly to the database
+// This operation cannot be performed back past the point of db pruning; it requires an archival node
+// for historical data
+func (sds *Service) WriteStateDiffAt(blockNumber uint64, params sd.Params) error {
+	logrus.Infof("Writing state diff at block %d", blockNumber)
+	t := time.Now()
+	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
+	if err != nil {
+		return err
+	}
+
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
+
+	parentRoot := common.Hash{}
+	if blockNumber != 0 {
+		parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
+		if err != nil {
+			return err
+		}
+		parentRoot = parentBlock.Root()
+	}
+	return sds.writeStateDiff(currentBlock, parentRoot, params, t)
+}
+
+// WriteStateDiffFor writes a state diff for the specific blockHash directly to the database
+// This operation cannot be performed back past the point of db pruning; it requires an archival node
+// for historical data
+func (sds *Service) WriteStateDiffFor(blockHash common.Hash, params sd.Params) error {
+	logrus.Infof("Writing state diff for block %s", blockHash.Hex())
+	t := time.Now()
+	currentBlock, err := sds.lvlDBReader.GetBlockByHash(blockHash)
+	if err != nil {
+		return err
+	}
+
+	// compute leaf paths of watched addresses in the params
+	params.ComputeWatchedAddressesLeafPaths()
+
+	parentRoot := common.Hash{}
+	if currentBlock.NumberU64() != 0 {
+		parentBlock, err := sds.lvlDBReader.GetBlockByHash(currentBlock.ParentHash())
+		if err != nil {
+			return err
+		}
+		parentRoot = parentBlock.Root()
+	}
+	return sds.writeStateDiff(currentBlock, parentRoot, params, t)
+}
+
+// Writes a state diff from the current block, parent state root, and provided params
+func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, params sd.Params, t time.Time) error {
+	var totalDifficulty *big.Int
+	var receipts types.Receipts
+	var err error
+	if params.IncludeTD {
+		totalDifficulty, err = sds.lvlDBReader.GetTdByHash(block.Hash())
+	}
+	if err != nil {
+		return err
+	}
+	if params.IncludeReceipts {
+		receipts, err = sds.lvlDBReader.GetReceiptsByHash(block.Hash())
+	}
+	if err != nil {
+		return err
+	}
+	height := block.Number().Int64()
+	prom.SetLastLoadedHeight(height)
+	prom.SetTimeMetric(prom.T_BLOCK_LOAD, time.Now().Sub(t))
+	t = time.Now()
+	tx, err := sds.indexer.PushBlock(block, receipts, totalDifficulty)
+	if err != nil {
+		return err
+	}
+	// defer handling of commit/rollback for any return case
+	output := func(node sdtypes.StateNode) error {
+		return sds.indexer.PushStateNode(tx, node, block.Hash().String())
+	}
+	codeOutput := func(c sdtypes.CodeAndCodeHash) error {
+		return sds.indexer.PushCodeAndCodeHash(tx, c)
+	}
+	prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t))
+	t = time.Now()
+	err = sds.Builder.WriteStateDiffObject(sdtypes.StateRoots{
+		NewStateRoot: block.Root(),
+		OldStateRoot: parentRoot,
+	}, params, output, codeOutput)
+	prom.SetTimeMetric(prom.T_STATE_PROCESSING, time.Now().Sub(t))
+	t = time.Now()
+	err = tx.Submit(err)
+	prom.SetLastProcessedHeight(height)
+	prom.SetTimeMetric(prom.T_POSTGRES_TX_COMMIT, time.Now().Sub(t))
+	return err
+}
+
+// WriteStateDiffsInRange adds a RangeRequest to the work queue
+func (sds *Service) WriteStateDiffsInRange(start, stop uint64, params sd.Params) error {
+	if stop < start {
+		return fmt.Errorf("invalid block range (%d, %d): stop height must be greater or equal to start height", start, stop)
+	}
+	blocked := time.NewTimer(30 * time.Second)
+	select {
+	case sds.queue <- RangeRequest{Start: start, Stop: stop, Params: params}:
+		prom.IncQueuedRanges()
+		logrus.Infof("added range (%d, %d) to the worker queue", start, stop)
+		return nil
+	case <-blocked.C:
+		return fmt.Errorf("unable to add range (%d, %d) to the worker queue", start, stop)
+	}
+}
@@ -0,0 +1,44 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Contains a batch of utility type declarations used by the tests. As the node
+// operates on unique types, a lot of them are needed to check various features.
+
+package statediff
+
+import (
+	"github.com/ethereum/go-ethereum/core/types"
+	sd "github.com/ethereum/go-ethereum/statediff"
+	sdTypes "github.com/ethereum/go-ethereum/statediff/types"
+)
+
+// AccountMap is a mapping of hex encoded path => account wrapper
+type AccountMap map[string]accountWrapper
+
+// accountWrapper is used to temporary associate the unpacked node with its raw values
+type accountWrapper struct {
+	Account   *types.StateAccount
+	NodeType  sdTypes.NodeType
+	Path      []byte
+	NodeValue []byte
+	LeafKey   []byte
+}
+
+// RangeRequest holds range quest work params
+type RangeRequest struct {
+	Start, Stop uint64
+	Params      sd.Params
+}
@@ -4,8 +4,17 @@ set -e
 set +x

 # Check the database variables are set
-test $VDB_COMMAND
+test "$VDB_COMMAND"
 set +e

-echo "Running the statediff service"
-./eth-statediff-service ${VDB_COMMAND} --config=config.toml
+# docker must be run in privilaged mode for mounts to work
+echo "Setting up /app/geth-rw overlayed /app/geth-ro"
+mkdir -p /tmp/overlay && \
+sudo mount -t tmpfs tmpfs /tmp/overlay && \
+mkdir -p /tmp/overlay/upper && \
+mkdir -p /tmp/overlay/work && \
+mkdir -p /app/geth-rw && \
+sudo mount -t overlay overlay -o lowerdir=/app/geth-ro,upperdir=/tmp/overlay/upper,workdir=/tmp/overlay/work /app/geth-rw && \
+
+echo "Running the statediff service" && \
+sudo ./eth-statediff-service "$VDB_COMMAND" --config=config.toml
@@ -0,0 +1,30 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package version
+
+import "fmt"
+
+const (
+	major = 3 // major version component of the current release
+	minor = 0 // minor version component of the current release
+	patch = 0 // patch version component of the current release
+)
+
+// Version holds the textual version string.
+var Version = func() string {
+	return fmt.Sprintf("%d.%d.%d", major, minor, patch)
+}()
Author	SHA1	Message	Date
Michael Shaw	67d8bced4f	one more reference changed to git.vdb.to	2022-09-21 13:27:05 -04:00
Michael Shaw	12f4810ced	missed Makefile update for cerc-io	2022-09-21 11:55:17 -04:00
Michael Shaw	7a8d38c955	indentation problem in yml	2022-09-21 11:24:43 -04:00
Michael Shaw	d09cd0afe6	update tagging and push to git.vdb.to registry	2022-09-21 11:13:23 -04:00
Michael	bcca82eaa3	Cerc refactor (#104 ) * cerc refactor waiting on unpublished dependencies * updated go.mod for published dependencies in cerc-io refactoring	2022-09-20 14:00:10 -04:00
Michael	dad77b561d	Merge pull request #102 from vulcanize/rebase-1.10.23-wip update for geth 1.10.23	2022-09-02 16:16:34 -04:00
Michael Shaw	c939822a95	update for geth 1.10.23	2022-09-02 15:56:56 -04:00
Michael	1fdb8763ac	Merge pull request #101 from vulcanize/geth_1_10_21_wip updates for geth 1.10.21	2022-08-03 10:21:22 -04:00
Michael Shaw	cd5aee30c7	missed go mod tidy	2022-08-03 10:08:01 -04:00
Michael Shaw	040638ca05	updates for geth 1.10.21	2022-08-03 10:03:57 -04:00
Ashwin Phatak	681e656034	Merge pull request #98 from deep-stack/pm-import-instructions Add instructions to import data output in file mode	2022-07-19 15:10:25 +05:30
prathamesh	85896f91b7	Expose database stats in metrics	2022-07-19 15:00:01 +05:30
prathamesh	52c7f84432	Update instructions to import data output	2022-07-18 19:14:41 +05:30
prathamesh	aa6ee578f4	Add instructions to import data output in file mode	2022-07-13 19:33:47 +05:30
prathamesh0	1f898f60c3	Upgrade geth after schema changes (#97 ) * Include intermediate nodes in builder tests for watched addresses * Upgrade geth	2022-07-12 16:38:05 +05:30
nabarun	0c56037e1f	Upgrade geth and add params for CSV file mode (#96 ) * Upgrade geth and add params for CSV file mode * Add options for file mode flag * index intermediate nodes when watching addresses	2022-07-01 13:29:48 +05:30
prathamesh0	9aa683442e	Reuse builder from vulcanize-geth (#92 ) * Reuse builder code from vulcanize-geth * Upgrade geth dependency	2022-06-24 14:12:58 +05:30
Abdul Rabbani	adf3dd4b6f	Update CICD pipeline to skip duplicate tests (#95 ) * Update CICD pipeline to skip duplicate tests * fix dependencies * Fix other dep	2022-06-17 09:45:53 -04:00
Abdul Rabbani	5ea4b6766a	Merge pull request #94 from vulcanize/feature/update-geth-v1.10.19 Update geth and check for duplicate jobs in CI	2022-06-17 09:00:19 -04:00
Abdul Rabbani	2c4fd6f099	Update geth and check for duplicate jobs in CI	2022-06-17 08:56:25 -04:00
Ashwin Phatak	f964b53fe3	Merge pull request #91 from vulcanize/ng-update-iterator Update go-eth-state-node-iterator version	2022-06-10 15:27:31 +05:30
nabarun	b8dad6a09b	Update go-eth-state-node-iterator version	2022-06-10 15:02:42 +05:30
Abdul Rabbani	1dc90d0417	Merge pull request #89 from vulcanize/feature/update-go-geth-sharding Feature/update go geth sharding	2022-05-31 13:41:15 -04:00
Abdul Rabbani	784ffb8726	Update geth version	2022-05-31 13:37:35 -04:00
Abdul Rabbani	1ddffe65be	Merge branch 'master' into feature/update-go-geth-sharding	2022-05-31 13:35:11 -04:00
Abdul Rabbani	fdeece4b5d	Merge pull request #88 from vulcanize/release-v3.0.3 Update go and geth versions	2022-05-31 07:30:38 -04:00
Abdul Rabbani	a58b16b0a1	Update go and dependencies	2022-05-27 15:10:57 -04:00
Abdul Rabbani	a99dcb4ca9	Update the PR and release pipeline	2022-05-26 15:32:48 -04:00
Abdul Rabbani	d3fd7ef532	Update geth version using 1.10.18	2022-05-26 08:30:55 -04:00
Abdul Rabbani	3e8f5d34fb	Merge pull request #86 from vulcanize/feature/cicd-docker-test Combine build and publish	2022-05-18 15:21:38 -04:00
Abdul Rabbani	1df0e8b840	Fix GHA issues	2022-05-18 15:16:42 -04:00
Abdul Rabbani	5bdec6bc41	Change pipeline to not rebuild the image.	2022-05-18 15:11:06 -04:00
Abdul Rabbani	9fbc07c836	Combine build and publish	2022-05-18 14:58:46 -04:00
nabarun	cdcc3df9f2	Upgrade geth version to v4 (#85 ) * Upgrade geth version to v4 * Upgrade go-eth-state-node-iterator version	2022-05-17 19:22:30 +05:30
nabarun	5f6aec35b2	Support remote RPC client to access LevelDB (#83 ) * Implement use of custom chain config to test locally * Use leveldb-ethdb-rpc instead of leveldb path * Implement flag for leveldb access mode * Remove custom chain config json * Move database client to leveldb-ethdb-rpc * Update leveldb-ethdb-rpc version	2022-05-17 14:35:11 +05:30
Ashwin Phatak	867f5c9971	Merge pull request #84 from deep-stack/ng-update-geth Update vulcanize geth version to 1.10.17	2022-05-16 10:10:58 +05:30
nabarun	257b16e184	Add missing tests for direct indexing mode	2022-05-13 18:28:58 +05:30
nabarun	b4a899cc8c	Remove WatchedStorageSlots param and local chain.json	2022-05-13 17:25:31 +05:30
nabarun	4363bf5c07	Implement use of custom chain config to test locally	2022-05-13 16:37:30 +05:30
nabarun	7c19e86698	Call method for computing watchedAddressesLeafKeys	2022-05-12 15:51:46 +05:30
nabarun	6ed1bc739d	Update builder tests to use geth statediff test helpers	2022-05-11 18:53:39 +05:30
nabarun	32724ff46c	Fix rlp.EncodeToBytes arg and creating node iterators	2022-05-11 15:21:23 +05:30
nabarun	9a7ecf9ef3	Add watched addresses direct indexing mode changes	2022-05-11 00:03:20 +05:30
nabarun	ea4fb6222b	Update geth version	2022-05-10 11:36:14 +05:30
Ian Norden	1d10009678	Merge pull request #78 from vulcanize/release-v3.0.0 Release v3.0.0	2022-02-01 12:15:32 -06:00
i-norden	feca7b67db	bump major version	2022-02-01 12:11:57 -06:00
i-norden	edef920ae4	bump iterator version	2022-02-01 12:11:21 -06:00
i-norden	ec0fdb5215	rebase	2022-02-01 12:08:25 -06:00
i-norden	040bc850f0	bump statediff geth version	2022-02-01 11:34:02 -06:00
Ian Norden	0c630b6e99	Merge pull request #77 from vulcanize/release-v2.0.0 update to use v1.10.15-statediff-2.0.0 geth	2022-01-26 09:27:35 -06:00
i-norden	b68bd29304	bump version	2022-01-26 09:26:18 -06:00
i-norden	c3b007de26	update to use v1.10.15-statediff-2.0.0 geth	2022-01-26 09:22:17 -06:00
Arijit Das	e9428ac208	Merge pull request #75 from vulcanize/docker-release-using-branch Release docker image using branch.	2022-01-13 21:21:38 +05:30
Ian Norden	9521bda9d0	Merge pull request #76 from vulcanize/impl-version-cmd Add version command.	2021-12-27 11:49:56 -06:00
i-norden	9f76540c3a	bump version, update license year	2021-12-27 11:49:10 -06:00
Arijit Das	747c5f5400	Use commit hash to build docker image and publish.	2021-12-23 19:53:53 +05:30
Arijit Das	3725006984	Add version command.	2021-12-23 16:40:23 +05:30
Arijit Das	e08d31e242	Use branch name.	2021-12-21 19:09:08 +05:30
Arijit Das	0ff60a69da	Small fix.	2021-12-21 18:12:22 +05:30
Arijit Das	a99b1975e4	Small fix.	2021-12-21 17:37:01 +05:30
Arijit Das	316926a6b5	Release docker image using branch.	2021-12-21 17:16:54 +05:30
Ian Norden	15cb0b8e8c	Merge pull request #72 from vulcanize/v1.10.13-statediff-0.0.29 bump go-ethereum version	2021-12-17 18:02:51 -06:00
i-norden	caa1938ce6	bump go-ethereum version	2021-12-17 18:00:10 -06:00
Ian Norden	0d60752b8a	Merge pull request #71 from vulcanize/bump bump go-ethereum version	2021-12-17 13:36:33 -06:00
i-norden	1126466d51	bump go-ethereum version	2021-12-17 12:19:44 -06:00
Arijit Das	76155a1516	Merge pull request #69 from vulcanize/v1.10.13-statediff-0.0.28 Upgrade geth-statediff to version v1.10.13-statediff-0.0.28.	2021-12-17 17:39:53 +05:30
Arijit Das	01ce8b26b5	Update go.mod.	2021-12-16 21:23:05 +05:30
Arijit Das	c5f2e209b4	Upgrade geth-statediff to version v1.10.13-statediff-0.0.28.	2021-12-15 18:32:03 +05:30
i-norden	6e604c3db6	bump go-ethereum version	2021-12-01 23:24:05 -06:00
Ian Norden	0c2a9bc171	Merge pull request #67 from vulcanize/rct_size_bug bump go-ethereum version	2021-12-01 22:47:37 -06:00
i-norden	f35c5cf71e	bump go-ethereum version	2021-12-01 22:46:45 -06:00
Ian Norden	fe94c5a33c	Merge pull request #66 from vulcanize/log_processing_fix bump go-ethereum version	2021-12-01 16:01:29 -06:00
i-norden	55a20ac15f	bump go-ethereum version	2021-12-01 16:00:53 -06:00
Ian Norden	eee4330bf4	Merge pull request #65 from vulcanize/log_processing_fix bump go-etheruem version	2021-11-30 18:21:12 -06:00
i-norden	5543a52472	bump go-etheruem version	2021-11-30 17:56:04 -06:00
Ian Norden	db786ae3b0	Merge pull request #62 from vulcanize/schema_updates Schema updates	2021-11-26 09:33:27 -06:00
i-norden	d8a3177a20	bump statediffing geth version	2021-11-26 09:29:22 -06:00
i-norden	fb54640823	add some missing ENVs	2021-11-26 09:25:01 -06:00
Ian Norden	731c986d75	Merge pull request #61 from vulcanize/schema_updates fix cli	2021-11-22 13:56:01 -06:00
i-norden	30b481db24	fix cli	2021-11-22 13:55:29 -06:00
Ian Norden	dcdfb9a05a	Merge pull request #60 from vulcanize/schema_updates update to new version of go-ethereum; minor README update	2021-11-20 14:50:19 -06:00
i-norden	d78a503ba4	update to new version of go-ethereum; minor README update	2021-11-20 14:49:36 -06:00
Ian Norden	ddbb690ee6	Merge pull request #59 from vulcanize/schema_updates reflect changes in integrated service	2021-11-18 19:25:36 -06:00
i-norden	bb5d5322d2	update README	2021-11-18 19:18:12 -06:00
i-norden	108a6f89f1	update to use new version of indexer; support file writing mode	2021-11-18 18:47:28 -06:00
i-norden	d9eb68b2d2	update to use refactored statediffing geth	2021-11-12 18:49:24 -06:00
i-norden	c5c230a2ea	update to use new version of statediffing geth pkgs with public.blocks insert batching	2021-11-08 19:08:36 -06:00
i-norden	dcf8afb96c	bump Go version to 1.16	2021-11-08 19:05:35 -06:00
Ian Norden	212bf4c4b2	Merge pull request #52 from vulcanize/add-grafana-dashboard Add grafana dashboard.	2021-11-04 10:09:21 -05:00
Arijit Das	a4a3ec1d7a	Add grafana dashboard.	2021-11-01 19:39:00 +05:30
Ian Norden	efb334aef3	Merge pull request #51 from vulcanize/prerun generate rand node id if none is provided	2021-10-28 10:52:35 -05:00
i-norden	c83e8835ab	generate rand node id if none is provided	2021-10-28 10:21:10 -05:00
S. R. Wadleigh	728642afec	Merge pull request #48 from vulcanize/make_startup_script_executable make startup_script.sh executable	2021-10-28 03:22:51 +00:00
Ian Norden	5875d1ec82	Update Dockerfile	2021-10-27 19:57:19 -05:00
erikdies	8d6919ab2b	make startup_script.sh executable	2021-10-27 20:46:25 -04:00
Ian Norden	abf72725ee	Merge pull request #47 from vulcanize/prerun cleanup	2021-10-27 17:46:13 -05:00
i-norden	8a55469403	cleanup	2021-10-27 17:45:47 -05:00
Ian Norden	a3857bffe4	Merge pull request #46 from vulcanize/prerun enable configuration of a prerun range by env variable; prerun only mode	2021-10-27 14:54:40 -05:00
i-norden	3956081874	sanitized prod config	2021-10-27 14:44:37 -05:00
i-norden	e1abb399cd	env vars for log stuff	2021-10-27 14:34:33 -05:00
i-norden	3d00e3ed05	enable configuration of a prerun range by env variable; prerun only mode	2021-10-27 13:36:19 -05:00
Ian Norden	84fff0aa03	Merge pull request #44 from vulcanize/stats make DB stats collector registration optional	2021-10-25 18:57:40 -05:00
i-norden	79cb508d83	make DB stats collector registration optional	2021-10-25 18:54:54 -05:00
Ian Norden	f51d89e873	Merge pull request #40 from vulcanize/metrics finish prom metrics	2021-10-25 15:40:53 -05:00
i-norden	a7ae868dd8	finish prom metrics	2021-10-25 15:40:11 -05:00
Ian Norden	ee59ff3650	Merge pull request #39 from vulcanize/metrics Metrics	2021-10-25 15:19:13 -05:00
i-norden	8571f35031	fixes	2021-10-25 14:54:55 -05:00
i-norden	f679082c3b	queue size metric; touchup	2021-10-25 14:27:11 -05:00
i-norden	aace7793ba	unify service RPC api and commands	2021-10-25 14:27:11 -05:00
Ian Norden	b49e30d8e9	Merge pull request #38 from vulcanize/metrics fix namespace	2021-10-24 18:39:07 -05:00
i-norden	f207284e3a	fix namespace	2021-10-24 18:37:01 -05:00
S. R. Wadleigh	3d3e68c0ae	Merge pull request #37 from vulcanize/feature-docker-overlay Update docker environment to support fs overlay	2021-10-21 20:19:53 +00:00
srwadleigh	66835c4c3e	add trailing space to start script	2021-10-21 19:07:19 +00:00
srwadleigh	0d092cc535	add sudo and user permissions to Dockerfile, update startup_script.sh with overlay commands	2021-10-21 19:02:22 +00:00
Ian Norden	a057940716	Merge pull request #35 from vulcanize/metrics Metrics	2021-10-21 13:36:49 -05:00
i-norden	7dc6c52dec	fix type	2021-10-21 13:29:30 -05:00
i-norden	ca9d2a7bec	update config	2021-10-21 13:10:16 -05:00
i-norden	072f035a02	wire up; customize metrics	2021-10-21 13:00:30 -05:00
i-norden	5a35f84de3	cli/cmd updates	2021-10-21 12:06:06 -05:00
i-norden	0f81dc32d2	prom pkg + monitoring	2021-10-21 11:57:44 -05:00
Arijit Das	6a912eef41	Merge pull request #34 from vulcanize/v1.10.11-statediff-0.0.27 Upgrade geth-statediff to version v1.10.11-statediff-0.0.27	2021-10-21 15:48:48 +05:30
Arijit Das	88f2eaed7f	Upgrade geth-statediff to version v1.10.11-statediff-0.0.27	2021-10-21 15:44:56 +05:30
Ian Norden	26a88f06e7	Merge pull request #33 from vulcanize/update fix CLI bug	2021-10-20 16:28:39 -05:00
i-norden	0dcec4ec8e	fix CLI bug	2021-10-20 16:25:52 -05:00
Ian Norden	4cbaa630c7	Merge pull request #30 from vulcanize/updates Updates	2021-10-19 17:25:50 -05:00
i-norden	fb37ee9881	cli and toml bindings; fix unit tests; remove unused code	2021-10-19 16:49:04 -05:00
i-norden	01d0c21fd5	make cache configurable	2021-10-19 16:19:44 -05:00
Arijit Das	64e80eeb6a	Merge pull request #28 from vulcanize/range-endpoint Add RPC endpoint to write statediff.	2021-10-19 20:03:59 +05:30
Arijit Das	05b90107ca	Add RPC endpoint to write statediff.	2021-10-19 17:27:51 +05:30
Arijit Das	c4b95cc1b7	Merge pull request #27 from vulcanize/upgrade-geth Upgrade geth to v1.10.10.	2021-10-19 11:44:40 +05:30
Arijit Das	7b9ca9b519	Update go.mod	2021-10-19 11:41:23 +05:30
Arijit Das	b09e58c22a	Upgrade geth to v1.10.9.	2021-10-14 12:17:38 +05:30
Ian Norden	450ff197a2	Merge pull request #20 from vulcanize/update update to use v1.10.1-statediff-0.0.16	2021-03-24 18:54:46 -05:00
Ian Norden	41e2ae176f	update to use v1.10.1-statediff-0.0.16; including fix for missing leaf keys in 'removed node' entries	2021-03-24 17:46:20 -05:00
Ian Norden	fa0be51041	Merge pull request #19 from vulcanize/updates Replicate latest statediffing geth updates here	2021-03-01 09:47:18 -06:00
Ian Norden	0315a31dd2	add endpoints for fetching/writing statediff by blockhash	2021-02-27 12:39:16 -06:00
Ian Norden	017daffb5a	update to v1.9.25-statediff-v0.0.15	2021-02-27 12:39:03 -06:00
Roy Crihfield	d3bed86fff	Write directly to postgres (#16 ) * Write directly to PostgreSQL (port from geth statediff) * Refactors for Node API change * add namespacing to avoid name conflicts	2020-12-03 11:04:14 +08:00
roysc	d61113ad12	doc headers	2020-12-03 10:08:20 +08:00
roysc	7a6b866163	docs, tweaks, fixes	2020-11-26 18:57:54 +08:00
roysc	bd5d2992fa	fix import add namespacing to avoid name conflicts	2020-11-17 09:37:36 +08:00
roysc	d886579f3d	go.sum	2020-11-16 18:16:01 +08:00
roysc	a93eb3ce35	fix http server	2020-11-16 18:15:53 +08:00
roysc	abff9bf045	refactors geth Node api change sd ctor refactor	2020-11-16 17:49:59 +08:00
roysc	ca8b3066d1	Write directly to PostgreSQL (port from geth statediff)	2020-11-15 15:18:34 +08:00
Ian Norden	294568da4e	Merge pull request #14 from vulcanize/github_actions GitHub actions	2020-11-09 12:53:37 -06:00
Ian Norden	a5eea3f428	fix Dockerfile	2020-11-09 12:49:33 -06:00
Ian Norden	05328a1763	Makefile	2020-11-09 12:19:31 -06:00
Ian Norden	4901b3d35b	github actions integration	2020-11-09 12:15:50 -06:00
Ian Norden	f0a631109e	env variables for server paths	2020-11-09 12:15:50 -06:00
Ian Norden	456043dc4c	Merge pull request #12 from vulcanize/codehash Emit codehash=>code mappings	2020-11-08 17:31:53 -06:00
Ian Norden	b2c00f9fc7	update go-modules	2020-11-08 17:30:43 -06:00
Ian Norden	e7d69e5d65	update unit tests to test codeAndCodeHash	2020-11-08 17:25:38 -06:00
Ian Norden	121f867a35	emit codeAndCodeHash in StateObject payloads	2020-11-08 17:25:22 -06:00
Ian Norden	f5be8d1a57	Merge pull request #10 from vulcanize/update-config Update config	2020-09-15 10:23:42 -05:00
roysc	03018fa58a	proper fail for worker count	2020-09-15 01:03:59 -05:00
roysc	d711fa7d7f	add workers to example.toml	2020-09-14 23:19:13 -05:00
roysc	e9e7144ac6	refactor tests	2020-09-14 22:51:54 -05:00
roysc	f251d82d9d	refactor config for workers	2020-09-14 22:51:54 -05:00
Ian Norden	d7003eb3be	Merge pull request #9 from vulcanize/fix-multithread Refactor geth statediff types back in	2020-09-14 21:07:19 -05:00
roysc	d99ceddcf8	refactor geth statediff types back in	2020-09-14 20:16:17 -05:00
Ian Norden	f8098cbefe	Merge pull request #6 from vulcanize/multithread Concurrent statediff	2020-09-09 12:12:16 -05:00
Ian Norden	1c12372be5	use tagged release of go-eth-state-node-iterator	2020-09-09 12:09:46 -05:00
roysc	7188c73032	goimports	2020-09-08 19:32:01 -05:00
roysc	cd80c2be42	update iterator	2020-09-08 19:04:20 -05:00
roysc	3100268677	nit	2020-09-08 01:32:29 -05:00
roysc	29e3d30a7d	update iterator	2020-09-08 01:32:25 -05:00
roysc	8ff057a1ea	clean up	2020-09-06 21:16:42 -05:00
roysc	c93735851a	Implement concurrent diff	2020-09-06 21:16:37 -05:00
roysc	d33949e32f	rip statediff from geth fork	2020-09-06 21:13:20 -05:00