Update to geth v1.11.6-statediff-5.0.8 (#135 )

* Update to geth v1.11.6-statediff-5.0.8
Merge pull request #133 from cerc-io/ian/v5_dev
2023-07-21 19:05:57 -05:00 · 2023-06-01 08:53:18 -05:00 · 2023-06-01 08:46:19 -05:00 · 2023-05-09 08:05:19 -07:00 · 2023-05-08 17:19:51 -05:00 · 2023-04-28 22:14:45 -05:00
34 changed files with 2977 additions and 2116 deletions
@@ -0,0 +1,35 @@
+name: MANUAL Override Publish from release SHA to TAG 
+on:
+  workflow_dispatch:
+    inputs:
+      giteaPublishTag:
+        description: 'Release TAG to publish TO on gitea; e.g. v4.1.5-alpha'
+        required: true
+      cercContainerTag:
+        description: 'Container (truncated!!! SHA) to release-tag FROM'
+        required: true
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    name: Pull SHA and add release-tag
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the version
+        id: vars
+        run: |
+          echo ::set-output name=sha::$(echo ${cercContainerTag:0:7})
+      - name: Pull docker image by SHA
+        run: docker pull git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}}
+      - name: Tag docker image TAG
+        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}}
+      - name: Tag docker image TAG
+        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.cercContainerTag}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest
+      - name: Docker Login
+        run: echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+      - name: Docker Push Release Tag
+        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{github.event.inputs.giteaPublishTag}}
+      - name: Docker Push LATEST Tag
+        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:latest
@@ -32,38 +32,42 @@ jobs:
    needs: run-tests
    steps:
      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag vulcanize/eth-statediff-service docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Push
-        run: docker push docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-  push_to_registries:
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    if: |
-      always() &&
-      (needs.build.result == 'success') &&
-      github.event_name == 'release'
-    needs: build
-    steps:
      - name: Get the version
        id: vars
        run: |
          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-      - name: Docker Login to Github Registry
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Pull
-        run: docker pull docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
-      - name: Docker Login to Docker Registry
-        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
+      - name: Run docker build
+        run: make docker-build
      - name: Tag docker image
-        run: docker tag docker.pkg.github.com/vulcanize/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} vulcanize/eth-statediff-service:${{steps.vars.outputs.tag}}
-      - name: Docker Push to Docker Hub
-        run: docker push vulcanize/eth-statediff-service:${{steps.vars.outputs.tag}}
+        run: docker tag cerc-io/eth-statediff-service git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+      - name: Tag docker image TAG
+        run: docker tag git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.tag}}
+      - name: Docker Login
+        run: echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+      - name: Docker Push
+        run: docker push git.vdb.to/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+#  push_to_registries:
+#    name: Push Docker image to Docker Hub
+#    runs-on: ubuntu-latest
+#    if: |
+#      always() &&
+#      (needs.build.result == 'success') &&
+#      github.event_name == 'release'
+#    needs: build
+#    steps:
+#      - name: Get the version
+#        id: vars
+#        run: |
+#          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+#          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
+#      - name: Docker Login to Github Registry
+#        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+#      - name: Docker Pull
+#        run: docker pull docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}}
+#      - name: Docker Login to Docker Registry
+#        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
+#      - name: Tag docker image
+#        run: docker tag docker.pkg.github.com/cerc-io/eth-statediff-service/eth-statediff-service:${{steps.vars.outputs.sha}} cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
+#      - name: Docker Push to Docker Hub
+#        run: docker push cerc-io/eth-statediff-service:${{steps.vars.outputs.tag}}
@@ -1,38 +1,34 @@
-FROM golang:1.18-alpine as builder
+FROM golang:1.19-alpine as builder

 RUN apk --update --no-cache add make git g++ linux-headers
 # DEBUG
 RUN apk add busybox-extras

 # Get and build ipfs-blockchain-watcher
-ADD . /go/src/github.com/vulcanize/eth-statediff-service
-#RUN git clone https://github.com/vulcanize/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service
+ADD . /go/src/github.com/cerc-io/eth-statediff-service
+#RUN git clone https://github.com/cerc-io/eth-statediff-service.git /go/src/github.com/vulcanize/eth-statediff-service

-WORKDIR /go/src/github.com/vulcanize/eth-statediff-service
+WORKDIR /go/src/github.com/cerc-io/eth-statediff-service
 RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o eth-statediff-service .

 # app container
 FROM alpine

-ARG USER="vdm"
 ARG CONFIG_FILE="./environments/config.toml"
 ARG EXPOSE_PORT=8545

-RUN adduser -Du 5000 $USER adm
-RUN adduser $USER adm; apk --no-cache add sudo; echo '%adm ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+RUN apk --no-cache add su-exec bash

 WORKDIR /app
-RUN chown $USER /app
-USER $USER

 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/$CONFIG_FILE config.toml
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/startup_script.sh .
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/eth-statediff-service/environments environments
+COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/$CONFIG_FILE config.toml
+COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/startup_script.sh .
+COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/environments environments

 # keep binaries immutable
-COPY --from=builder /go/src/github.com/vulcanize/eth-statediff-service/eth-statediff-service eth-statediff-service
+COPY --from=builder /go/src/github.com/cerc-io/eth-statediff-service/eth-statediff-service eth-statediff-service

 EXPOSE $EXPOSE_PORT

@@ -1,10 +1,10 @@
 ## Build docker image
 .PHONY: docker-build
 docker-build:
-	docker build -t vulcanize/eth-statediff-service .
+	docker build -t cerc-io/eth-statediff-service .

 .PHONY: test
-test: | $(GOOSE)
+test:
 	go test -p 1 ./pkg/... -v

 build:
@@ -8,7 +8,7 @@ Purpose:

 Stand up a statediffing service directly on top of a go-ethereum LevelDB instance.
 This service can serve historical state data over the same rpc interface as
-[statediffing geth](https://github.com/vulcanize/go-ethereum) without needing to run a full node.
+[statediffing geth](https://github.com/cerc-io/go-ethereum) without needing to run a full node.

 ## Setup

@@ -47,7 +47,8 @@ An example config file:
    trieWorkers     = 4     # STATEDIFF_TRIE_WORKERS

 [prerun]
-    only = false    # PRERUN_ONLY
+    only = false     # PRERUN_ONLY
+    parallel = true  # PRERUN_PARALLEL

    # to perform prerun in a specific range (optional)
    start = 0   # PRERUN_RANGE_START
@@ -60,8 +61,6 @@ An example config file:

    # statediffing params for prerun
    [prerun.params]
-        intermediateStateNodes   = true # PRERUN_INTERMEDIATE_STATE_NODES
-        intermediateStorageNodes = true # PRERUN_INTERMEDIATE_STORAGE_NODES
        includeBlock             = true # PRERUN_INCLUDE_BLOCK
        includeReceipts          = true # PRERUN_INCLUDE_RECEIPTS
        includeTD                = true # PRERUN_INCLUDE_TD
@@ -122,6 +121,9 @@ An example config file:
    # path to custom chain config file (optional)
    # keep chainID same as that in chain config file
    chainConfig  = "./chain.json"           # ETH_CHAIN_CONFIG
+
+[debug]
+    pprof = false                           # DEBUG_PPROF
 ```

 ### Local Setup
@@ -216,6 +218,8 @@ An example config file:

 * When `eth-statediff-service` is run in file mode (`database.type`) the output is in form of a SQL file or multiple CSV files.

+### SQL
+
 * Assuming the output files are located in host's `./output_dir` directory.

 * Create a directory to store post-processed output:
@@ -224,7 +228,11 @@ An example config file:
    mkdir -p output_dir/processed_output
    ```

-### SQL
+* (Optional) Get row counts in the output:
+
+    ```bash
+    wc -l output_dir/statediff.sql > output_stats.txt
+    ```

 * De-duplicate data:

@@ -242,86 +250,57 @@ An example config file:

 ### CSV

-* De-duplicate data and copy to post-processed output directory:
+* Create an env file with the required variables. Refer [.sample.env](./scripts/.sample.env).
+
+* (Optional) Get row counts in the output:

    ```bash
-    # public.blocks
-    sort -u output_dir/public.blocks.csv -o output_dir/processed_output/deduped-public.blocks.csv
-
-    # eth.header_cids
-    sort -u output_dir/eth.header_cids.csv -o output_dir/processed_output/deduped-eth.header_cids.csv
-
-    # eth.uncle_cids
-    sort -u output_dir/eth.uncle_cids.csv -o output_dir/processed_output/deduped-eth.uncle_cids.csv
-
-    # eth.transaction_cids
-    sort -u output_dir/eth.transaction_cids.csv -o output_dir/processed_output/deduped-eth.transaction_cids.csv
-
-    # eth.access_list_elements
-    sort -u output_dir/eth.access_list_elements.csv -o output_dir/processed_output/deduped-eth.access_list_elements.csv
-
-    # eth.receipt_cids
-    sort -u output_dir/eth.receipt_cids.csv -o output_dir/processed_output/deduped-eth.receipt_cids.csv
-
-    # eth.log_cids
-    sort -u output_dir/eth.log_cids.csv -o output_dir/processed_output/deduped-eth.log_cids.csv
-
-    # eth.state_cids
-    sort -u output_dir/eth.state_cids.csv -o output_dir/processed_output/deduped-eth.state_cids.csv
-
-    # eth.storage_cids
-    sort -u output_dir/eth.storage_cids.csv -o output_dir/processed_output/deduped-eth.storage_cids.csv
-
-    # eth.state_accounts
-    sort -u output_dir/eth.state_accounts.csv -o output_dir/processed_output/deduped-eth.state_accounts.csv
-
-    # public.nodes
-    cp output_dir/public.nodes.csv output_dir/processed_output/public.nodes.csv
+    ./scripts/count-lines.sh <ENV_FILE_PATH>
    ```

-* Copy over the post-processed output files to the DB server (say in `/output_dir`).
-
-* Start `psql` to run the import commands:
+* De-duplicate data:

    ```bash
-    psql -U <DATABASE_USER> -h <DATABASE_HOSTNAME> -p <DATABASE_PORT> <DATABASE_NAME>
+    ./scripts/dedup.sh <ENV_FILE_PATH>
    ```

-* Run the following to import data:
+* Perform column checks:

    ```bash
-    # public.nodes
-    COPY public.nodes FROM '/output_dir/processed_output/public.nodes.csv' CSV;
+    ./scripts/check-columns.sh <ENV_FILE_PATH>
+    ```

-    # public.nodes
-    COPY public.blocks FROM '/output_dir/processed_output/deduped-public.blocks.csv' CSV;
+    Check the output logs for any rows detected with unexpected number of columns.

-    # eth.header_cids
-    COPY eth.header_cids FROM '/output_dir/processed_output/deduped-eth.header_cids.csv' CSV;
+    Example:

-    # eth.uncle_cids
-    COPY eth.uncle_cids FROM '/output_dir/processed_output/deduped-eth.uncle_cids.csv' CSV;
+    ```bash
+    # log
+    eth.header_cids
+    Start: Wednesday 21 September 2022 06:00:38 PM IST
+    Time taken: 00:00:05
+    End: Wednesday 21 September 2022 06:00:43 PM IST
+    Total bad rows: 1 ./check-columns/eth.header_cids.txt

-    # eth.transaction_cids
-    COPY eth.transaction_cids FROM '/output_dir/processed_output/deduped-eth.transaction_cids.csv' CSV FORCE NOT NULL dst;
+    # bad row output
+    # line number, num. of columns, data
+    23 17 22,xxxxxx,0x07f5ea5c94aa8dea60b28f6b6315d92f2b6d78ca4b74ea409adeb191b5a114f2,0x5918487321aa57dd0c50977856c6231e7c4ee79e95b694c7c8830227d77a1ecc,bagiacgzaa726uxeuvkg6uyfsr5vwgfozf4vw26gkjn2ouqe232yzdnnbctza,45,geth,0,0xad8fa8df61b98dbda7acd6ca76d5ce4cbba663d5f608cc940957adcdb94cee8d,0xc621412320a20b4aaff5363bdf063b9d13e394ef82e55689ab703aae5db08e26,0x71ec1c7d81269ce115be81c81f13e1cc2601c292a7f20440a77257ecfdc69940,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,\x2000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000,1658408419,/blocks/DMQAP5PKLSKKVDPKMCZI623DCXMS6K3NPDFEW5HKICNN5MMRWWQRJ4Q,1,0x0000000000000000000000000000000000000000
+    ```

-    # eth.access_list_elements
-    COPY eth.access_list_elements FROM '/output_dir/processed_output/deduped-eth.access_list_elements.csv' CSV;
+* Import data using `timescaledb-parallel-copy`:  
+  (requires [`timescaledb-parallel-copy`](https://github.com/timescale/timescaledb-parallel-copy) installation; readily comes with TimescaleDB docker image)

-    # eth.receipt_cids
-    COPY eth.receipt_cids FROM '/output_dir/processed_output/deduped-eth.receipt_cids.csv' CSV FORCE NOT NULL post_state, contract, contract_hash;
-
-    # eth.log_cids
-    COPY eth.log_cids FROM '/output_dir/processed_output/deduped-eth.log_cids.csv' CSV FORCE NOT NULL topic0, topic1, topic2, topic3;
-
-    # eth.state_cids
-    COPY eth.state_cids FROM '/output_dir/processed_output/deduped-eth.state_cids.csv' CSV FORCE NOT NULL state_leaf_key;
-
-    # eth.storage_cids
-    COPY eth.storage_cids FROM '/output_dir/processed_output/deduped-eth.storage_cids.csv' CSV FORCE NOT NULL storage_leaf_key;
-
-    # eth.state_accounts
-    COPY eth.state_accounts FROM '/output_dir/processed_output/deduped-eth.state_accounts.csv' CSV;
+    ```bash
+    ./scripts/timescaledb-import.sh <ENV_FILE_PATH>
    ```

 * NOTE: `COPY` command on CSVs inserts empty strings as `NULL` in the DB. Passing `FORCE_NOT_NULL <COLUMN_NAME>` forces it to insert empty strings instead. This is required to maintain compatibility of the imported statediff data with the data generated in `postgres` mode. Reference: https://www.postgresql.org/docs/14/sql-copy.html
+
+### Stats
+
+The binary includes a `stats` command which reports stats for the offline or remote levelDB.
+
+At this time, the only stat supported is to return the latest/highest block height and hash found the levelDB, this is
+useful for determining what the upper limit is for a standalone statediffing process on a given levelDB.
+
+`./eth-statediff-service stats --config={path to toml config file}`
@@ -49,15 +49,14 @@ const (
 	PROM_HTTP_PORT = "PROM_HTTP_PORT"
 	PROM_DB_STATS  = "PROM_DB_STATS"

-	PRERUN_ONLY                       = "PRERUN_ONLY"
-	PRERUN_RANGE_START                = "PRERUN_RANGE_START"
-	PRERUN_RANGE_STOP                 = "PRERUN_RANGE_STOP"
-	PRERUN_INTERMEDIATE_STATE_NODES   = "PRERUN_INTERMEDIATE_STATE_NODES"
-	PRERUN_INTERMEDIATE_STORAGE_NODES = "PRERUN_INTERMEDIATE_STORAGE_NODES"
-	PRERUN_INCLUDE_BLOCK              = "PRERUN_INCLUDE_BLOCK"
-	PRERUN_INCLUDE_RECEIPTS           = "PRERUN_INCLUDE_RECEIPTS"
-	PRERUN_INCLUDE_TD                 = "PRERUN_INCLUDE_TD"
-	PRERUN_INCLUDE_CODE               = "PRERUN_INCLUDE_CODE"
+	PRERUN_ONLY             = "PRERUN_ONLY"
+	PRERUN_PARALLEL         = "PRERUN_PARALLEL"
+	PRERUN_RANGE_START      = "PRERUN_RANGE_START"
+	PRERUN_RANGE_STOP       = "PRERUN_RANGE_STOP"
+	PRERUN_INCLUDE_BLOCK    = "PRERUN_INCLUDE_BLOCK"
+	PRERUN_INCLUDE_RECEIPTS = "PRERUN_INCLUDE_RECEIPTS"
+	PRERUN_INCLUDE_TD       = "PRERUN_INCLUDE_TD"
+	PRERUN_INCLUDE_CODE     = "PRERUN_INCLUDE_CODE"

 	LOG_LEVEL     = "LOG_LEVEL"
 	LOG_FILE_PATH = "LOG_FILE_PATH"
@@ -81,6 +80,8 @@ const (
 	DATABASE_MAX_CONN_LIFETIME    = "DATABASE_MAX_CONN_LIFETIME"
 	DATABASE_CONN_TIMEOUT         = "DATABSE_CONN_TIMEOUT"
 	DATABASE_MAX_CONN_IDLE_TIME   = "DATABASE_MAX_CONN_IDLE_TIME"
+
+	DEBUG_PPROF = "DEBUG_PPROF"
 )

 // Bind env vars for eth node and DB configuration
@@ -135,10 +136,9 @@ func init() {

 	viper.BindEnv("statediff.prerun", STATEDIFF_PRERUN)
 	viper.BindEnv("prerun.only", PRERUN_ONLY)
+	viper.BindEnv("prerun.parallel", PRERUN_PARALLEL)
 	viper.BindEnv("prerun.start", PRERUN_RANGE_START)
 	viper.BindEnv("prerun.stop", PRERUN_RANGE_STOP)
-	viper.BindEnv("prerun.params.intermediateStateNodes", PRERUN_INTERMEDIATE_STATE_NODES)
-	viper.BindEnv("prerun.params.intermediateStorageNodes", PRERUN_INTERMEDIATE_STORAGE_NODES)
 	viper.BindEnv("prerun.params.includeBlock", PRERUN_INCLUDE_BLOCK)
 	viper.BindEnv("prerun.params.includeReceipts", PRERUN_INCLUDE_RECEIPTS)
 	viper.BindEnv("prerun.params.includeTD", PRERUN_INCLUDE_TD)
@@ -146,4 +146,6 @@ func init() {

 	viper.BindEnv("log.level", LOG_LEVEL)
 	viper.BindEnv("log.file", LOG_FILE_PATH)
+
+	viper.BindEnv("debug.pprof", DEBUG_PPROF)
 }
@@ -34,7 +34,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"

-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 var (
@@ -163,8 +163,6 @@ func init() {
 	rootCmd.PersistentFlags().Bool("prerun-only", false, "only process pre-configured ranges; exit afterwards")
 	rootCmd.PersistentFlags().Int("prerun-start", 0, "start height for a prerun range")
 	rootCmd.PersistentFlags().Int("prerun-stop", 0, "stop height for a prerun range")
-	rootCmd.PersistentFlags().Bool("prerun-intermediate-state-nodes", true, "include intermediate state nodes in state diff")
-	rootCmd.PersistentFlags().Bool("prerun-intermediate-storage-nodes", true, "include intermediate storage nodes in state diff")
 	rootCmd.PersistentFlags().Bool("prerun-include-block", true, "include block data in the statediff payload")
 	rootCmd.PersistentFlags().Bool("prerun-include-receipts", true, "include receipts in the statediff payload")
 	rootCmd.PersistentFlags().Bool("prerun-include-td", true, "include td in the statediff payload")
@@ -221,15 +219,16 @@ func init() {
 	viper.BindPFlag("prom.metrics", rootCmd.PersistentFlags().Lookup("prom-metrics"))

 	viper.BindPFlag("prerun.only", rootCmd.PersistentFlags().Lookup("prerun-only"))
+	viper.BindPFlag("prerun.parallel", rootCmd.PersistentFlags().Lookup("prerun-parallel"))
 	viper.BindPFlag("prerun.start", rootCmd.PersistentFlags().Lookup("prerun-start"))
 	viper.BindPFlag("prerun.stop", rootCmd.PersistentFlags().Lookup("prerun-stop"))
-	viper.BindPFlag("prerun.params.intermediateStateNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-state-nodes"))
-	viper.BindPFlag("prerun.params.intermediateStorageNodes", rootCmd.PersistentFlags().Lookup("prerun-intermediate-storage-nodes"))
 	viper.BindPFlag("prerun.params.includeBlock", rootCmd.PersistentFlags().Lookup("prerun-include-block"))
 	viper.BindPFlag("prerun.params.includeReceipts", rootCmd.PersistentFlags().Lookup("prerun-include-receipts"))
 	viper.BindPFlag("prerun.params.includeTD", rootCmd.PersistentFlags().Lookup("prerun-include-td"))
 	viper.BindPFlag("prerun.params.includeCode", rootCmd.PersistentFlags().Lookup("prerun-include-code"))

+	viper.BindPFlag("debug.pprof", rootCmd.PersistentFlags().Lookup("debug-pprof"))
+
 	rand.Seed(time.Now().UnixNano())
 }

@@ -301,11 +300,11 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 	if err != nil {
 		return nil, err
 	}
-	logWithCommand.Infof("configuring service for database type: %s", dbType)
+	logWithCommand.Infof("Configuring service for database type: %s", dbType)
 	var indexerConfig interfaces.Config
 	switch dbType {
 	case shared.FILE:
-		logWithCommand.Info("starting in sql file writing mode")
+		logWithCommand.Info("Starting in sql file writing mode")

 		fileModeStr := viper.GetString("database.fileMode")
 		fileMode, err := file.ResolveFileMode(fileModeStr)
@@ -315,12 +314,12 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {

 		filePathStr := viper.GetString("database.filePath")
 		if fileMode == file.SQL && filePathStr == "" {
-			logWithCommand.Fatal("when operating in sql file writing mode a file path must be provided")
+			logWithCommand.Fatal("When operating in sql file writing mode a file path must be provided")
 		}

 		fileCsvDirStr := viper.GetString("database.fileCsvDir")
 		if fileMode == file.CSV && fileCsvDirStr == "" {
-			logWithCommand.Fatal("when operating in csv file writing mode a directory path must be provided")
+			logWithCommand.Fatal("When operating in csv file writing mode a directory path must be provided")
 		}

 		indexerConfig = file.Config{
@@ -329,7 +328,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 			FilePath:  filePathStr,
 		}
 	case shared.DUMP:
-		logWithCommand.Info("starting in data dump mode")
+		logWithCommand.Info("Starting in data dump mode")
 		dumpDstStr := viper.GetString("database.dumpDestination")
 		dumpDst, err := dump.ResolveDumpType(dumpDstStr)
 		if err != nil {
@@ -346,7 +345,7 @@ func getConfig(nodeInfo node.Info) (interfaces.Config, error) {
 			return nil, fmt.Errorf("unrecognized dump destination: %s", dumpDst)
 		}
 	case shared.POSTGRES:
-		logWithCommand.Info("starting in postgres mode")
+		logWithCommand.Info("Starting in postgres mode")
 		driverTypeStr := viper.GetString("database.driver")
 		driverType, err := postgres.ResolveDriverType(driverTypeStr)
 		if err != nil {
@@ -16,8 +16,11 @@
 package cmd

 import (
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 	"os/signal"
+	"runtime"
 	"sync"

 	"github.com/ethereum/go-ethereum/rpc"
@@ -25,8 +28,8 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"

-	sd "github.com/vulcanize/eth-statediff-service/pkg"
-	srpc "github.com/vulcanize/eth-statediff-service/pkg/rpc"
+	sd "github.com/cerc-io/eth-statediff-service/pkg"
+	srpc "github.com/cerc-io/eth-statediff-service/pkg/rpc"
 )

 // serveCmd represents the serve command
@@ -47,18 +50,51 @@ func init() {
 	rootCmd.AddCommand(serveCmd)
 }

+func maxParallelism() int {
+	maxProcs := runtime.GOMAXPROCS(0)
+	numCPU := runtime.NumCPU()
+	if maxProcs < numCPU {
+		return maxProcs
+	}
+	return numCPU
+}
+
 func serve() {
 	logWithCommand.Info("Running eth-statediff-service serve command")
+	logWithCommand.Infof("Parallelism: %d", maxParallelism())

-	statediffService, err := createStateDiffService()
+	reader, chainConf, nodeInfo := instantiateLevelDBReader()
+
+	// report latest block info
+	header, err := reader.GetLatestHeader()
+	if err != nil {
+		logWithCommand.Fatalf("Unable to determine latest header height and hash: %s", err.Error())
+	}
+	if header.Number == nil {
+		logWithCommand.Fatal("Latest header found in levelDB has a nil block height")
+	}
+	logWithCommand.Infof("Latest block found in the levelDB\r\nheight: %s, hash: %s", header.Number.String(), header.Hash().Hex())
+
+	statediffService, err := createStateDiffService(reader, chainConf, nodeInfo)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}

+	// Enable the pprof agent if configured
+	if viper.GetBool("debug.pprof") {
+		// See: https://www.farsightsecurity.com/blog/txt-record/go-remote-profiling-20161028/
+		// For security reasons: do not use the default http multiplexor elsewhere in this process.
+		go func() {
+			logWithCommand.Info("Starting pprof listener on port 6060")
+			logWithCommand.Fatal(http.ListenAndServe("localhost:6060", nil))
+		}()
+	}
+
 	// short circuit if we only want to perform prerun
 	if viper.GetBool("prerun.only") {
-		if err := statediffService.Run(nil); err != nil {
-			logWithCommand.Fatal("unable to perform prerun: %v", err)
+		parallel := viper.GetBool("prerun.parallel")
+		if err := statediffService.Run(nil, parallel); err != nil {
+			logWithCommand.Fatal("Unable to perform prerun: %v", err)
 		}
 		return
 	}
@@ -88,17 +124,17 @@ func startServers(serv sd.StateDiffService) error {
 	ipcPath := viper.GetString("server.ipcPath")
 	httpPath := viper.GetString("server.httpPath")
 	if ipcPath == "" && httpPath == "" {
-		logWithCommand.Fatal("need an ipc path and/or an http path")
+		logWithCommand.Fatal("Need an ipc path and/or an http path")
 	}
 	if ipcPath != "" {
-		logWithCommand.Info("starting up IPC server")
+		logWithCommand.Info("Starting up IPC server")
 		_, _, err := srpc.StartIPCEndpoint(ipcPath, serv.APIs())
 		if err != nil {
 			return err
 		}
 	}
 	if httpPath != "" {
-		logWithCommand.Info("starting up HTTP server")
+		logWithCommand.Info("Starting up HTTP server")
 		_, err := srpc.StartHTTPEndpoint(httpPath, serv.APIs(), []string{"statediff"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
 		if err != nil {
 			return err
@@ -0,0 +1,54 @@
+// Copyright © 2022 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+// statsCmd represents the serve command
+var statsCmd = &cobra.Command{
+	Use:   "stats",
+	Short: "Report stats for cold levelDB",
+	Long: `Usage
+
+./eth-statediff-service stats --config={path to toml config file}`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *logrus.WithField("SubCommand", subCommand)
+		stats()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(statsCmd)
+}
+
+func stats() {
+	logWithCommand.Info("Running eth-statediff-service stats command")
+
+	reader, _, _ := instantiateLevelDBReader()
+
+	header, err := reader.GetLatestHeader()
+	if err != nil {
+		logWithCommand.Fatalf("Unable to determine latest header height and hash: %s", err.Error())
+	}
+	if header.Number == nil {
+		logWithCommand.Fatal("Latest header found in levelDB has a nil block height")
+	}
+	logWithCommand.Infof("Latest block found in the levelDB\r\nheight: %s, hash: %s", header.Number.String(), header.Hash().Hex())
+}
@@ -7,17 +7,85 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/statediff"
 	ind "github.com/ethereum/go-ethereum/statediff/indexer"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
 	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/spf13/viper"

-	sd "github.com/vulcanize/eth-statediff-service/pkg"
-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	sd "github.com/cerc-io/eth-statediff-service/pkg"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 type blockRange [2]uint64

-func createStateDiffService() (sd.StateDiffService, error) {
+func createStateDiffService(lvlDBReader sd.Reader, chainConf *params.ChainConfig, nodeInfo node.Info) (sd.StateDiffService, error) {
+	// create statediff service
+	logWithCommand.Info("Setting up database")
+	conf, err := getConfig(nodeInfo)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	logWithCommand.Info("Creating statediff indexer")
+	db, indexer, err := ind.NewStateDiffIndexer(context.Background(), chainConf, nodeInfo, conf)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	if conf.Type() == shared.POSTGRES && viper.GetBool("prom.dbStats") {
+		prom.RegisterDBCollector(viper.GetString("database.name"), db)
+	}
+
+	logWithCommand.Info("Creating statediff service")
+	sdConf := sd.Config{
+		ServiceWorkers:  viper.GetUint("statediff.serviceWorkers"),
+		TrieWorkers:     viper.GetUint("statediff.trieWorkers"),
+		WorkerQueueSize: viper.GetUint("statediff.workerQueueSize"),
+		PreRuns:         setupPreRunRanges(),
+	}
+	return sd.NewStateDiffService(lvlDBReader, indexer, sdConf)
+}
+
+func setupPreRunRanges() []sd.RangeRequest {
+	if !viper.GetBool("statediff.prerun") {
+		return nil
+	}
+	preRunParams := statediff.Params{
+		IncludeBlock:    viper.GetBool("prerun.params.includeBlock"),
+		IncludeReceipts: viper.GetBool("prerun.params.includeReceipts"),
+		IncludeTD:       viper.GetBool("prerun.params.includeTD"),
+		IncludeCode:     viper.GetBool("prerun.params.includeCode"),
+	}
+	var addrStrs []string
+	viper.UnmarshalKey("prerun.params.watchedAddresses", &addrStrs)
+	addrs := make([]common.Address, len(addrStrs))
+	for i, addrStr := range addrStrs {
+		addrs[i] = common.HexToAddress(addrStr)
+	}
+	preRunParams.WatchedAddresses = addrs
+	var rawRanges []blockRange
+	viper.UnmarshalKey("prerun.ranges", &rawRanges)
+	blockRanges := make([]sd.RangeRequest, len(rawRanges))
+	for i, rawRange := range rawRanges {
+		blockRanges[i] = sd.RangeRequest{
+			Start:  rawRange[0],
+			Stop:   rawRange[1],
+			Params: preRunParams,
+		}
+	}
+	if viper.IsSet("prerun.start") && viper.IsSet("prerun.stop") {
+		hardStart := viper.GetInt("prerun.start")
+		hardStop := viper.GetInt("prerun.stop")
+		blockRanges = append(blockRanges, sd.RangeRequest{
+			Start:  uint64(hardStart),
+			Stop:   uint64(hardStop),
+			Params: preRunParams,
+		})
+	}
+
+	return blockRanges
+}
+
+func instantiateLevelDBReader() (sd.Reader, *params.ChainConfig, node.Info) {
 	// load some necessary params
 	logWithCommand.Info("Loading statediff service parameters")
 	mode := viper.GetString("leveldb.mode")
@@ -50,7 +118,7 @@ func createStateDiffService() (sd.StateDiffService, error) {
 	}

 	if err != nil {
-		logWithCommand.Fatal(err)
+		logWithCommand.Fatalf("Unable to instantiate chain config: %s", err.Error())
 	}

 	// create LevelDB reader
@@ -68,75 +136,9 @@ func createStateDiffService() (sd.StateDiffService, error) {
 		Url:         url,
 		DBCacheSize: viper.GetInt("cache.database"),
 	}
-	lvlDBReader, err := sd.NewLvlDBReader(readerConf)
+	reader, err := sd.NewLvlDBReader(readerConf)
 	if err != nil {
-		logWithCommand.Fatal(err)
+		logWithCommand.Fatalf("Unable to instantiate levelDB reader: %s", err.Error())
 	}
-
-	// create statediff service
-	logWithCommand.Info("Setting up database")
-	conf, err := getConfig(nodeInfo)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-
-	logWithCommand.Info("Creating statediff indexer")
-	db, indexer, err := ind.NewStateDiffIndexer(context.Background(), chainConf, nodeInfo, conf)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	if conf.Type() == shared.POSTGRES && viper.GetBool("prom.dbStats") {
-		prom.RegisterDBCollector(viper.GetString("database.name"), db)
-	}
-
-	logWithCommand.Info("Creating statediff service")
-	sdConf := sd.Config{
-		ServiceWorkers:  viper.GetUint("statediff.serviceWorkers"),
-		TrieWorkers:     viper.GetUint("statediff.trieWorkers"),
-		WorkerQueueSize: viper.GetUint("statediff.workerQueueSize"),
-		PreRuns:         setupPreRunRanges(),
-	}
-	return sd.NewStateDiffService(lvlDBReader, indexer, sdConf)
-}
-
-func setupPreRunRanges() []sd.RangeRequest {
-	if !viper.GetBool("statediff.prerun") {
-		return nil
-	}
-	preRunParams := statediff.Params{
-		IntermediateStateNodes:   viper.GetBool("prerun.params.intermediateStateNodes"),
-		IntermediateStorageNodes: viper.GetBool("prerun.params.intermediateStorageNodes"),
-		IncludeBlock:             viper.GetBool("prerun.params.includeBlock"),
-		IncludeReceipts:          viper.GetBool("prerun.params.includeReceipts"),
-		IncludeTD:                viper.GetBool("prerun.params.includeTD"),
-		IncludeCode:              viper.GetBool("prerun.params.includeCode"),
-	}
-	var addrStrs []string
-	viper.UnmarshalKey("prerun.params.watchedAddresses", &addrStrs)
-	addrs := make([]common.Address, len(addrStrs))
-	for i, addrStr := range addrStrs {
-		addrs[i] = common.HexToAddress(addrStr)
-	}
-	preRunParams.WatchedAddresses = addrs
-	var rawRanges []blockRange
-	viper.UnmarshalKey("prerun.ranges", &rawRanges)
-	blockRanges := make([]sd.RangeRequest, len(rawRanges))
-	for i, rawRange := range rawRanges {
-		blockRanges[i] = sd.RangeRequest{
-			Start:  rawRange[0],
-			Stop:   rawRange[1],
-			Params: preRunParams,
-		}
-	}
-	if viper.IsSet("prerun.start") && viper.IsSet("prerun.stop") {
-		hardStart := viper.GetInt("prerun.start")
-		hardStop := viper.GetInt("prerun.stop")
-		blockRanges = append(blockRanges, sd.RangeRequest{
-			Start:  uint64(hardStart),
-			Stop:   uint64(hardStop),
-			Params: preRunParams,
-		})
-	}
-
-	return blockRanges
+	return reader, chainConf, nodeInfo
 }
@@ -20,7 +20,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"

-	v "github.com/vulcanize/eth-statediff-service/version"
+	v "github.com/cerc-io/eth-statediff-service/version"
 )

 // versionCmd represents the version command
@@ -17,8 +17,6 @@
    only = true
    ranges = []
    [prerun.params]
-        intermediateStateNodes = true
-        intermediateStorageNodes = true
        includeBlock = true
        includeReceipts = true
        includeTD = true
@@ -20,8 +20,6 @@
        [0, 1000]
    ]
    [prerun.params]
-        intermediateStateNodes = true
-        intermediateStorageNodes = true
        includeBlock = true
        includeReceipts = true
        includeTD = true
@@ -1,63 +1,58 @@
-module github.com/vulcanize/eth-statediff-service
+module github.com/cerc-io/eth-statediff-service

 go 1.18

 require (
-	github.com/ethereum/go-ethereum v1.10.23
-	github.com/jmoiron/sqlx v1.2.0 // indirect
-	github.com/prometheus/client_golang v1.4.0
-	github.com/sirupsen/logrus v1.7.0
+	github.com/cerc-io/leveldb-ethdb-rpc v1.1.13
+	github.com/ethereum/go-ethereum v1.11.6
+	github.com/jmoiron/sqlx v1.3.5 // indirect
+	github.com/prometheus/client_golang v1.14.0
+	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.3.0
 	github.com/spf13/viper v1.10.1
-	github.com/vulcanize/go-eth-state-node-iterator v1.1.5
-	github.com/vulcanize/leveldb-ethdb-rpc v0.1.7
 )

 require (
+	github.com/DataDog/zstd v1.5.2 // indirect
 	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.9.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect
+	github.com/cockroachdb/redact v1.1.3 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/deckarep/golang-set v1.8.0 // indirect
+	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
 	github.com/deepmap/oapi-codegen v1.8.2 // indirect
 	github.com/edsrzf/mmap-go v1.0.0 // indirect
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
-	github.com/georgysavva/scany v0.2.9 // indirect
+	github.com/georgysavva/scany v1.2.1 // indirect
+	github.com/getsentry/sentry-go v0.18.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/graph-gophers/graphql-go v1.3.0 // indirect
 	github.com/hashicorp/go-bexpr v0.1.10 // indirect
-	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
-	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect
 	github.com/huin/goupnp v1.0.3 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/influxdata/influxdb v1.8.3 // indirect
 	github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
+	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c // indirect
 	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
-	github.com/ipfs/bbloom v0.0.4 // indirect
-	github.com/ipfs/go-block-format v0.0.3 // indirect
 	github.com/ipfs/go-cid v0.2.0 // indirect
-	github.com/ipfs/go-datastore v0.5.1 // indirect
-	github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
-	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
-	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
-	github.com/ipfs/go-ipld-format v0.4.0 // indirect
-	github.com/ipfs/go-log v1.0.5 // indirect
-	github.com/ipfs/go-log/v2 v2.1.3 // indirect
-	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgconn v1.10.0 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
@@ -68,14 +63,16 @@ require (
 	github.com/jackc/pgx/v4 v4.13.0 // indirect
 	github.com/jackc/puddle v1.1.3 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
-	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
-	github.com/lib/pq v1.10.6 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/lib/pq v1.10.7 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
@@ -90,49 +87,44 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
 	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
-	github.com/pganalyze/pg_query_go/v2 v2.1.0 // indirect
+	github.com/pganalyze/pg_query_go/v2 v2.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.9.1 // indirect
-	github.com/prometheus/procfs v0.0.8 // indirect
-	github.com/prometheus/tsdb v0.10.0 // indirect
-	github.com/rjeczalik/notify v0.9.1 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.39.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/rs/cors v1.7.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/stretchr/testify v1.7.2 // indirect
+	github.com/status-im/keycard-go v0.2.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
-	github.com/thoas/go-funk v0.9.2 // indirect
-	github.com/tklauser/go-sysconf v0.3.6 // indirect
-	github.com/tklauser/numcpus v0.2.2 // indirect
-	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
-	github.com/urfave/cli/v2 v2.10.2 // indirect
+	github.com/thoas/go-funk v0.9.3 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
+	github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.17.0 // indirect
-	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
-	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
-	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/ini.v1 v1.66.2 // indirect
+	golang.org/x/crypto v0.6.0 // indirect
+	golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.1.6 // indirect
 )

-replace github.com/ethereum/go-ethereum v1.10.23 => github.com/vulcanize/go-ethereum v1.10.23-statediff-4.2.0-alpha
+replace github.com/ethereum/go-ethereum v1.11.6 => github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8
@@ -15,7 +15,7 @@

 package main

-import "github.com/vulcanize/eth-statediff-service/cmd"
+import "github.com/cerc-io/eth-statediff-service/cmd"

 func main() {
 	cmd.Execute()
@@ -45,11 +45,6 @@ func (api *PublicStateDiffAPI) StateDiffAt(ctx context.Context, blockNumber uint
 	return api.sds.StateDiffAt(blockNumber, params)
 }

-// StateTrieAt returns a state trie payload at the specific blockheight
-func (api *PublicStateDiffAPI) StateTrieAt(ctx context.Context, blockNumber uint64, params sd.Params) (*sd.Payload, error) {
-	return api.sds.StateTrieAt(blockNumber, params)
-}
-
 // WriteStateDiffAt writes a state diff object directly to DB at the specific blockheight
 func (api *PublicStateDiffAPI) WriteStateDiffAt(ctx context.Context, blockNumber uint64, params sd.Params) error {
 	return api.sds.WriteStateDiffAt(blockNumber, params)
@@ -25,11 +25,11 @@ import (
 	"sync"

 	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/log"
 	sd "github.com/ethereum/go-ethereum/statediff"
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	iter "github.com/ethereum/go-ethereum/trie/concurrent_iterator"
 	"github.com/sirupsen/logrus"
-
-	iter "github.com/vulcanize/go-eth-state-node-iterator"
 )

 type builder struct {
@@ -55,24 +55,24 @@ func NewBuilder(stateCache state.Database, workers uint) (sd.Builder, error) {

 // BuildStateDiffObject builds a statediff object from two blocks and the provided parameters
 func (sdb *builder) BuildStateDiffObject(args sd.Args, params sd.Params) (sdtypes.StateObject, error) {
-	var stateNodes []sdtypes.StateNode
-	var codeAndCodeHashes []sdtypes.CodeAndCodeHash
+	var stateNodes []sdtypes.StateLeafNode
+	var codeAndCodeHashes []sdtypes.IPLD
 	err := sdb.WriteStateDiffObject(
-		sdtypes.StateRoots{OldStateRoot: args.OldStateRoot, NewStateRoot: args.NewStateRoot},
-		params, sd.StateNodeAppender(&stateNodes), sd.CodeMappingAppender(&codeAndCodeHashes))
+		args,
+		params, sd.StateNodeAppender(&stateNodes), sd.IPLDMappingAppender(&codeAndCodeHashes))
 	if err != nil {
 		return sdtypes.StateObject{}, err
 	}
 	return sdtypes.StateObject{
-		BlockHash:         args.BlockHash,
-		BlockNumber:       args.BlockNumber,
-		Nodes:             stateNodes,
-		CodeAndCodeHashes: codeAndCodeHashes,
+		BlockHash:   args.BlockHash,
+		BlockNumber: args.BlockNumber,
+		Nodes:       stateNodes,
+		IPLDs:       codeAndCodeHashes,
 	}, nil
 }

 // WriteStateDiffObject writes a statediff object to output callback
-func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.CodeSink) error {
+func (sdb *builder) WriteStateDiffObject(args sd.Args, params sd.Params, output sdtypes.StateNodeSink, codeOutput sdtypes.IPLDSink) error {
 	// Load tries for old and new states
 	oldTrie, err := sdb.StateCache.OpenTrie(args.OldStateRoot)
 	if err != nil {
@@ -84,10 +84,10 @@ func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Para
 	}

 	// Split old and new tries into corresponding subtrie iterators
-	oldIters1 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
-	oldIters2 := iter.SubtrieIterators(oldTrie, sdb.numWorkers)
-	newIters1 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
-	newIters2 := iter.SubtrieIterators(newTrie, sdb.numWorkers)
+	oldIters1 := iter.SubtrieIterators(oldTrie.NodeIterator, sdb.numWorkers)
+	oldIters2 := iter.SubtrieIterators(oldTrie.NodeIterator, sdb.numWorkers)
+	newIters1 := iter.SubtrieIterators(newTrie.NodeIterator, sdb.numWorkers)
+	newIters2 := iter.SubtrieIterators(newTrie.NodeIterator, sdb.numWorkers)

 	// Create iterators ahead of time to avoid race condition in state.Trie access
 	// We do two state iterations per subtrie: one for new/updated nodes,
@@ -101,12 +101,12 @@ func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Para
 	}

 	// Dispatch workers to process trie data; sync and collect results here via channels
-	nodeChan := make(chan sdtypes.StateNode)
-	codeChan := make(chan sdtypes.CodeAndCodeHash)
+	nodeChan := make(chan sdtypes.StateLeafNode)
+	codeChan := make(chan sdtypes.IPLD)

 	go func() {
-		nodeSender := func(node sdtypes.StateNode) error { nodeChan <- node; return nil }
-		codeSender := func(code sdtypes.CodeAndCodeHash) error { codeChan <- code; return nil }
+		nodeSender := func(node sdtypes.StateLeafNode) error { nodeChan <- node; return nil }
+		ipldSender := func(code sdtypes.IPLD) error { codeChan <- code; return nil }
 		var wg sync.WaitGroup

 		for w := uint(0); w < sdb.numWorkers; w++ {
@@ -114,12 +114,8 @@ func (sdb *builder) WriteStateDiffObject(args sdtypes.StateRoots, params sd.Para
 			go func(worker uint) {
 				defer wg.Done()
 				var err error
-				if !params.IntermediateStateNodes {
-					err = sdb.BuildStateDiffWithoutIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender)
-				} else {
-					err = sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs[worker], params, nodeSender, codeSender)
-				}
-
+				logger := log.New("hash", args.BlockHash.Hex(), "number", args.BlockNumber)
+				err = sdb.BuildStateDiffWithIntermediateStateNodes(iterPairs[worker], params, nodeSender, ipldSender, logger, nil)
 				if err != nil {
 					logrus.Errorf("buildStateDiff error for worker %d, params %+v", worker, params)
 				}
@@ -1,72 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Contains a batch of utility type declarations used by the tests. As the node
-// operates on unique types, a lot of them are needed to check various features.
-
-package statediff
-
-import (
-	"sort"
-	"strings"
-)
-
-func sortKeys(data AccountMap) []string {
-	keys := make([]string, 0, len(data))
-	for key := range data {
-		keys = append(keys, key)
-	}
-	sort.Strings(keys)
-
-	return keys
-}
-
-// findIntersection finds the set of strings from both arrays that are equivalent
-// a and b must first be sorted
-// this is used to find which keys have been both "deleted" and "created" i.e. they were updated
-func findIntersection(a, b []string) []string {
-	lenA := len(a)
-	lenB := len(b)
-	iOfA, iOfB := 0, 0
-	updates := make([]string, 0)
-	if iOfA >= lenA || iOfB >= lenB {
-		return updates
-	}
-	for {
-		switch strings.Compare(a[iOfA], b[iOfB]) {
-		// -1 when a[iOfA] < b[iOfB]
-		case -1:
-			iOfA++
-			if iOfA >= lenA {
-				return updates
-			}
-			// 0 when a[iOfA] == b[iOfB]
-		case 0:
-			updates = append(updates, a[iOfA])
-			iOfA++
-			iOfB++
-			if iOfA >= lenA || iOfB >= lenB {
-				return updates
-			}
-			// 1 when a[iOfA] > b[iOfB]
-		case 1:
-			iOfB++
-			if iOfB >= lenB {
-				return updates
-			}
-		}
-	}
-}
@@ -19,7 +19,7 @@ package prom
 import (
 	"github.com/prometheus/client_golang/prometheus"

-	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	dbmetrics "github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
 )

 const (
@@ -29,7 +29,7 @@ const (

 // DBStatsGetter is an interface that gets sql.DBStats.
 type DBStatsGetter interface {
-	Stats() sql.Stats
+	Stats() dbmetrics.DbStats
 }

 // DBStatsCollector implements the prometheus.Collector interface.
@@ -16,9 +16,11 @@
 package statediff

 import (
+	"errors"
 	"fmt"
 	"math/big"

+	"github.com/cerc-io/leveldb-ethdb-rpc/pkg/client"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
@@ -26,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
-	"github.com/vulcanize/leveldb-ethdb-rpc/pkg/client"
 )

 // Reader interface required by the statediffing service
@@ -36,6 +37,7 @@ type Reader interface {
 	GetReceiptsByHash(hash common.Hash) (types.Receipts, error)
 	GetTdByHash(hash common.Hash) (*big.Int, error)
 	StateDB() state.Database
+	GetLatestHeader() (*types.Header, error)
 }

 // LvlDBReader exposes the necessary Reader methods on lvldb
@@ -60,7 +62,8 @@ func NewLvlDBReader(conf LvLDBReaderConfig) (*LvlDBReader, error) {
 	var err error

 	if conf.Mode == "local" {
-		edb, err = rawdb.NewLevelDBDatabaseWithFreezer(conf.Path, conf.DBCacheSize, 256, conf.AncientPath, "eth-statediff-service", true)
+		kvdb, _ := rawdb.NewLevelDBDatabase(conf.Path, conf.DBCacheSize, 256, "eth-statediff-service", true)
+		edb, err = rawdb.NewDatabaseWithFreezer(kvdb, conf.AncientPath, "eth-statediff-service", true)
 	}

 	if conf.Mode == "remote" {
@@ -129,3 +132,12 @@ func (ldr *LvlDBReader) GetTdByHash(hash common.Hash) (*big.Int, error) {
 func (ldr *LvlDBReader) StateDB() state.Database {
 	return ldr.stateDB
 }
+
+// GetLatestHeader gets the latest header from the levelDB
+func (ldr *LvlDBReader) GetLatestHeader() (*types.Header, error) {
+	header := rawdb.ReadHeadHeader(ldr.ethDB)
+	if header == nil {
+		return nil, errors.New("unable to read head header")
+	}
+	return header, nil
+}
@@ -1,37 +0,0 @@
-// VulcanizeDB
-// Copyright © 2020 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package rpc
-
-import "github.com/ethereum/go-ethereum/rpc"
-
-// checkModuleAvailability check that all names given in modules are actually
-// available API services.
-func checkModuleAvailability(modules []string, apis []rpc.API) (bad, available []string) {
-	availableSet := make(map[string]struct{})
-	for _, api := range apis {
-		if _, ok := availableSet[api.Namespace]; !ok {
-			availableSet[api.Namespace] = struct{}{}
-			available = append(available, api.Namespace)
-		}
-	}
-	for _, name := range modules {
-		if _, ok := availableSet[name]; !ok {
-			bad = append(bad, name)
-		}
-	}
-	return bad, available
-}
@@ -24,7 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"

-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 // StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules.
@@ -26,7 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"

-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 var (
@@ -33,7 +33,7 @@ import (
 	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 	"github.com/sirupsen/logrus"

-	"github.com/vulcanize/eth-statediff-service/pkg/prom"
+	"github.com/cerc-io/eth-statediff-service/pkg/prom"
 )

 const defaultQueueSize = 1024
@@ -48,13 +48,11 @@ type StateDiffService interface {
 	// Loop is the main event loop for processing state diffs
 	Loop(wg *sync.WaitGroup) error
 	// Run is a one-off command to run on a predefined set of ranges
-	Run(ranges []RangeRequest) error
+	Run(ranges []RangeRequest, parallel bool) error
 	// StateDiffAt method to get state diff object at specific block
 	StateDiffAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
 	// StateDiffFor method to get state diff object at specific block
 	StateDiffFor(blockHash common.Hash, params sd.Params) (*sd.Payload, error)
-	// StateTrieAt method to get state trie object at specific block
-	StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error)
 	// WriteStateDiffAt method to write state diff object directly to DB
 	WriteStateDiffAt(blockNumber uint64, params sd.Params) error
 	// WriteStateDiffFor method to get state trie object at specific block
@@ -117,19 +115,82 @@ func (sds *Service) APIs() []rpc.API {
 	}
 }

+func segmentRange(workers, start, stop uint64, params sd.Params) []RangeRequest {
+	segmentSize := ((stop - start) + 1) / workers
+	remainder := ((stop - start) + 1) % workers
+	numOfSegments := workers
+	if remainder > 0 {
+		numOfSegments++
+	}
+	segments := make([]RangeRequest, numOfSegments)
+	for i := range segments {
+		end := start + segmentSize - 1
+		if end > stop {
+			end = stop
+		}
+		segments[i] = RangeRequest{start, end, params}
+		start = end + 1
+	}
+	return segments
+}
+
 // Run does a one-off processing run on the provided RangeRequests + any pre-runs, exiting afterwards
-func (sds *Service) Run(rngs []RangeRequest) error {
+func (sds *Service) Run(rngs []RangeRequest, parallel bool) error {
 	for _, preRun := range sds.preruns {
-		logrus.Infof("processing prerun range (%d, %d)", preRun.Start, preRun.Stop)
-		for i := preRun.Start; i <= preRun.Stop; i++ {
-			if err := sds.WriteStateDiffAt(i, preRun.Params); err != nil {
-				return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, preRun.Start, preRun.Stop, err)
+		// if the rangeSize is smaller than the number of workers
+		// make sure we do synchronous processing to avoid quantization issues
+		rangeSize := (preRun.Stop - preRun.Start) + 1
+		numWorkers := uint64(sds.workers)
+		if rangeSize < numWorkers {
+			parallel = false
+		}
+		if parallel {
+			logrus.Infof("parallel processing prerun range (%d, %d) (%d blocks) divided into %d sized chunks with %d workers", preRun.Start, preRun.Stop,
+				rangeSize, rangeSize/numWorkers, numWorkers)
+			workChan := make(chan RangeRequest)
+			quitChan := make(chan struct{})
+			// spin up numWorkers number of worker goroutines
+			wg := new(sync.WaitGroup)
+			for i := 0; i < int(numWorkers); i++ {
+				wg.Add(1)
+				go func(id int) {
+					defer wg.Done()
+					for {
+						select {
+						case workerSegment := <-workChan:
+							for j := workerSegment.Start; j <= workerSegment.Stop; j++ {
+								if err := sds.WriteStateDiffAt(j, workerSegment.Params); err != nil {
+									logrus.Errorf("error writing statediff at height %d in range (%d, %d) : %v", id, workerSegment.Start, workerSegment.Stop, err)
+								}
+							}
+							logrus.Infof("prerun worker %d finished processing range (%d, %d)", id, workerSegment.Start, workerSegment.Stop)
+						case <-quitChan:
+							return
+						}
+					}
+				}(i)
+			}
+			// break range up into segments
+			segments := segmentRange(numWorkers, preRun.Start, preRun.Stop, preRun.Params)
+			// send the segments to the work channel
+			for _, segment := range segments {
+				workChan <- segment
+			}
+			close(quitChan)
+			wg.Wait()
+		} else {
+			logrus.Infof("sequential processing prerun range (%d, %d)", preRun.Start, preRun.Stop)
+			for i := preRun.Start; i <= preRun.Stop; i++ {
+				if err := sds.WriteStateDiffAt(i, preRun.Params); err != nil {
+					return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, preRun.Start, preRun.Stop, err)
+				}
 			}
 		}
 	}
 	sds.preruns = nil
+	// At present this code is never called so we have not written the parallel version:
 	for _, rng := range rngs {
-		logrus.Infof("processing prerun range (%d, %d)", rng.Start, rng.Stop)
+		logrus.Infof("processing requested range (%d, %d)", rng.Start, rng.Stop)
 		for i := rng.Start; i <= rng.Stop; i++ {
 			if err := sds.WriteStateDiffAt(i, rng.Params); err != nil {
 				return fmt.Errorf("error writing statediff at height %d in range (%d, %d) : %v", i, rng.Start, rng.Stop, err)
@@ -281,34 +342,6 @@ func (sds *Service) newPayload(stateObject []byte, block *types.Block, params sd
 	return payload, nil
 }

-// StateTrieAt returns a state trie object payload at the specified blockheight
-// This operation cannot be performed back past the point of db pruning; it requires an archival node for historical data
-func (sds *Service) StateTrieAt(blockNumber uint64, params sd.Params) (*sd.Payload, error) {
-	currentBlock, err := sds.lvlDBReader.GetBlockByNumber(blockNumber)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Infof("sending state trie at block %d", blockNumber)
-
-	// compute leaf paths of watched addresses in the params
-	params.ComputeWatchedAddressesLeafPaths()
-
-	return sds.processStateTrie(currentBlock, params)
-}
-
-func (sds *Service) processStateTrie(block *types.Block, params sd.Params) (*sd.Payload, error) {
-	stateNodes, err := sds.Builder.BuildStateTrieObject(block)
-	if err != nil {
-		return nil, err
-	}
-	stateTrieRlp, err := rlp.EncodeToBytes(&stateNodes)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Infof("state trie object at block %d is %d bytes in length", block.Number().Uint64(), len(stateTrieRlp))
-	return sds.newPayload(stateTrieRlp, block, params)
-}
-
 // Start is used to begin the service
 func (sds *Service) Start() error {
 	logrus.Info("starting statediff service")
@@ -398,15 +431,15 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
 		return err
 	}
 	// defer handling of commit/rollback for any return case
-	output := func(node sdtypes.StateNode) error {
+	output := func(node sdtypes.StateLeafNode) error {
 		return sds.indexer.PushStateNode(tx, node, block.Hash().String())
 	}
-	codeOutput := func(c sdtypes.CodeAndCodeHash) error {
-		return sds.indexer.PushCodeAndCodeHash(tx, c)
+	codeOutput := func(c sdtypes.IPLD) error {
+		return sds.indexer.PushIPLD(tx, c)
 	}
 	prom.SetTimeMetric(prom.T_BLOCK_PROCESSING, time.Now().Sub(t))
 	t = time.Now()
-	err = sds.Builder.WriteStateDiffObject(sdtypes.StateRoots{
+	err = sds.Builder.WriteStateDiffObject(sd.Args{
 		NewStateRoot: block.Root(),
 		OldStateRoot: parentRoot,
 	}, params, output, codeOutput)
@@ -20,23 +20,9 @@
 package statediff

 import (
-	"github.com/ethereum/go-ethereum/core/types"
 	sd "github.com/ethereum/go-ethereum/statediff"
-	sdTypes "github.com/ethereum/go-ethereum/statediff/types"
 )

-// AccountMap is a mapping of hex encoded path => account wrapper
-type AccountMap map[string]accountWrapper
-
-// accountWrapper is used to temporary associate the unpacked node with its raw values
-type accountWrapper struct {
-	Account   *types.StateAccount
-	NodeType  sdTypes.NodeType
-	Path      []byte
-	NodeValue []byte
-	LeafKey   []byte
-}
-
 // RangeRequest holds range quest work params
 type RangeRequest struct {
 	Start, Stop uint64
@@ -0,0 +1,28 @@
+# Used by the script to count rows (count-lines.sh)
+COUNT_LINES_LOG=./count-lines.log
+COUNT_LINES_INPUT_DIR=~/eth-statediff-service/output_dir
+COUNT_LINES_OUTPUT_FILE=./output-stats.txt
+
+# Used by the script to dedup output files (dedup.sh)
+DEDUP_LOG=./dedup.log
+DEDUP_INPUT_DIR=~/eth-statediff-service/output_dir
+DEDUP_OUTPUT_DIR=~/eth-statediff-service/dedup_dir
+DEDUP_SORT_DIR=./.sort
+
+# Used by the script to perform column checks (check-columns.sh)
+CHECK_COLUMNS_LOG=./check-columns.log
+CHECK_COLUMNS_INPUT_DIR=~/eth-statediff-service/output_dir
+CHECK_COLUMNS_INPUT_DEDUP_DIR=~/eth-statediff-service/dedup_dir
+CHECK_COLUMNS_OUTPUT_DIR=./check-columns
+
+# Used by the script to import data (timescaledb-import.sh)
+IMPORT_LOG=./tsdb-import.log
+IMPORT_INPUT_DIR=~/eth-statediff-service/output_dir
+IMPORT_INPUT_DEDUP_DIR=~/eth-statediff-service/dedup_dir
+TIMESCALEDB_WORKERS=8
+
+DATABASE_USER=vdbm
+DATABASE_HOSTNAME=localhost
+DATABASE_PORT=8077
+DATABASE_NAME=vulcanize_testing
+DATABASE_PASSWORD=password
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# Requires:
+# CHECK_COLUMNS_LOG
+# CHECK_COLUMNS_INPUT_DIR
+# CHECK_COLUMNS_INPUT_DEDUP_DIR
+# CHECK_COLUMNS_OUTPUT_DIR
+
+# env file arg
+ENV=$1
+echo "Using env file: ${ENV}"
+
+# read env file
+export $(grep -v '^#' ${ENV} | xargs)
+
+# redirect stdout/stderr to a file
+exec >"${CHECK_COLUMNS_LOG}" 2>&1
+
+# create output dir if not exists
+mkdir -p "${CHECK_COLUMNS_OUTPUT_DIR}"
+
+start_timestamp=$(date +%s)
+
+declare -A expected_columns
+expected_columns=(
+  ["public.nodes"]="5"
+  ["public.blocks"]="3"
+  # ["eth.access_list_elements"]="?" # skipping as values include ','
+  ["eth.log_cids"]="12"
+  ["eth.state_accounts"]="7"
+  ["eth.storage_cids"]="9"
+  ["eth.uncle_cids"]="7"
+  ["eth.header_cids"]="16"
+  ["eth.receipt_cids"]="10"
+  ["eth.state_cids"]="8"
+  ["eth.transaction_cids"]="11"
+)
+
+for table_name in "${!expected_columns[@]}";
+do
+  if [ "${table_name}" = "public.blocks" ];
+  then
+    command="$(dirname "$0")/find-bad-rows.sh -i ${CHECK_COLUMNS_INPUT_DEDUP_DIR}/deduped-${table_name}.csv -c ${expected_columns[${table_name}]} -d true -o ${CHECK_COLUMNS_OUTPUT_DIR}/${table_name}.txt"
+  else
+    command="$(dirname "$0")/find-bad-rows.sh -i ${CHECK_COLUMNS_INPUT_DIR}/${table_name}.csv -c ${expected_columns[${table_name}]} -d true -o ${CHECK_COLUMNS_OUTPUT_DIR}/${table_name}.txt"
+  fi
+
+  echo "${table_name}"
+  echo Start: "$(date)"
+  eval "${command}"
+  echo End: "$(date)"
+  echo Total bad rows: $(wc -l ${CHECK_COLUMNS_OUTPUT_DIR}/${table_name}.txt)
+  echo
+done
+
+difference=$(($(date +%s)-start_timestamp))
+echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)
+echo
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Requires:
+# COUNT_LINES_LOG
+# COUNT_LINES_INPUT_DIR
+# COUNT_LINES_OUTPUT_FILE
+
+# env file arg
+ENV=$1
+echo "Using env file: ${ENV}"
+
+# read env file
+export $(grep -v '^#' ${ENV} | xargs)
+
+# redirect stdout/stderr to a file
+exec >"${COUNT_LINES_LOG}" 2>&1
+
+start_timestamp=$(date +%s)
+
+table_names=(
+  "public.nodes"
+  "public.blocks"
+  "eth.access_list_elements"
+  "eth.log_cids"
+  "eth.state_accounts"
+  "eth.storage_cids"
+  "eth.uncle_cids"
+  "eth.header_cids"
+  "eth.receipt_cids"
+  "eth.state_cids"
+  "eth.transaction_cids"
+)
+
+echo "Row counts:" > "${COUNT_LINES_OUTPUT_FILE}"
+
+for table_name in "${table_names[@]}";
+do
+  echo "${table_name}";
+  echo Start: "$(date)"
+  wc -l "${COUNT_LINES_INPUT_DIR}"/"${table_name}.csv" >> "${COUNT_LINES_OUTPUT_FILE}"
+  echo End: "$(date)"
+  echo
+done
+
+difference=$(($(date +%s)-start_timestamp))
+echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Requires:
+# DEDUP_LOG
+# DEDUP_INPUT_DIR
+# DEDUP_OUTPUT_DIR
+# DEDUP_SORT_DIR
+
+# env file arg
+ENV=$1
+echo "Using env file: ${ENV}"
+
+# read env file
+export $(grep -v '^#' ${ENV} | xargs)
+
+# redirect stdout/stderr to a file
+exec >"${DEDUP_LOG}" 2>&1
+
+# create output dir if not exists
+mkdir -p "${DEDUP_OUTPUT_DIR}"
+
+start_timestamp=$(date +%s)
+
+echo "public.blocks"
+echo Start: "$(date)"
+sort -T "${DEDUP_SORT_DIR}" -u "${DEDUP_INPUT_DIR}"/public.blocks.csv -o "${DEDUP_OUTPUT_DIR}"/deduped-public.blocks.csv
+echo End: "$(date)"
+echo Total deduped rows: $(wc -l ${DEDUP_OUTPUT_DIR}/deduped-public.blocks.csv)
+echo
+
+difference=$(($(date +%s)-start_timestamp))
+echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)
+
+# NOTE: This script currently only dedups public.blocks output file.
+# If the output contains blocks that were statediffed more than once, output files for other tables will have to be deduped as well.
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# flags
+# -i <input-file>:        Input data file path
+# -c <expected-columns>:  Expected number of columns in each row of the input file
+# -o [output-file]:       Output destination file path (default: STDOUT)
+# -d [include-data]:      Whether to include the data row in output (true | false) (default: false)
+
+# eg: ./scripts/find-bad-rows.sh -i eth.state_cids.csv -c 8 -o res.txt -d true
+# output: 1 9 1500000,xxxxxxxx,0x83952d392f9b0059eea94b10d1a095eefb1943ea91595a16c6698757127d4e1c,,
+#         baglacgzasvqcntdahkxhufdnkm7a22s2eetj6mx6nzkarwxtkvy4x3bubdgq,\x0f,0,f,/blocks/,
+#         DMQJKYBGZRQDVLT2CRWVGPQNNJNCCJU7GL7G4VAI3LZVK4OL5Q2ARTI
+
+while getopts i:c:o:d: OPTION
+do
+  case "${OPTION}" in
+    i) inputFile=${OPTARG};;
+    c) expectedColumns=${OPTARG};;
+    o) outputFile=${OPTARG};;
+    d) data=${OPTARG};;
+  esac
+done
+
+timestamp=$(date +%s)
+
+# if data requested, dump row number, number of columns and the row
+if [ "${data}" = true ] ; then
+  if [ -z "${outputFile}" ]; then
+    awk -F"," "NF!=${expectedColumns} {print NR, NF, \$0}" < ${inputFile}
+  else
+    awk -F"," "NF!=${expectedColumns} {print NR, NF, \$0}" < ${inputFile} > ${outputFile}
+  fi
+# else, dump only row number, number of columns
+else
+  if [ -z "${outputFile}" ]; then
+    awk -F"," "NF!=${expectedColumns} {print NR, NF}" < ${inputFile}
+  else
+    awk -F"," "NF!=${expectedColumns} {print NR, NF}" < ${inputFile} > ${outputFile}
+  fi
+fi
+
+difference=$(($(date +%s)-timestamp))
+echo Time taken: $(date -d@${difference} -u +%H:%M:%S)
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Requires:
+# IMPORT_LOG
+# IMPORT_INPUT_DIR
+# IMPORT_INPUT_DEDUP_DIR
+# TIMESCALEDB_WORKERS
+# DATABASE_USER
+# DATABASE_HOSTNAME
+# DATABASE_PORT
+# DATABASE_NAME
+# DATABASE_PASSWORD
+
+DEFAULT_TIMESCALEDB_WORKERS=8
+
+# env file arg
+ENV=$1
+echo "Using env file: ${ENV}"
+
+# read env file
+export $(grep -v '^#' ${ENV} | xargs)
+
+if [ "$TIMESCALEDB_WORKERS" = "" ]; then
+	TIMESCALEDB_WORKERS=$DEFAULT_TIMESCALEDB_WORKERS
+fi
+
+# redirect stdout/stderr to a file
+exec >"${IMPORT_LOG}" 2>&1
+
+start_timestamp=$(date +%s)
+
+declare -a tables
+# schema-table-copyOptions
+tables=(
+  "public-nodes"
+  "public-blocks"
+  "eth-access_list_elements"
+  "eth-log_cids-FORCE NOT NULL topic0, topic1, topic2, topic3 CSV"
+  "eth-state_accounts"
+  "eth-storage_cids-FORCE NOT NULL storage_leaf_key CSV"
+  "eth-uncle_cids"
+  "eth-header_cids"
+  "eth-receipt_cids-FORCE NOT NULL post_state, contract, contract_hash CSV"
+  "eth-state_cids-FORCE NOT NULL state_leaf_key CSV"
+  "eth-transaction_cids-FORCE NOT NULL dst CSV"
+)
+
+for elem in "${tables[@]}";
+do
+	IFS='-' read -a arr <<< "${elem}"
+
+	if [ "${arr[0]}.${arr[1]}" = "public.blocks" ];
+	then
+		copy_command="timescaledb-parallel-copy --connection \"host=${DATABASE_HOSTNAME} port=${DATABASE_PORT} user=${DATABASE_USER} password=${DATABASE_PASSWORD} sslmode=disable\" --db-name ${DATABASE_NAME} --schema ${arr[0]} --table ${arr[1]} --file ${IMPORT_INPUT_DEDUP_DIR}/deduped-${arr[0]}.${arr[1]}.csv --workers ${TIMESCALEDB_WORKERS} --reporting-period 300s"
+	else
+		copy_command="timescaledb-parallel-copy --connection \"host=${DATABASE_HOSTNAME} port=${DATABASE_PORT} user=${DATABASE_USER} password=${DATABASE_PASSWORD} sslmode=disable\" --db-name ${DATABASE_NAME} --schema ${arr[0]} --table ${arr[1]} --file ${IMPORT_INPUT_DIR}/${arr[0]}.${arr[1]}.csv --workers ${TIMESCALEDB_WORKERS} --reporting-period 300s"
+	fi
+
+	if [ "${arr[2]}" != "" ];
+	then
+		copy_with_options="${copy_command} --copy-options \"${arr[2]}\""
+	else
+		copy_with_options=${copy_command}
+	fi
+
+	echo "${arr[0]}.${arr[1]}"
+	echo Start: "$(date)"
+	eval "${copy_with_options}"
+	echo End: "$(date)"
+	echo
+done
+
+difference=$(($(date +%s)-start_timestamp))
+echo Time taken: $((difference/86400)):$(date -d@${difference} -u +%H:%M:%S)
+echo
@@ -1,20 +1,62 @@
-#!/bin/sh
+#!/bin/bash
 # Exit if the variable tests fail
 set -e
-set +x
+set -o pipefail
+
+if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+    set -x
+fi

 # Check the database variables are set
 test "$VDB_COMMAND"
-set +e

 # docker must be run in privilaged mode for mounts to work
 echo "Setting up /app/geth-rw overlayed /app/geth-ro"
-mkdir -p /tmp/overlay && \
-sudo mount -t tmpfs tmpfs /tmp/overlay && \
-mkdir -p /tmp/overlay/upper && \
-mkdir -p /tmp/overlay/work && \
-mkdir -p /app/geth-rw && \
-sudo mount -t overlay overlay -o lowerdir=/app/geth-ro,upperdir=/tmp/overlay/upper,workdir=/tmp/overlay/work /app/geth-rw && \
+mkdir -p /tmp/overlay
+mount -t tmpfs tmpfs /tmp/overlay
+mkdir -p /tmp/overlay/upper
+mkdir -p /tmp/overlay/work
+mkdir -p /app/geth-rw

+mount -t overlay overlay -o lowerdir=/app/geth-ro,upperdir=/tmp/overlay/upper,workdir=/tmp/overlay/work /app/geth-rw
+
+mkdir /var/run/statediff
+cd /var/run/statediff
+
+SETUID=""
+if [[ -n "$TARGET_UID" ]] && [[ -n "$TARGET_GID" ]]; then
+    SETUID="su-exec $TARGET_UID:$TARGET_GID"
+    chown -R $TARGET_UID:$TARGET_GID /var/run/statediff
+fi
+
+START_TIME=`date -u +"%Y-%m-%dT%H:%M:%SZ"`
 echo "Running the statediff service" && \
-sudo ./eth-statediff-service "$VDB_COMMAND" --config=config.toml
+if [[ ! -z "$LOG_FILE_PATH" ]]; then
+  $SETUID /app/eth-statediff-service "$VDB_COMMAND" --config=/app/config.toml $* |& $SETUID tee ${LOG_FILE_PATH}.console
+  rc=$?
+else
+  $SETUID /app/eth-statediff-service "$VDB_COMMAND" --config=/app/config.toml $*
+  rc=$?
+fi
+STOP_TIME=`date -u +"%Y-%m-%dT%H:%M:%SZ"`
+
+if [ $rc -eq 0 ] && [ "$VDB_COMMAND" == "serve" ] && [ "$PRERUN_ONLY" == "true" ] && [ ! -z "$PRERUN_RANGE_START" ] && [ ! -z "$PRERUN_RANGE_STOP" ] && [ ! -z "$DATABASE_FILE_CSV_DIR" ] && [ "$DATABASE_FILE_MODE" == "csv" ]; then
+  cat >metadata.json <<EOF
+{
+  "type": "statediff",
+  "range": { "start": $PRERUN_RANGE_START, "stop": $PRERUN_RANGE_STOP },
+  "nodeId": "$ETH_NODE_ID",
+  "genesisBlock": "$ETH_GENESIS_BLOCK",
+  "networkId": "$ETH_NETWORK_ID",
+  "chainId": "$ETH_CHAIN_ID",
+  "time": { "start": "$START_TIME", "stop": "$STOP_TIME" }
+}
+EOF
+  if [[ -n "$TARGET_UID" ]] && [[ -n "$TARGET_GID" ]]; then
+    echo 'metadata.json' | cpio -p --owner $TARGET_UID:$TARGET_GID $DATABASE_FILE_CSV_DIR
+  else
+    cp metadata.json $DATABASE_FILE_CSV_DIR
+  fi
+fi
+
+exit $rc
Author	SHA1	Message	Date
telackey	151a805b53	Update to geth v1.11.6-statediff-5.0.8 (#135 ) * Update to geth v1.11.6-statediff-5.0.8	2023-07-21 19:05:57 -05:00
Ian Norden	232cb1bcbf	Merge pull request #133 from cerc-io/ian/v5_dev update geth dep	2023-06-01 08:53:18 -05:00
i-norden	36c104f4db	update geth dep	2023-06-01 08:46:19 -05:00
dboreham	f144863b3b	Merge pull request #131 from cerc-io/telackey/type Include type in metadata output	2023-05-09 08:05:19 -07:00
telackey	5dafc1fbc0	Include type	2023-05-08 17:19:51 -05:00
telackey	fb040d7484	Run as root, but switch to the target UID/GID is supplied (fixes file permission issues with mounted volumes). (#130 )	2023-04-28 22:14:45 -05:00
telackey	60e951a172	tee logs (#129 )	2023-04-27 22:24:22 -05:00
telackey	9016be38a0	Merge pull request #128 from cerc-io/telackey/docker_prerun Output metadata on prerun.	2023-04-27 11:42:52 -05:00
telackey	a17ace0187	Output metadata on prerun.	2023-04-26 22:58:28 -05:00
Ian Norden	5a623fa8d3	Merge pull request #127 from cerc-io/ian/v5_dev fix builder unit tests	2023-04-10 01:37:49 -05:00
i-norden	75c2c4c348	don't skip	2023-04-10 01:27:53 -05:00
i-norden	834930a498	fix builder unit tests	2023-04-10 01:18:34 -05:00
Ian Norden	0a92769bc5	Merge pull request #126 from cerc-io/ian/v5_dev revert to using os.Interrupt	2023-04-09 14:33:19 -05:00
i-norden	8bcb2c312f	revert to using os.Interrupt	2023-04-09 14:26:21 -05:00
Ian Norden	2bb0b1aad6	Merge pull request #125 from cerc-io/ian/v5_dev Update to v5	2023-04-09 13:35:09 -05:00
i-norden	4ddd3d0e20	go mod tidy	2023-04-09 11:01:31 -05:00
i-norden	07bd875752	update tomls and README	2023-04-09 11:01:23 -05:00
i-norden	f3379e33dd	update cmd to v5	2023-04-09 11:01:12 -05:00
i-norden	2ff4e95040	update pkgs to v5	2023-04-09 11:01:05 -05:00
Ian Norden	18a7f23173	Merge pull request #124 from cerc-io/ian/v4_dev bump leveldb-ethdb-rpc version	2023-03-31 10:25:24 -05:00
i-norden	ef1846f58c	bump leveldb-ethdb-rpc version	2023-03-31 10:17:34 -05:00
Ian Norden	2eaa2c2262	Merge pull request #123 from cerc-io/ian/v4_dev [v4] bump to vdb v4 geth v1.11.5	2023-03-31 09:53:11 -05:00
i-norden	bfb0447710	use internal concurrent iterator pkg	2023-03-31 09:04:54 -05:00
i-norden	2c08f5594c	bump to vdb v4 geth v1.11.5	2023-03-31 09:02:54 -05:00
Ian Norden	253b1087bf	Merge pull request #112 from cerc-io/ian_test_3 Avoid quantization problems during segmentation and use worker chan to spread work across set # of workers	2023-03-20 07:53:09 -05:00
Michael	8a3b6bf2ac	Merge pull request #121 from cerc-io/rebase-1.11.4-wip version update for geth 1.11.4 statediff and container to go 1.19	2023-03-17 14:05:56 -04:00
Michael Shaw	b221bde694	version update for geth 1.11.4 statediff and container to go 1.19	2023-03-17 13:51:46 -04:00
Michael	2c41537636	Merge pull request #120 from cerc-io/rebase-v1.11.2-wip update interface for failing docker build step	2023-03-15 11:18:30 -04:00
Michael Shaw	d83b088c37	update interface for failing docker build step	2023-03-15 01:38:22 -04:00
Michael	aca78f89b7	Merge pull request #118 from cerc-io/rebase-v1.11.2-wip update for interface changes in 1.11.2 geth statediffing 4.3.7	2023-03-14 15:09:38 -04:00
Michael Shaw	1788b899a4	update for interface changes in 1.11.2 geth statediffing 4.3.7	2023-03-13 18:35:54 -04:00
Michael	1fe7a04af0	Merge pull request #116 from cerc-io/v4.1.6-wip dependency updates for geth 1.10.26 and leveldb cerc migraiton	2022-11-08 11:41:19 -05:00
Michael Shaw	68ebdca6f9	dependency updates for geth 1.10.26 and leveldb cerc migraiton	2022-11-08 11:19:27 -05:00
i-norden	5f7915649d	avoid quantization and use worker channel to ensure expected number of workers	2022-10-12 18:39:28 -05:00
Ian Norden	f6df15cb38	Merge pull request #110 from cerc-io/ian_util latest block height util	2022-10-10 18:31:14 -05:00
i-norden	03517a0eb4	add short description for stats command in the readme	2022-10-10 18:24:24 -05:00
i-norden	54205d8787	conform capitalization in logWithCommand logs	2022-10-10 13:44:46 -05:00
i-norden	8d8ff99d19	cmd for checking latest header height/hash; update serve command to report the latest header height/hash	2022-10-10 13:39:06 -05:00
i-norden	3054063942	add method to reader interface for accessing latest header	2022-10-10 13:38:33 -05:00
Michael	54e181ca68	Update manual_publish.yml publish SHA as latest	2022-09-28 17:50:24 -04:00
Michael	269333bb17	Update manual_publish.yml	2022-09-28 17:47:42 -04:00
Michael	db532467cc	Update manual_publish.yml	2022-09-28 17:45:25 -04:00
Michael	ccdf9d91fc	Update manual_publish.yml	2022-09-28 17:43:19 -04:00
Michael	aea3decebf	Update manual_publish.yml	2022-09-28 17:40:14 -04:00
Michael	2db16d69da	Create manual_publish.yml	2022-09-28 17:37:08 -04:00
Michael	e3d694e63c	Merge pull request #106 from cerc-io/geth-1.10.25-wip dependency updates for geth 1.10.25 rebase	2022-09-23 14:25:02 -04:00
Michael Shaw	2c0f3456f5	dependency updates for geth 1.10.25 rebase	2022-09-23 14:17:51 -04:00
dboreham	46cd8b1834	Merge pull request #103 from cerc-io/dboreham/prerun-workers use workers in prerun mode	2022-09-22 07:37:38 -06:00
dboreham	c0cd87ba6a	Make doc consistent	2022-09-22 07:26:32 -06:00
prathamesh0	2db235f244	Add helper scripts for processing and importing statediffed data (#105 ) * Add helper scripts for processing and importing statediffed data * Add instructions to use the helper scripts * Update scripts to use arrays * Update README with bad row output example * Remove delimiter option from xargs command	2022-09-22 13:52:47 +05:30
dboreham	a827c4a36b	Merge branch 'v4' into dboreham/prerun-workers	2022-09-21 14:04:46 -06:00
dboreham	14b9c169bc	Add viper boiler plate	2022-09-21 14:00:45 -06:00
dboreham	a2772762e1	Make pprof configurable, remove todos	2022-09-21 13:33:15 -06:00
dboreham	b1440d9673	Fix off by one error	2022-09-21 13:06:41 -06:00
Michael Shaw	67d8bced4f	one more reference changed to git.vdb.to	2022-09-21 13:27:05 -04:00
Michael Shaw	12f4810ced	missed Makefile update for cerc-io	2022-09-21 11:55:17 -04:00
Michael Shaw	7a8d38c955	indentation problem in yml	2022-09-21 11:24:43 -04:00
Michael Shaw	d09cd0afe6	update tagging and push to git.vdb.to registry	2022-09-21 11:13:23 -04:00
Michael	bcca82eaa3	Cerc refactor (#104 ) * cerc refactor waiting on unpublished dependencies * updated go.mod for published dependencies in cerc-io refactoring	2022-09-20 14:00:10 -04:00
dboreham	6d103cb1f1	Add some logging	2022-09-02 10:51:13 -06:00
dboreham	99f84b6fe6	Comment	2022-09-01 23:43:38 -06:00
dboreham	5b7f5feb1b	Enable pprof	2022-09-01 23:42:55 -06:00
dboreham	7f8885f044	Add comment	2022-08-31 07:54:16 -06:00
dboreham	d235f3b84c	Initial version of parallel workers for prerun-only mode	2022-08-30 22:31:54 -06:00