broken build

Reviewed-on: cerc-io/ipld-eth-server#267
Reviewed-by: Roy Crihfield <roy@manteia.ltd>

.dockerignore

@@ -1,16 +1,18 @@
 .git
 .travis.yml
 .idea
-bin
 .gitignore
-integration_test
-LICENSE
-postgraphile
 .private_blockchain_password
-README.md
-scripts
-Supfile
-test_config
 .travis.yml
-vulcanizedb.log
+
+LICENSE
+README.md
+
+integration
+test
+scripts
 Dockerfile
+environments
+
+**/node_modules
+build

.github/workflows/on-master.yaml

@@ -1,25 +0,0 @@
-name: Docker Compose Build
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag vulcanize/ipld-eth-server docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Push
-        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
-

.github/workflows/on-pr.yaml

@@ -1,12 +0,0 @@
-name: Docker Build
-
-on: [pull_request]
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run docker build
-        run: make docker-build

.github/workflows/publish.yaml

@@ -1,25 +1,28 @@
 name: Publish Docker image
 on:
   release:
-    types: [published]
+    types: [published, edited]
 jobs:
-  push_to_registries:
-    name: Push Docker image to Docker Hub
+  build:
+    name: Build and publish image
     runs-on: ubuntu-latest
     steps:
-      - name: Get the version
-        id: vars
+      - uses: actions/checkout@v3
+      - id: vars
+        name: Output SHA and version tag
         run: |
-          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-      - name: Docker Login to Github Registry
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Pull
-        run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
-      - name: Docker Login to Docker Registry
-        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
-      - name: Tag docker image
-        run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}}
-      - name: Docker Push to Docker Hub
-        run: docker push vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}}
-
+          echo "sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
+          echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+      - name: Build and tag Docker image
+        env:
+          GIT_VDBTO_TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
+        run: |
+          docker build . \
+            --build-arg GIT_VDBTO_TOKEN \
+            -t git.vdb.to/cerc-io/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}} \
+            -t git.vdb.to/cerc-io/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.tag}}
+      - name: Push Docker tags
+        run: |
+          echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+          docker push git.vdb.to/cerc-io/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
+          docker push git.vdb.to/cerc-io/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.tag}}

.github/workflows/tests.yaml

@@ -0,0 +1,132 @@
+name: Test the stack.
+on:
+  workflow_call:
+
+  # Job headers are hidden when not top-level - run them directly for readability until fixed:
+  # https://github.com/go-gitea/gitea/issues/26736
+  pull_request:
+    branches: '*'
+  push:
+    branches:
+      - main
+      - ci-test
+
+env:
+  SO_VERSION: v1.1.0-36d4969-202407091537
+  FIXTURENET_ETH_STACKS_REF: main
+  SYSTEM_TESTS_REF: main
+
+jobs:
+  test:
+    name: Run unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: Run DB container
+        run: docker compose -f test/compose-db.yml up --wait --quiet-pull
+      - name: Build and run tests
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo
+          ginkgo -v -r --skip-package=./integration
+
+  integration-test:
+    name: Run integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+
+      - name: Build server image
+        env:
+          GIT_VDBTO_TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
+        run: docker build . -t cerc/ipld-eth-server:local --build-arg GIT_VDBTO_TOKEN
+      - name: Install jq
+        env:
+          DEBIAN_FRONTEND: noninteractive
+        run: apt-get update && apt-get install -y jq
+      # At present the stock setup-python action fails on Linux/aarch64
+      # Conditional steps below workaroud this by using deadsnakes for that case only
+      - name: "Install Python for ARM on Linux"
+        if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
+        uses: deadsnakes/action@v3.0.1
+        with:
+          python-version: 3.11
+      - name: "Install Python cases other than ARM on Linux"
+        if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+      - name: Install stack-orchestrator
+        uses: actions/checkout@v3
+        with:
+          repository: cerc-io/stack-orchestrator
+          ref: ${{ env.SO_VERSION }}
+          path: ./stack-orchestrator
+      - run: pip install ./stack-orchestrator
+      - name: Clone fixturenet stack repo
+        uses: actions/checkout@v4
+        with:
+          repository: cerc-io/fixturenet-eth-stacks
+          ref: ${{ env.FIXTURENET_ETH_STACKS_REF }}
+          path: ./fixturenet-eth-stacks
+          progress: false
+
+      - name: Run testnet stack
+        env:
+          CERC_GO_AUTH_TOKEN: ${{ secrets.CICD_REPO_TOKEN }}
+        run: ./scripts/run-test-stack.sh ./fixturenet-eth-stacks/stack-orchestrator/stacks/fixturenet-plugeth
+      - name: Run server
+        env:
+          ETH_FORWARD_ETH_CALLS: false
+        run: docker compose -f test/compose-server.yml up --wait --quiet-pull
+      - name: Run tests
+        run: |
+          sleep 30
+          go install github.com/onsi/ginkgo/v2/ginkgo
+          ginkgo -v --label-filter '!proxy' -r ./integration
+
+      - name: Clone system-tests
+        uses: actions/checkout@v4
+        with:
+          repository: cerc-io/system-tests
+          ref: ${{ env.SYSTEM_TESTS_REF }}
+          path: ./system-tests
+          token: ${{ secrets.CICD_REPO_TOKEN }}
+          progress: false
+      - name: Run system tests
+        working-directory: ./system-tests
+        # Work around dependency conflict in system-tests:
+        # web3 uses an older eth-account until (unreleased) v7
+        # web3 also requires eth-typing<5.0.0 (ContractName was removed in 5.0.0)
+        run: |
+          pip3 install pytest
+          pip3 install 'eth-typing<5.0.0'
+          pip3 install -r requirements.txt
+          # Reinstall eth-typing<5.0.0 after requirements.txt to override any newer version
+          pip3 install --force-reinstall 'eth-typing<5.0.0'
+          pip3 install --no-deps 'eth-account>=0.12.3,<0.13'
+          pip3 install 'pydantic>=2.0.0'
+          # Skips tests that require Blob indexing
+          python3 -m pytest -vv -m "not blob_db"
+
+      - name: Run testnet stack without statediff
+        env:
+          CERC_RUN_STATEDIFF: false
+          SKIP_BUILD: 1
+        run: ./scripts/run-test-stack.sh ./fixturenet-eth-stacks/stack-orchestrator/stacks/fixturenet-plugeth
+      - name: Run server with call forwarding
+        env:
+          ETH_FORWARD_ETH_CALLS: true
+        run: docker compose -f test/compose-server.yml up --wait --quiet-pull
+      - name: Run eth_call proxy tests
+        run: |
+          sleep 30
+          go install github.com/onsi/ginkgo/v2/ginkgo
+          ginkgo -v --label-filter 'proxy' -r ./integration

.gitignore

@@ -1,19 +1,2 @@
 .idea
-.vscode
-test_data_dir/
-contracts/*
 environments/*.toml
-Vagrantfile
-vagrant*.sh
-.vagrant
-test_scripts/
-ipld-eth-server
-postgraphile/build/
-postgraphile/node_modules/
-postgraphile/package-lock.json
-vulcanizedb.log
-db/migrations/20*.sql
-plugins/*.so
-postgraphile/*.toml
-postgraphile/schema.graphql
-vulcanizedb.pem

Dockerfile

@@ -1,25 +1,34 @@
-FROM golang:1.13-alpine as builder
+FROM golang:1.21-alpine AS debugger
 
-RUN apk --update --no-cache add make git g++ linux-headers
-# DEBUG
-RUN apk add busybox-extras
+# Include dlv
+RUN go install github.com/go-delve/delve/cmd/dlv@v1.22.1
+
+FROM golang:1.21-alpine AS builder
+
+RUN apk --update --no-cache add gcc musl-dev binutils-gold git
 
 # Build ipld-eth-server
-WORKDIR /go/src/github.com/vulcanize/ipld-eth-server
-ADD . .
-RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-eth-server .
+WORKDIR /go/src/github.com/cerc-io/ipld-eth-server
 
-# Copy migration tool
-WORKDIR /
-ARG GOOSE_VER="v2.6.0"
-ADD https://github.com/pressly/goose/releases/download/${GOOSE_VER}/goose-linux64 ./goose
-RUN chmod +x ./goose
+ARG GIT_VDBTO_TOKEN
+
+# Cache the modules
+ENV GO111MODULE=on
+COPY go.mod .
+COPY go.sum .
+RUN if [ -n "$GIT_VDBTO_TOKEN" ]; then git config --global url."https://$GIT_VDBTO_TOKEN:@git.vdb.to/".insteadOf "https://git.vdb.to/"; fi && \
+    go mod download && \
+    rm -f ~/.gitconfig
+
+COPY . .
+
+# Build the binary
+RUN GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-eth-server .
 
 # app container
 FROM alpine
 
 ARG USER="vdm"
-ARG CONFIG_FILE="./environments/example.toml"
 
 RUN adduser -Du 5000 $USER
 WORKDIR /app
@@ -28,14 +37,12 @@ USER $USER
 
 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/$CONFIG_FILE config.toml
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/entrypoint.sh .
-
+COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/ipld-eth-server/entrypoint.sh .
 
 # keep binaries immutable
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/ipld-eth-server ipld-eth-server
-COPY --from=builder /goose goose
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/environments environments
+COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-server/ipld-eth-server ipld-eth-server
+
+# Allow for debugging
+COPY --from=debugger  /go/bin/dlv /usr/local/bin/
 
 ENTRYPOINT ["/app/entrypoint.sh"]

Makefile

@@ -1,144 +0,0 @@
-BIN = $(GOPATH)/bin
-BASE = $(GOPATH)/src/$(PACKAGE)
-PKGS = go list ./... | grep -v "^vendor/"
-
-# Tools
-## Testing library
-GINKGO = $(BIN)/ginkgo
-$(BIN)/ginkgo:
-	go get -u github.com/onsi/ginkgo/ginkgo
-
-## Migration tool
-GOOSE = $(BIN)/goose
-$(BIN)/goose:
-	go get -u -d github.com/pressly/goose/cmd/goose
-	go build -tags='no_mysql no_sqlite' -o $(BIN)/goose github.com/pressly/goose/cmd/goose
-
-## Source linter
-LINT = $(BIN)/golint
-$(BIN)/golint:
-	go get -u golang.org/x/lint/golint
-
-## Combination linter
-METALINT = $(BIN)/gometalinter.v2
-$(BIN)/gometalinter.v2:
-	go get -u gopkg.in/alecthomas/gometalinter.v2
-	$(METALINT) --install
-
-
-.PHONY: installtools
-installtools: | $(LINT) $(GOOSE) $(GINKGO)
-	echo "Installing tools"
-
-.PHONY: metalint
-metalint: | $(METALINT)
-	$(METALINT) ./... --vendor \
-	--fast \
-	--exclude="exported (function)|(var)|(method)|(type).*should have comment or be unexported" \
-	--format="{{.Path.Abs}}:{{.Line}}:{{if .Col}}{{.Col}}{{end}}:{{.Severity}}: {{.Message}} ({{.Linter}})"
-
-.PHONY: lint
-lint:
-	$(LINT) $$($(PKGS)) | grep -v -E "exported (function)|(var)|(method)|(type).*should have comment or be unexported"
-
-#Database
-HOST_NAME = localhost
-PORT = 5432
-NAME =
-USER = postgres
-CONNECT_STRING=postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
-
-#Test
-TEST_DB = vulcanize_testing
-TEST_CONNECT_STRING = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
-
-.PHONY: test
-test: | $(GINKGO) $(LINT)
-	go vet ./...
-	go fmt ./...
-	dropdb --if-exists $(TEST_DB)
-	createdb $(TEST_DB)
-	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
-	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" reset
-	make migrate NAME=$(TEST_DB)
-	$(GINKGO) -r --skipPackage=integration_tests,integration
-
-.PHONY: integrationtest
-integrationtest: | $(GINKGO) $(LINT)
-	go vet ./...
-	go fmt ./...
-	dropdb --if-exists $(TEST_DB)
-	createdb $(TEST_DB)
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" up
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" reset
-	make migrate NAME=$(TEST_DB)
-	$(GINKGO) -r integration_test/
-
-build:
-	go fmt ./...
-	GO111MODULE=on go build
-
-# Parameter checks
-## Check that DB variables are provided
-.PHONY: checkdbvars
-checkdbvars:
-	test -n "$(HOST_NAME)" # $$HOST_NAME
-	test -n "$(PORT)" # $$PORT
-	test -n "$(NAME)" # $$NAME
-	@echo $(CONNECT_STRING)
-
-## Check that the migration variable (id/timestamp) is provided
-.PHONY: checkmigration
-checkmigration:
-	test -n "$(MIGRATION)" # $$MIGRATION
-
-# Check that the migration name is provided
-.PHONY: checkmigname
-checkmigname:
-	test -n "$(NAME)" # $$NAME
-
-# Migration operations
-## Rollback the last migration
-.PHONY: rollback
-rollback: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down
-	pg_dump -O -s $(CONNECT_STRING) > db/schema.sql
-
-
-## Rollback to a select migration (id/timestamp)
-.PHONY: rollback_to
-rollback_to: $(GOOSE) checkmigration checkdbvars
-	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" down-to "$(MIGRATION)"
-
-## Apply all migrations not already run
-.PHONY: migrate
-migrate: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" up
-	pg_dump -O -s $(CONNECT_STRING) > db/schema.sql
-
-## Create a new migration file
-.PHONY: new_migration
-new_migration: $(GOOSE) checkmigname
-	$(GOOSE) -dir db/migrations create $(NAME) sql
-
-## Check which migrations are applied at the moment
-.PHONY: migration_status
-migration_status: $(GOOSE) checkdbvars
-	$(GOOSE) -dir db/migrations postgres "$(CONNECT_STRING)" status
-
-# Convert timestamped migrations to versioned (to be run in CI);
-# merge timestamped files to prevent conflict
-.PHONY: version_migrations
-version_migrations:
-	$(GOOSE) -dir db/migrations fix
-
-# Import a psql schema to the database
-.PHONY: import
-import:
-	test -n "$(NAME)" # $$NAME
-	psql $(NAME) < db/schema.sql
-
-## Build docker image
-.PHONY: docker-build
-docker-build:
-	docker build -t vulcanize/ipld-eth-server .

README.md

@@ -14,7 +14,7 @@
 ## Background
 NOTE: WIP
 
-ipld-eth-server is used to service queries against the indexed Ethereum IPLD objects indexed by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer).
+ipld-eth-server is used to service queries against the Ethereum IPLD objects indexed by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer).
 
 It exposes standard Ethereum JSON RPC endpoints on top of the database, in some cases these endpoints can leverage the unique indexes to improve query performance.
 Additional, unique endpoints are exposed which utilize the new indexes and state diff data objects.
@@ -22,20 +22,20 @@ Additional, unique endpoints are exposed which utilize the new indexes and state
 
 ## Dependencies
 Minimal build dependencies
-* Go (1.13)
+* Go (1.19)
 * Git
 * GCC compiler
 * This repository
 
 External dependency
-* Postgres database populated by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer)
+* Postgres database populated by [ipld-eth-db](https://github.com/cerc-io/ipld-eth-db)
 
 ## Install
 Start by downloading ipld-eth-server and moving into the repo:
 
-`GO111MODULE=off go get -d github.com/vulcanize/ipld-eth-server`
+`GO111MODULE=off go get -d github.com/cerc-io/ipld-eth-server/v5`
 
-`cd $GOPATH/src/github.com/vulcanize/ipld-eth-server`
+`cd $GOPATH/src/github.com/cerc-io/ipld-eth-server/v5@v5.x.x`
 
 Then, build the binary:
 
@@ -60,16 +60,28 @@ The corresponding CLI flags can be found with the `./ipld-eth-server serve --hel
     password = "" # $DATABASE_PASSWORD
 
 [log]
-    level = "info" # $LOGRUS_LEVEL
+    level = "info" # $LOG_LEVEL
 
 [server]
     ipcPath = "~/.vulcanize/vulcanize.ipc" # $SERVER_IPC_PATH
     wsPath = "127.0.0.1:8081" # $SERVER_WS_PATH
     httpPath = "127.0.0.1:8082" # $SERVER_HTTP_PATH
+    graphql = true # $SERVER_GRAPHQL
+    graphqlPath = "" # $SERVER_GRAPHQL_PATH
+
+[ethereum]
+    chainID = "1" # $ETH_CHAIN_ID
+    rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
+    httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    nodeID = "arch1" # $ETH_NODE_ID
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
+    networkID = "1" # $ETH_NETWORK_ID
 ```
 
-The `database` fields are for connecting to a Postgres database that has been/is being populated by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer).
+The `database` fields are for connecting to a Postgres database that has been/is being populated by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer)
 The `server` fields set the paths for exposing the ipld-eth-server endpoints
+The `ethereum` fields set the chainID and default sender address to use for EVM simulation, and can optionally be used to configure a remote eth node to forward cache misses to
 
 
 ### Endpoints
@@ -79,19 +91,48 @@ TODO: Port the IPLD RPC subscription endpoints after the decoupling
 #### Ethereum JSON-RPC
 ipld-eth-server currently recapitulates portions of the Ethereum JSON-RPC api standard.
 
-The currently supported standard endpoints are:  
-`eth_blockNumber`  
-`eth_getLogs`  
-`eth_getHeaderByNumber`  
-`eth_getBlockByNumber`  
-`eth_getBlockByHash`  
-`eth_getTransactionByHash`  
+The currently supported standard endpoints are:
+- `eth_call`
+- `eth_getBalance`
+- `eth_getStorageAt`
+- `eth_getCode`
+- `eth_getProof`
+- `eth_blockNumber`
+- `eth_getHeaderByNumber`
+- `eth_getHeaderByHash`
+- `eth_getBlockByNumber`
+- `eth_getBlockByHash`
+- `eth_getTransactionCount`
+- `eth_getBlockTransactionCountByHash`
+- `eth_getBlockTransactionCountByNumber`
+- `eth_getTransactionByHash`
+- `eth_getRawTransactionByHash`
+- `eth_getTransactionByBlockHashAndIndex`
+- `eth_getTransactionByBlockNumberAndIndex`
+- `eth_getRawTransactionByBlockHashAndIndex`
+- `eth_getRawTransactionByBlockNumberAndIndex`
+- `eth_getTransactionReceipt`
+- `eth_getLogs`
+- `eth_getUncleCountByBlockHash`
+- `eth_getUncleCountByBlockNumber`
+- `eth_getUncleByBlockHashAndIndex`
+- `eth_getUncleByBlockNumberAndIndex`
 
-TODO: Add the rest of the standard endpoints add unique endpoints (e.g. getSlice)
+TODO: Add the rest of the standard endpoints and unique endpoints (e.g. getSlice)
 
 ### Testing
-`make test` will run the unit tests  
-`make test` setups a clean `vulcanize_testing` db
+
+Follow steps in [test/README.md](./test/README.md)
+
+## Monitoring
+
+* Enable http server and metrics using parameters `--http --metrics`
+* ipld-eth-server exposes prometheus metrics at `/metric` endpoint
+* start prometheus using `monitoring/prometheus.yml` config (`prometheus --config.file=monitoring/prometheus.yml`)
+* start grafana, connect to prometheus datasource and import dashboard from `monitoring/grafana/dashboard_main.json`
+
+![](monitoring/grafana.png)
+
 
 ## Contributing
 Contributions are welcome!

cmd/common.go

@@ -0,0 +1,37 @@
+// Copyright © 2021 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func addDatabaseFlags(command *cobra.Command) {
+	// database flags
+	command.PersistentFlags().String("database-name", "vulcanize_public", "database name")
+	command.PersistentFlags().Int("database-port", 5432, "database port")
+	command.PersistentFlags().String("database-hostname", "localhost", "database hostname")
+	command.PersistentFlags().String("database-user", "", "database user")
+	command.PersistentFlags().String("database-password", "", "database password")
+
+	// database flag bindings
+	viper.BindPFlag("database.name", command.PersistentFlags().Lookup("database-name"))
+	viper.BindPFlag("database.port", command.PersistentFlags().Lookup("database-port"))
+	viper.BindPFlag("database.hostname", command.PersistentFlags().Lookup("database-hostname"))
+	viper.BindPFlag("database.user", command.PersistentFlags().Lookup("database-user"))
+	viper.BindPFlag("database.password", command.PersistentFlags().Lookup("database-password"))
+}

cmd/root.go

@@ -18,16 +18,20 @@ package cmd
 
 import (
 	"fmt"
-	"os"
+	"path/filepath"
 	"strings"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/joho/godotenv"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/prom"
 )
 
 var (
 	cfgFile        string
+	envFile        string
 	subCommand     string
 	logWithCommand log.Entry
 )
@@ -45,37 +49,20 @@ func Execute() {
 }
 
 func initFuncs(cmd *cobra.Command, args []string) {
-	logfile := viper.GetString("logfile")
-	if logfile != "" {
-		file, err := os.OpenFile(logfile,
-			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
-		if err == nil {
-			log.Infof("Directing output to %s", logfile)
-			log.SetOutput(file)
-		} else {
-			log.SetOutput(os.Stdout)
-			log.Info("Failed to log to file, using default stdout")
-		}
-	} else {
-		log.SetOutput(os.Stdout)
-	}
-	if err := logLevel(); err != nil {
-		log.Fatal("Could not set log level: ", err)
-	}
-}
+	log.Init()
 
-func logLevel() error {
-	viper.BindEnv("log.level", "LOGRUS_LEVEL")
-	lvl, err := log.ParseLevel(viper.GetString("log.level"))
-	if err != nil {
-		return err
+	if viper.GetBool("metrics") {
+		prom.Init()
 	}
-	log.SetLevel(lvl)
-	if lvl > log.InfoLevel {
-		log.SetReportCaller(true)
+
+	if viper.GetBool("prom.http") {
+		addr := fmt.Sprintf(
+			"%s:%s",
+			viper.GetString("prom.http.addr"),
+			viper.GetString("prom.http.port"),
+		)
+		prom.Serve(addr)
 	}
-	log.Info("Log level set to ", lvl.String())
-	return nil
 }
 
 func init() {
@@ -84,33 +71,55 @@ func init() {
 	viper.AutomaticEnv()
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
-	rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
-	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
-	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
-	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
-	rootCmd.PersistentFlags().String("database-user", "", "database user")
-	rootCmd.PersistentFlags().String("database-password", "", "database password")
+	rootCmd.PersistentFlags().StringVar(&envFile, "env", "", "environment file location")
+
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
-	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
+	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
+	rootCmd.PersistentFlags().String("log-file", "", "file path for logging")
+
+	rootCmd.PersistentFlags().Bool("metrics", false, "enable metrics")
+
+	rootCmd.PersistentFlags().Bool("prom-http", false, "enable http service for prometheus")
+	rootCmd.PersistentFlags().String("prom-http-addr", "127.0.0.1", "http host for prometheus")
+	rootCmd.PersistentFlags().String("prom-http-port", "8090", "http port for prometheus")
 
-	viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
-	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
-	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
-	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
-	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
-	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
+	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
+
+	viper.BindPFlag("metrics", rootCmd.PersistentFlags().Lookup("metrics"))
+
+	viper.BindPFlag("prom.http", rootCmd.PersistentFlags().Lookup("prom-http"))
+	viper.BindPFlag("prom.http.addr", rootCmd.PersistentFlags().Lookup("prom-http-addr"))
+	viper.BindPFlag("prom.http.port", rootCmd.PersistentFlags().Lookup("prom-http-port"))
 }
 
 func initConfig() {
+	if cfgFile == "" && envFile == "" {
+		log.Warn("No configuration file specified, use --config , --env flag to provide configuration")
+	}
+
 	if cfgFile != "" {
-		viper.SetConfigFile(cfgFile)
-		if err := viper.ReadInConfig(); err == nil {
-			log.Printf("Using config file: %s", viper.ConfigFileUsed())
-		} else {
-			log.Fatal(fmt.Sprintf("Couldn't read config file: %s", err.Error()))
+		if filepath.Ext(cfgFile) != ".toml" {
+			log.Fatal("Provide .toml file for --config flag")
 		}
-	} else {
-		log.Warn("No config file passed with --config flag")
+
+		viper.SetConfigFile(cfgFile)
+		if err := viper.ReadInConfig(); err != nil {
+			log.Fatalf("Couldn't read config file: %s", err.Error())
+		}
+
+		log.Infof("Using config file: %s", viper.ConfigFileUsed())
+	}
+
+	if envFile != "" {
+		if filepath.Ext(envFile) != ".env" {
+			log.Fatal("Provide .env file for --env flag")
+		}
+
+		if err := godotenv.Load(envFile); err != nil {
+			log.Fatalf("Failed to set environment variable from env file: %s", err.Error())
+		}
+
+		log.Infof("Using env file: %s", envFile)
 	}
 }

cmd/serve.go

@@ -16,28 +16,34 @@
 package cmd
 
 import (
+	"errors"
+	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
+	"strings"
 	"sync"
+	"time"
 
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
 	"github.com/ethereum/go-ethereum/rpc"
-	log "github.com/sirupsen/logrus"
+	"github.com/mailgun/groupcache/v2"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-
-	s "github.com/vulcanize/ipld-eth-server/pkg/serve"
-	v "github.com/vulcanize/ipld-eth-server/version"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/graphql"
+	srpc "github.com/cerc-io/ipld-eth-server/v5/pkg/rpc"
+	s "github.com/cerc-io/ipld-eth-server/v5/pkg/serve"
+	v "github.com/cerc-io/ipld-eth-server/v5/version"
 )
 
+var ErrNoRpcEndpoints = errors.New("no rpc endpoints is available")
+
 // serveCmd represents the serve command
 var serveCmd = &cobra.Command{
 	Use:   "serve",
 	Short: "serve chain data from PG-IPFS",
-	Long: `This command configures a VulcanizeDB ipld-eth-server.
-
-`,
+	Long:  `This command configures a VulcanizeDB ipld-eth-server.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		subCommand = cmd.CalledAs()
 		logWithCommand = *log.WithField("SubCommand", subCommand)
@@ -46,76 +52,291 @@ var serveCmd = &cobra.Command{
 }
 
 func serve() {
-	logWithCommand.Infof("running ipld-eth-server version: %s", v.VersionWithMeta)
+	logWithCommand.Infof("ipld-eth-server version: %s", v.VersionWithMeta)
 
-	var forwardPayloadChan chan eth.ConvertedPayload
 	wg := new(sync.WaitGroup)
-	logWithCommand.Debug("loading server configuration variables")
 	serverConfig, err := s.NewConfig()
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
-	logWithCommand.Infof("server config: %+v", serverConfig)
-	logWithCommand.Debug("initializing new server service")
+	logWithCommand.Debugf("server config: %+v", serverConfig)
+	logWithCommand.Debugf("chain config: %+v", serverConfig.ChainConfig)
 	server, err := s.NewServer(serverConfig)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
+	if serverConfig.ForwardEthCalls {
+		logWithCommand.Info("Fowarding eth_call")
+	}
+	if serverConfig.ForwardGetStorageAt {
+		logWithCommand.Info("Fowarding eth_getStorageAt")
+	}
+	if serverConfig.ProxyOnError {
+		logWithCommand.Info("Proxy on error is enabled")
+	}
 
-	logWithCommand.Info("starting up server servers")
-	forwardPayloadChan = make(chan eth.ConvertedPayload, s.PayloadChanBufferSize)
-	server.Serve(wg, forwardPayloadChan)
+	server.Serve(wg)
 	if err := startServers(server, serverConfig); err != nil {
 		logWithCommand.Fatal(err)
 	}
+	graphQL, err := startEthGraphQL(server, serverConfig)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
 
-	shutdown := make(chan os.Signal)
+	err = startGroupCacheService(serverConfig)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	if serverConfig.StateValidationEnabled {
+		go startStateTrieValidator(serverConfig, server)
+		logWithCommand.Info("state validator enabled")
+	} else {
+		logWithCommand.Debug("state validator disabled")
+	}
+
+	shutdown := make(chan os.Signal, 1)
 	signal.Notify(shutdown, os.Interrupt)
 	<-shutdown
+	if graphQL != nil {
+		graphQL.Stop()
+	}
 	server.Stop()
 	wg.Wait()
 }
 
 func startServers(server s.Server, settings *s.Config) error {
-	logWithCommand.Info("starting up IPC server")
-	_, _, err := rpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
-	if err != nil {
-		return err
+	if settings.IPCEnabled {
+		logWithCommand.Debug("starting up IPC server")
+		_, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
+		if err != nil {
+			return err
+		}
+	} else {
+		logWithCommand.Debug("IPC server is disabled")
 	}
-	logWithCommand.Info("starting up WS server")
-	_, _, err = rpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb"}, nil, true)
-	if err != nil {
-		return err
+
+	if settings.WSEnabled {
+		logWithCommand.Debug("starting up WS server")
+		_, _, err := srpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb", "net"}, nil)
+		if err != nil {
+			return err
+		}
+	} else {
+		logWithCommand.Debug("WS server is disabled")
 	}
-	logWithCommand.Info("starting up HTTP server")
-	_, _, err = rpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"eth"}, nil, nil, rpc.HTTPTimeouts{})
-	return err
+
+	if settings.HTTPEnabled {
+		logWithCommand.Debug("starting up HTTP server")
+		_, err := srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"vdb", "eth", "debug", "net"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
+		if err != nil {
+			return err
+		}
+	} else {
+		logWithCommand.Debug("HTTP server is disabled")
+	}
+
+	return nil
+}
+
+func startEthGraphQL(server s.Server, settings *s.Config) (graphQLServer *graphql.Service, err error) {
+	if settings.EthGraphqlEnabled {
+		logWithCommand.Debug("starting up ETH GraphQL server")
+		endPoint := settings.EthGraphqlEndpoint
+		if endPoint != "" {
+			graphQLServer, err = graphql.New(server.Backend(), endPoint, nil, []string{"*"}, rpc.HTTPTimeouts{})
+			if err != nil {
+				return
+			}
+			err = graphQLServer.Start(nil)
+		}
+	} else {
+		logWithCommand.Debug("ETH GraphQL server is disabled")
+	}
+
+	return
+}
+
+func startGroupCacheService(settings *s.Config) error {
+	gcc := settings.GroupCache
+
+	if gcc.Pool.Enabled {
+		logWithCommand.Debug("starting up groupcache pool HTTTP server")
+
+		pool := groupcache.NewHTTPPoolOpts(gcc.Pool.HttpEndpoint, &groupcache.HTTPPoolOptions{})
+		pool.Set(gcc.Pool.PeerHttpEndpoints...)
+
+		httpURL, err := url.Parse(gcc.Pool.HttpEndpoint)
+		if err != nil {
+			return err
+		}
+
+		server := http.Server{
+			Addr:    httpURL.Host,
+			Handler: pool,
+		}
+
+		// Start a HTTP server to listen for peer requests from the groupcache
+		go server.ListenAndServe()
+
+		logWithCommand.Infof("groupcache pool endpoint opened at %s", httpURL)
+	} else {
+		logWithCommand.Debug("Groupcache pool is disabled")
+	}
+
+	return nil
+}
+
+func startStateTrieValidator(config *s.Config, server s.Server) {
+	validateEveryNthBlock := config.StateValidationEveryNthBlock
+
+	var lastBlockNumber uint64
+	backend := server.Backend()
+
+	for {
+		time.Sleep(5 * time.Second)
+
+		block, err := backend.CurrentBlock()
+		if err != nil {
+			log.Error("Error fetching current block for state trie validator")
+			continue
+		}
+
+		stateRoot := block.Root()
+		blockNumber := block.NumberU64()
+		blockHash := block.Hash()
+
+		if validateEveryNthBlock <= 0 || // Used for static replicas where block number doesn't progress.
+			(blockNumber > lastBlockNumber) && (blockNumber%validateEveryNthBlock == 0) {
+
+			// The validate trie call will take a long time on mainnet, e.g. a few hours.
+			if err = backend.ValidateTrie(stateRoot); err != nil {
+				log.Fatalf("Error validating trie for block number %d hash %s state root %s",
+					blockNumber,
+					blockHash,
+					stateRoot,
+				)
+			}
+
+			log.Infof("Successfully validated trie for block number %d hash %s state root %s",
+				blockNumber,
+				blockHash,
+				stateRoot,
+			)
+
+			if validateEveryNthBlock <= 0 {
+				// Static replica, sleep a long-ish time (1/2 of cache expiry time) since we only need to keep the cache warm.
+				time.Sleep((time.Minute * time.Duration(config.GroupCache.StateDB.CacheExpiryInMins)) / 2)
+			}
+
+			lastBlockNumber = blockNumber
+		}
+	}
+}
+
+func parseRpcAddresses(value string) ([]*rpc.Client, error) {
+	rpcAddresses := strings.Split(value, ",")
+	rpcClients := make([]*rpc.Client, 0, len(rpcAddresses))
+	for _, address := range rpcAddresses {
+		rpcClient, err := rpc.Dial(address)
+		if err != nil {
+			logWithCommand.Errorf("couldn't connect to %s. Error: %s", address, err)
+			continue
+		}
+
+		rpcClients = append(rpcClients, rpcClient)
+	}
+
+	if len(rpcClients) == 0 {
+		logWithCommand.Error(ErrNoRpcEndpoints)
+		return nil, ErrNoRpcEndpoints
+	}
+
+	return rpcClients, nil
 }
 
 func init() {
 	rootCmd.AddCommand(serveCmd)
 
+	addDatabaseFlags(serveCmd)
+
 	// flags for all config variables
-	serveCmd.PersistentFlags().String("server-ws-path", "", "vdb server ws path")
-	serveCmd.PersistentFlags().String("server-http-path", "", "vdb server http path")
-	serveCmd.PersistentFlags().String("server-ipc-path", "", "vdb server ipc path")
+	// eth graphql and json-rpc parameters
+	serveCmd.PersistentFlags().Bool("server-graphql", false, "turn on the eth graphql server")
+	serveCmd.PersistentFlags().String("server-graphql-path", "", "endpoint url for eth graphql server (host:port)")
+	serveCmd.PersistentFlags().Bool("server-http", true, "turn on the eth http json-rpc server")
+	serveCmd.PersistentFlags().String("server-http-path", "", "endpoint url for eth http json-rpc server (host:port)")
+	serveCmd.PersistentFlags().Bool("server-ws", false, "turn on the eth websocket json-rpc server")
+	serveCmd.PersistentFlags().String("server-ws-path", "", "endpoint url for eth websocket json-rpc server (host:port)")
+	serveCmd.PersistentFlags().Bool("server-ipc", false, "turn on the eth ipc json-rpc server")
+	serveCmd.PersistentFlags().String("server-ipc-path", "", "path for eth ipc json-rpc server")
 
-	serveCmd.PersistentFlags().String("eth-ws-path", "", "ws url for ethereum node")
 	serveCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
 	serveCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
-	serveCmd.PersistentFlags().String("eth-client-name", "", "eth client name")
-	serveCmd.PersistentFlags().String("eth-genesis-block", "", "eth genesis block hash")
-	serveCmd.PersistentFlags().String("eth-network-id", "", "eth network id")
+	serveCmd.PersistentFlags().String("eth-client-name", "Geth", "eth client name")
+	serveCmd.PersistentFlags().String("eth-genesis-block", "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", "eth genesis block hash")
+	serveCmd.PersistentFlags().String("eth-network-id", "1", "eth network id")
+	serveCmd.PersistentFlags().String("eth-chain-id", "", "eth chain id")
+	serveCmd.PersistentFlags().String("eth-chain-config", "", "json chain config file location")
+	serveCmd.PersistentFlags().String("eth-default-sender", "", "default sender address")
+	serveCmd.PersistentFlags().String("eth-rpc-gas-cap", "", "rpc gas cap (for eth_Call execution)")
+	serveCmd.PersistentFlags().Bool("eth-supports-state-diff", false, "whether the proxy ethereum client supports statediffing endpoints")
+	serveCmd.PersistentFlags().Bool("eth-forward-eth-calls", false, "whether to immediately forward eth_calls to proxy client")
+	serveCmd.PersistentFlags().Bool("eth-proxy-on-error", true, "whether to forward all failed calls to proxy client")
+
+	// groupcache flags
+	serveCmd.PersistentFlags().Bool("gcache-pool-enabled", false, "turn on the groupcache pool")
+	serveCmd.PersistentFlags().String("gcache-pool-http-path", "", "http url for groupcache node")
+	serveCmd.PersistentFlags().StringArray("gcache-pool-http-peers", []string{}, "http urls for groupcache peers")
+	serveCmd.PersistentFlags().Int("gcache-statedb-cache-size", 16, "state DB cache size in MB")
+	serveCmd.PersistentFlags().Int("gcache-statedb-cache-expiry", 60, "state DB cache expiry time in mins")
+	serveCmd.PersistentFlags().Int("gcache-statedb-log-stats-interval", 60, "state DB cache stats log interval in secs")
+
+	// state validator flags
+	serveCmd.PersistentFlags().Bool("validator-enabled", false, "turn on the state validator")
+	serveCmd.PersistentFlags().Uint("validator-every-nth-block", 1500, "only validate every Nth block")
 
 	// and their bindings
-	viper.BindPFlag("server.wsPath", serveCmd.PersistentFlags().Lookup("server-ws-path"))
+	// eth graphql server
+	viper.BindPFlag("server.graphql", serveCmd.PersistentFlags().Lookup("server-graphql"))
+	viper.BindPFlag("server.graphqlPath", serveCmd.PersistentFlags().Lookup("server-graphql-path"))
+
+	// eth http json-rpc server
+	viper.BindPFlag("server.http", serveCmd.PersistentFlags().Lookup("server-http"))
 	viper.BindPFlag("server.httpPath", serveCmd.PersistentFlags().Lookup("server-http-path"))
+
+	// eth websocket json-rpc server
+	viper.BindPFlag("server.ws", serveCmd.PersistentFlags().Lookup("server-ws"))
+	viper.BindPFlag("server.wsPath", serveCmd.PersistentFlags().Lookup("server-ws-path"))
+
+	// eth ipc json-rpc server
+	viper.BindPFlag("server.ipc", serveCmd.PersistentFlags().Lookup("server-ipc"))
 	viper.BindPFlag("server.ipcPath", serveCmd.PersistentFlags().Lookup("server-ipc-path"))
 
-	viper.BindPFlag("ethereum.wsPath", serveCmd.PersistentFlags().Lookup("eth-ws-path"))
 	viper.BindPFlag("ethereum.httpPath", serveCmd.PersistentFlags().Lookup("eth-http-path"))
 	viper.BindPFlag("ethereum.nodeID", serveCmd.PersistentFlags().Lookup("eth-node-id"))
 	viper.BindPFlag("ethereum.clientName", serveCmd.PersistentFlags().Lookup("eth-client-name"))
 	viper.BindPFlag("ethereum.genesisBlock", serveCmd.PersistentFlags().Lookup("eth-genesis-block"))
 	viper.BindPFlag("ethereum.networkID", serveCmd.PersistentFlags().Lookup("eth-network-id"))
+	viper.BindPFlag("ethereum.chainID", serveCmd.PersistentFlags().Lookup("eth-chain-id"))
+	viper.BindPFlag("ethereum.rpcGasCap", serveCmd.PersistentFlags().Lookup("eth-rpc-gas-cap"))
+	viper.BindPFlag("ethereum.chainConfig", serveCmd.PersistentFlags().Lookup("eth-chain-config"))
+	viper.BindPFlag("ethereum.supportsStateDiff", serveCmd.PersistentFlags().Lookup("eth-supports-state-diff"))
+	viper.BindPFlag("ethereum.forwardEthCalls", serveCmd.PersistentFlags().Lookup("eth-forward-eth-calls"))
+	viper.BindPFlag("ethereum.forwardGetStorageAt", serveCmd.PersistentFlags().Lookup("eth-forward-get-storage-at"))
+	viper.BindPFlag("ethereum.proxyOnError", serveCmd.PersistentFlags().Lookup("eth-proxy-on-error"))
+	viper.BindPFlag("ethereum.getLogsBlockLimit", serveCmd.PersistentFlags().Lookup("eth-getlogs-block-limit"))
+
+	// groupcache flags
+	viper.BindPFlag("groupcache.pool.enabled", serveCmd.PersistentFlags().Lookup("gcache-pool-enabled"))
+	viper.BindPFlag("groupcache.pool.httpEndpoint", serveCmd.PersistentFlags().Lookup("gcache-pool-http-path"))
+	viper.BindPFlag("groupcache.pool.peerHttpEndpoints", serveCmd.PersistentFlags().Lookup("gcache-pool-http-peers"))
+	viper.BindPFlag("groupcache.statedb.cacheSizeInMB", serveCmd.PersistentFlags().Lookup("gcache-statedb-cache-size"))
+	viper.BindPFlag("groupcache.statedb.cacheExpiryInMins", serveCmd.PersistentFlags().Lookup("gcache-statedb-cache-expiry"))
+	viper.BindPFlag("groupcache.statedb.logStatsIntervalInSecs", serveCmd.PersistentFlags().Lookup("gcache-statedb-log-stats-interval"))
+
+	// state validator flags
+	viper.BindPFlag("validator.enabled", serveCmd.PersistentFlags().Lookup("validator-enabled"))
+	viper.BindPFlag("validator.everyNthBlock", serveCmd.PersistentFlags().Lookup("validator-every-nth-block"))
 }

cmd/subscribe.go

@@ -1,175 +0,0 @@
-// Copyright © 2019 Vulcanize, Inc
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bytes"
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/rpc"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/client"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	w "github.com/vulcanize/ipld-eth-server/pkg/serve"
-)
-
-// subscribeCmd represents the subscribe command
-var subscribeCmd = &cobra.Command{
-	Use:   "subscribe",
-	Short: "This command is used to subscribe to the eth ipfs watcher data stream with the provided filters",
-	Long: `This command is for demo and testing purposes and is used to subscribe to the watcher with the provided subscription configuration parameters.
-It does not do anything with the data streamed from the watcher other than unpack it and print it out for demonstration purposes.`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		subscribe()
-	},
-}
-
-func init() {
-	rootCmd.AddCommand(subscribeCmd)
-}
-
-func subscribe() {
-	// Prep the subscription config/filters to be sent to the server
-	ethSubConfig, err := eth.NewEthSubscriptionConfig()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	// Create a new rpc client and a subscription streamer with that client
-	rpcClient, err := getRPCClient()
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	subClient := client.NewClient(rpcClient)
-
-	// Buffered channel for reading subscription payloads
-	payloadChan := make(chan w.SubscriptionPayload, 20000)
-
-	// Subscribe to the watcher service with the given config/filter parameters
-	sub, err := subClient.Stream(payloadChan, *ethSubConfig)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	logWithCommand.Info("awaiting payloads")
-	// Receive response payloads and print out the results
-	for {
-		select {
-		case payload := <-payloadChan:
-			if payload.Err != "" {
-				logWithCommand.Error(payload.Err)
-				continue
-			}
-			var ethData eth2.IPLDs
-			if err := rlp.DecodeBytes(payload.Data, &ethData); err != nil {
-				logWithCommand.Error(err)
-				continue
-			}
-			var header types.Header
-			err = rlp.Decode(bytes.NewBuffer(ethData.Header.Data), &header)
-			if err != nil {
-				logWithCommand.Error(err)
-				continue
-			}
-			fmt.Printf("Header number %d, hash %s\n", header.Number.Int64(), header.Hash().Hex())
-			fmt.Printf("header: %v\n", header)
-			for _, trxRlp := range ethData.Transactions {
-				var trx types.Transaction
-				buff := bytes.NewBuffer(trxRlp.Data)
-				stream := rlp.NewStream(buff, 0)
-				err := trx.DecodeRLP(stream)
-				if err != nil {
-					logWithCommand.Error(err)
-					continue
-				}
-				fmt.Printf("Transaction with hash %s\n", trx.Hash().Hex())
-				fmt.Printf("trx: %v\n", trx)
-			}
-			for _, rctRlp := range ethData.Receipts {
-				var rct types.Receipt
-				buff := bytes.NewBuffer(rctRlp.Data)
-				stream := rlp.NewStream(buff, 0)
-				err = rct.DecodeRLP(stream)
-				if err != nil {
-					logWithCommand.Error(err)
-					continue
-				}
-				fmt.Printf("Receipt with block hash %s, trx hash %s\n", rct.BlockHash.Hex(), rct.TxHash.Hex())
-				fmt.Printf("rct: %v\n", rct)
-				for _, l := range rct.Logs {
-					if len(l.Topics) < 1 {
-						logWithCommand.Error(fmt.Sprintf("log only has %d topics", len(l.Topics)))
-						continue
-					}
-					fmt.Printf("Log for block hash %s, trx hash %s, address %s, and with topic0 %s\n",
-						l.BlockHash.Hex(), l.TxHash.Hex(), l.Address.Hex(), l.Topics[0].Hex())
-					fmt.Printf("log: %v\n", l)
-				}
-			}
-			// This assumes leafs only
-			for _, stateNode := range ethData.StateNodes {
-				var acct state.Account
-				err = rlp.DecodeBytes(stateNode.IPLD.Data, &acct)
-				if err != nil {
-					logWithCommand.Error(err)
-					continue
-				}
-				fmt.Printf("Account for key %s, and root %s, with balance %s\n",
-					stateNode.StateLeafKey.Hex(), acct.Root.Hex(), acct.Balance.String())
-				fmt.Printf("state account: %+v\n", acct)
-			}
-			for _, storageNode := range ethData.StorageNodes {
-				fmt.Printf("Storage for state key %s ", storageNode.StateLeafKey.Hex())
-				fmt.Printf("with storage key %s\n", storageNode.StorageLeafKey.Hex())
-				var i []interface{}
-				err := rlp.DecodeBytes(storageNode.IPLD.Data, &i)
-				if err != nil {
-					logWithCommand.Error(err)
-					continue
-				}
-				// if a value node
-				if len(i) == 1 {
-					valueBytes, ok := i[0].([]byte)
-					if !ok {
-						continue
-					}
-					fmt.Printf("Storage leaf key: %s, and value hash: %s\n",
-						storageNode.StorageLeafKey.Hex(), common.BytesToHash(valueBytes).Hex())
-				}
-			}
-		case err = <-sub.Err():
-			logWithCommand.Fatal(err)
-		}
-	}
-}
-
-func getRPCClient() (*rpc.Client, error) {
-	vulcPath := viper.GetString("watcher.ethSubscription.wsPath")
-	if vulcPath == "" {
-		vulcPath = "ws://127.0.0.1:8080" // default to and try the default ws url if no path is provided
-	}
-	return rpc.Dial(vulcPath)
-}

cmd/version.go

@@ -16,19 +16,17 @@
 package cmd
 
 import (
-	log "github.com/sirupsen/logrus"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
 	"github.com/spf13/cobra"
 
-	v "github.com/vulcanize/ipld-eth-server/version"
+	v "github.com/cerc-io/ipld-eth-server/v5/version"
 )
 
 // versionCmd represents the version command
 var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Prints the version of ipld-eth-server",
-	Long: `Use this command to fetch the version of ipld-eth-server
-
-Usage: ./ipld-eth-server version`,
+	Long:  `Use this command to fetch the version of ipld-eth-server`,
 	Run: func(cmd *cobra.Command, args []string) {
 		subCommand = cmd.CalledAs()
 		logWithCommand = *log.WithField("SubCommand", subCommand)

db/migrations/00001_create_ipfs_blocks_table.sql

@@ -1,8 +0,0 @@
--- +goose Up
-CREATE TABLE IF NOT EXISTS public.blocks (
-  key TEXT UNIQUE NOT NULL,
-  data BYTEA NOT NULL
-);
-
--- +goose Down
-DROP TABLE public.blocks;

db/migrations/00002_create_nodes_table.sql

@@ -1,12 +0,0 @@
--- +goose Up
-CREATE TABLE nodes (
-  id            SERIAL PRIMARY KEY,
-  client_name   VARCHAR,
-  genesis_block VARCHAR(66),
-  network_id    VARCHAR,
-  node_id       VARCHAR(128),
-  CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id)
-);
-
--- +goose Down
-DROP TABLE nodes;

db/migrations/00003_create_eth_schema.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE SCHEMA eth;
-
--- +goose Down
-DROP SCHEMA eth;

db/migrations/00004_create_eth_header_cids_table.sql

@@ -1,23 +0,0 @@
--- +goose Up
-CREATE TABLE eth.header_cids (
-  id                    SERIAL PRIMARY KEY,
-  block_number          BIGINT NOT NULL,
-  block_hash            VARCHAR(66) NOT NULL,
-  parent_hash           VARCHAR(66) NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  td                    NUMERIC NOT NULL,
-  node_id               INTEGER NOT NULL REFERENCES nodes (id) ON DELETE CASCADE,
-  reward                NUMERIC NOT NULL,
-  state_root            VARCHAR(66) NOT NULL,
-  tx_root               VARCHAR(66) NOT NULL,
-  receipt_root          VARCHAR(66) NOT NULL,
-  uncle_root            VARCHAR(66) NOT NULL,
-  bloom                 BYTEA NOT NULL,
-  timestamp             NUMERIC NOT NULL,
-  times_validated       INTEGER NOT NULL DEFAULT 1,
-  UNIQUE (block_number, block_hash)
-);
-
--- +goose Down
-DROP TABLE eth.header_cids;

db/migrations/00005_create_eth_uncle_cids_table.sql

@@ -1,14 +0,0 @@
--- +goose Up
-CREATE TABLE eth.uncle_cids (
-  id                    SERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  block_hash            VARCHAR(66) NOT NULL,
-  parent_hash           VARCHAR(66) NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  reward                NUMERIC NOT NULL,
-  UNIQUE (header_id, block_hash)
-);
-
--- +goose Down
-DROP TABLE eth.uncle_cids;

db/migrations/00006_create_eth_transaction_cids_table.sql

@@ -1,17 +0,0 @@
--- +goose Up
-CREATE TABLE eth.transaction_cids (
-  id                    SERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  tx_hash               VARCHAR(66) NOT NULL,
-  index                 INTEGER NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  dst                   VARCHAR(66) NOT NULL,
-  src                   VARCHAR(66) NOT NULL,
-  deployment            BOOL NOT NULL,
-  tx_data               BYTEA,
-  UNIQUE (header_id, tx_hash)
-);
-
--- +goose Down
-DROP TABLE eth.transaction_cids;

db/migrations/00007_create_eth_receipt_cids_table.sql

@@ -1,18 +0,0 @@
--- +goose Up
-CREATE TABLE eth.receipt_cids (
-  id                    SERIAL PRIMARY KEY,
-  tx_id                 INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  contract              VARCHAR(66),
-  contract_hash         VARCHAR(66),
-  topic0s               VARCHAR(66)[],
-  topic1s               VARCHAR(66)[],
-  topic2s               VARCHAR(66)[],
-  topic3s               VARCHAR(66)[],
-  log_contracts         VARCHAR(66)[],
-  UNIQUE (tx_id)
-);
-
--- +goose Down
-DROP TABLE eth.receipt_cids;

db/migrations/00008_create_eth_state_cids_table.sql

@@ -1,15 +0,0 @@
--- +goose Up
-CREATE TABLE eth.state_cids (
-  id                    SERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  state_leaf_key        VARCHAR(66),
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  state_path            BYTEA,
-  node_type             INTEGER NOT NULL,
-  diff                  BOOLEAN NOT NULL DEFAULT FALSE,
-  UNIQUE (header_id, state_path)
-);
-
--- +goose Down
-DROP TABLE eth.state_cids;

db/migrations/00009_create_eth_storage_cids_table.sql

@@ -1,15 +0,0 @@
--- +goose Up
-CREATE TABLE eth.storage_cids (
-  id                    SERIAL PRIMARY KEY,
-  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  storage_leaf_key      VARCHAR(66),
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  storage_path          BYTEA,
-  node_type             INTEGER NOT NULL,
-  diff                  BOOLEAN NOT NULL DEFAULT FALSE,
-  UNIQUE (state_id, storage_path)
-);
-
--- +goose Down
-DROP TABLE eth.storage_cids;

db/migrations/00010_create_eth_state_accouts_table.sql

@@ -1,13 +0,0 @@
--- +goose Up
-CREATE TABLE eth.state_accounts (
-  id                    SERIAL PRIMARY KEY,
-  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  balance               NUMERIC NOT NULL,
-  nonce                 INTEGER NOT NULL,
-  code_hash             BYTEA NOT NULL,
-  storage_root          VARCHAR(66) NOT NULL,
-  UNIQUE (state_id)
-);
-
--- +goose Down
-DROP TABLE eth.state_accounts;

db/migrations/00011_create_postgraphile_comments.sql

@@ -1,6 +0,0 @@
--- +goose Up
-COMMENT ON TABLE public.nodes IS E'@name NodeInfo';
-COMMENT ON TABLE eth.transaction_cids IS E'@name EthTransactionCids';
-COMMENT ON TABLE eth.header_cids IS E'@name EthHeaderCids';
-COMMENT ON COLUMN public.nodes.node_id IS E'@name ChainNodeID';
-COMMENT ON COLUMN eth.header_cids.node_id IS E'@name EthNodeID';

db/schema.sql

@@ -1,716 +0,0 @@
---
--- PostgreSQL database dump
---
-
--- Dumped from database version 12.1
--- Dumped by pg_dump version 12.1
-
-SET statement_timeout = 0;
-SET lock_timeout = 0;
-SET idle_in_transaction_session_timeout = 0;
-SET client_encoding = 'UTF8';
-SET standard_conforming_strings = on;
-SELECT pg_catalog.set_config('search_path', '', false);
-SET check_function_bodies = false;
-SET xmloption = content;
-SET client_min_messages = warning;
-SET row_security = off;
-
---
--- Name: eth; Type: SCHEMA; Schema: -; Owner: -
---
-
-CREATE SCHEMA eth;
-
-
-SET default_tablespace = '';
-
-SET default_table_access_method = heap;
-
---
--- Name: header_cids; Type: TABLE; Schema: eth; Owner: -
---
-
-CREATE TABLE eth.header_cids (
-    id integer NOT NULL,
-    block_number bigint NOT NULL,
-    block_hash character varying(66) NOT NULL,
-    parent_hash character varying(66) NOT NULL,
-    cid text NOT NULL,
-    mh_key text NOT NULL,
-    td numeric NOT NULL,
-    node_id integer NOT NULL,
-    reward numeric NOT NULL,
-    state_root character varying(66) NOT NULL,
-    tx_root character varying(66) NOT NULL,
-    receipt_root character varying(66) NOT NULL,
-    uncle_root character varying(66) NOT NULL,
-    bloom bytea NOT NULL,
-    "timestamp" numeric NOT NULL,
-    times_validated integer DEFAULT 1 NOT NULL
-);
-
-
---
--- Name: TABLE header_cids; Type: COMMENT; Schema: eth; Owner: -
---
-
-COMMENT ON TABLE eth.header_cids IS '@name EthHeaderCids';
-
-
---
--- Name: COLUMN header_cids.node_id; Type: COMMENT; Schema: eth; Owner: -
---
-
-COMMENT ON COLUMN eth.header_cids.node_id IS '@name EthNodeID';
-
-
---
--- Name: header_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
---
-
-CREATE SEQUENCE eth.header_cids_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: header_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: -
---
-
-ALTER SEQUENCE eth.header_cids_id_seq OWNED BY eth.header_cids.id;
-
-
---
--- Name: receipt_cids; Type: TABLE; Schema: eth; Owner: -
---
-
-CREATE TABLE eth.receipt_cids (
-    id integer NOT NULL,
-    tx_id integer NOT NULL,
-    cid text NOT NULL,
-    mh_key text NOT NULL,
-    contract character varying(66),
-    contract_hash character varying(66),
-    topic0s character varying(66)[],
-    topic1s character varying(66)[],
-    topic2s character varying(66)[],
-    topic3s character varying(66)[],
-    log_contracts character varying(66)[]
-);
-
-
---
--- Name: receipt_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
---
-
-CREATE SEQUENCE eth.receipt_cids_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: receipt_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: -
---
-
-ALTER SEQUENCE eth.receipt_cids_id_seq OWNED BY eth.receipt_cids.id;
-
-
---
--- Name: state_accounts; Type: TABLE; Schema: eth; Owner: -
---
-
-CREATE TABLE eth.state_accounts (
-    id integer NOT NULL,
-    state_id integer NOT NULL,
-    balance numeric NOT NULL,
-    nonce integer NOT NULL,
-    code_hash bytea NOT NULL,
-    storage_root character varying(66) NOT NULL
-);
-
-
---
--- Name: state_accounts_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
---
-
-CREATE SEQUENCE eth.state_accounts_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: state_accounts_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: -
---
-
-ALTER SEQUENCE eth.state_accounts_id_seq OWNED BY eth.state_accounts.id;
-
-
---
--- Name: state_cids; Type: TABLE; Schema: eth; Owner: -
---
-
-CREATE TABLE eth.state_cids (
-    id integer NOT NULL,
-    header_id integer NOT NULL,
-    state_leaf_key character varying(66),
-    cid text NOT NULL,
-    mh_key text NOT NULL,
-    state_path bytea,
-    node_type integer NOT NULL,
-    diff boolean DEFAULT false NOT NULL
-);
-
-
---
--- Name: state_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
---
-
-CREATE SEQUENCE eth.state_cids_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: state_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: -
---
-
-ALTER SEQUENCE eth.state_cids_id_seq OWNED BY eth.state_cids.id;
-
-
---
--- Name: storage_cids; Type: TABLE; Schema: eth; Owner: -
---
-
-CREATE TABLE eth.storage_cids (
-    id integer NOT NULL,
-    state_id integer NOT NULL,
-    storage_leaf_key character varying(66),
-    cid text NOT NULL,
-    mh_key text NOT NULL,
-    storage_path bytea,
-    node_type integer NOT NULL,
-    diff boolean DEFAULT false NOT NULL
-);
-
-
---
--- Name: storage_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
---
-
-CREATE SEQUENCE eth.storage_cids_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: storage_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: -
---
-
-ALTER SEQUENCE eth.storage_cids_id_seq OWNED BY eth.storage_cids.id;
-
-
---
--- Name: transaction_cids; Type: TABLE; Schema: eth; Owner: -
---
-
-CREATE TABLE eth.transaction_cids (
-    id integer NOT NULL,
-    header_id integer NOT NULL,
-    tx_hash character varying(66) NOT NULL,
-    index integer NOT NULL,
-    cid text NOT NULL,
-    mh_key text NOT NULL,
-    dst character varying(66) NOT NULL,
-    src character varying(66) NOT NULL,
-    deployment boolean NOT NULL,
-    tx_data bytea
-);
-
-
---
--- Name: TABLE transaction_cids; Type: COMMENT; Schema: eth; Owner: -
---
-
-COMMENT ON TABLE eth.transaction_cids IS '@name EthTransactionCids';
-
-
---
--- Name: transaction_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
---
-
-CREATE SEQUENCE eth.transaction_cids_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: transaction_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: -
---
-
-ALTER SEQUENCE eth.transaction_cids_id_seq OWNED BY eth.transaction_cids.id;
-
-
---
--- Name: uncle_cids; Type: TABLE; Schema: eth; Owner: -
---
-
-CREATE TABLE eth.uncle_cids (
-    id integer NOT NULL,
-    header_id integer NOT NULL,
-    block_hash character varying(66) NOT NULL,
-    parent_hash character varying(66) NOT NULL,
-    cid text NOT NULL,
-    mh_key text NOT NULL,
-    reward numeric NOT NULL
-);
-
-
---
--- Name: uncle_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
---
-
-CREATE SEQUENCE eth.uncle_cids_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: uncle_cids_id_seq; Type: SEQUENCE OWNED BY; Schema: eth; Owner: -
---
-
-ALTER SEQUENCE eth.uncle_cids_id_seq OWNED BY eth.uncle_cids.id;
-
-
---
--- Name: blocks; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.blocks (
-    key text NOT NULL,
-    data bytea NOT NULL
-);
-
-
---
--- Name: goose_db_version; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.goose_db_version (
-    id integer NOT NULL,
-    version_id bigint NOT NULL,
-    is_applied boolean NOT NULL,
-    tstamp timestamp without time zone DEFAULT now()
-);
-
-
---
--- Name: goose_db_version_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.goose_db_version_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: goose_db_version_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.goose_db_version_id_seq OWNED BY public.goose_db_version.id;
-
-
---
--- Name: nodes; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.nodes (
-    id integer NOT NULL,
-    client_name character varying,
-    genesis_block character varying(66),
-    network_id character varying,
-    node_id character varying(128)
-);
-
-
---
--- Name: TABLE nodes; Type: COMMENT; Schema: public; Owner: -
---
-
-COMMENT ON TABLE public.nodes IS '@name NodeInfo';
-
-
---
--- Name: COLUMN nodes.node_id; Type: COMMENT; Schema: public; Owner: -
---
-
-COMMENT ON COLUMN public.nodes.node_id IS '@name ChainNodeID';
-
-
---
--- Name: nodes_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.nodes_id_seq
-    AS integer
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: nodes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.nodes_id_seq OWNED BY public.nodes.id;
-
-
---
--- Name: header_cids id; Type: DEFAULT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.header_cids ALTER COLUMN id SET DEFAULT nextval('eth.header_cids_id_seq'::regclass);
-
-
---
--- Name: receipt_cids id; Type: DEFAULT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.receipt_cids ALTER COLUMN id SET DEFAULT nextval('eth.receipt_cids_id_seq'::regclass);
-
-
---
--- Name: state_accounts id; Type: DEFAULT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.state_accounts ALTER COLUMN id SET DEFAULT nextval('eth.state_accounts_id_seq'::regclass);
-
-
---
--- Name: state_cids id; Type: DEFAULT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.state_cids ALTER COLUMN id SET DEFAULT nextval('eth.state_cids_id_seq'::regclass);
-
-
---
--- Name: storage_cids id; Type: DEFAULT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.storage_cids ALTER COLUMN id SET DEFAULT nextval('eth.storage_cids_id_seq'::regclass);
-
-
---
--- Name: transaction_cids id; Type: DEFAULT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.transaction_cids ALTER COLUMN id SET DEFAULT nextval('eth.transaction_cids_id_seq'::regclass);
-
-
---
--- Name: uncle_cids id; Type: DEFAULT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.uncle_cids ALTER COLUMN id SET DEFAULT nextval('eth.uncle_cids_id_seq'::regclass);
-
-
---
--- Name: goose_db_version id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.goose_db_version ALTER COLUMN id SET DEFAULT nextval('public.goose_db_version_id_seq'::regclass);
-
-
---
--- Name: nodes id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass);
-
-
---
--- Name: header_cids header_cids_block_number_block_hash_key; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.header_cids
-    ADD CONSTRAINT header_cids_block_number_block_hash_key UNIQUE (block_number, block_hash);
-
-
---
--- Name: header_cids header_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.header_cids
-    ADD CONSTRAINT header_cids_pkey PRIMARY KEY (id);
-
-
---
--- Name: receipt_cids receipt_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.receipt_cids
-    ADD CONSTRAINT receipt_cids_pkey PRIMARY KEY (id);
-
-
---
--- Name: receipt_cids receipt_cids_tx_id_key; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.receipt_cids
-    ADD CONSTRAINT receipt_cids_tx_id_key UNIQUE (tx_id);
-
-
---
--- Name: state_accounts state_accounts_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.state_accounts
-    ADD CONSTRAINT state_accounts_pkey PRIMARY KEY (id);
-
-
---
--- Name: state_accounts state_accounts_state_id_key; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.state_accounts
-    ADD CONSTRAINT state_accounts_state_id_key UNIQUE (state_id);
-
-
---
--- Name: state_cids state_cids_header_id_state_path_key; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.state_cids
-    ADD CONSTRAINT state_cids_header_id_state_path_key UNIQUE (header_id, state_path);
-
-
---
--- Name: state_cids state_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.state_cids
-    ADD CONSTRAINT state_cids_pkey PRIMARY KEY (id);
-
-
---
--- Name: storage_cids storage_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.storage_cids
-    ADD CONSTRAINT storage_cids_pkey PRIMARY KEY (id);
-
-
---
--- Name: storage_cids storage_cids_state_id_storage_path_key; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.storage_cids
-    ADD CONSTRAINT storage_cids_state_id_storage_path_key UNIQUE (state_id, storage_path);
-
-
---
--- Name: transaction_cids transaction_cids_header_id_tx_hash_key; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.transaction_cids
-    ADD CONSTRAINT transaction_cids_header_id_tx_hash_key UNIQUE (header_id, tx_hash);
-
-
---
--- Name: transaction_cids transaction_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.transaction_cids
-    ADD CONSTRAINT transaction_cids_pkey PRIMARY KEY (id);
-
-
---
--- Name: uncle_cids uncle_cids_header_id_block_hash_key; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.uncle_cids
-    ADD CONSTRAINT uncle_cids_header_id_block_hash_key UNIQUE (header_id, block_hash);
-
-
---
--- Name: uncle_cids uncle_cids_pkey; Type: CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.uncle_cids
-    ADD CONSTRAINT uncle_cids_pkey PRIMARY KEY (id);
-
-
---
--- Name: blocks blocks_key_key; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.blocks
-    ADD CONSTRAINT blocks_key_key UNIQUE (key);
-
-
---
--- Name: goose_db_version goose_db_version_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.goose_db_version
-    ADD CONSTRAINT goose_db_version_pkey PRIMARY KEY (id);
-
-
---
--- Name: nodes node_uc; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.nodes
-    ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id);
-
-
---
--- Name: nodes nodes_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.nodes
-    ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
-
-
---
--- Name: header_cids header_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.header_cids
-    ADD CONSTRAINT header_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- Name: header_cids header_cids_node_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.header_cids
-    ADD CONSTRAINT header_cids_node_id_fkey FOREIGN KEY (node_id) REFERENCES public.nodes(id) ON DELETE CASCADE;
-
-
---
--- Name: receipt_cids receipt_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.receipt_cids
-    ADD CONSTRAINT receipt_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- Name: receipt_cids receipt_cids_tx_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.receipt_cids
-    ADD CONSTRAINT receipt_cids_tx_id_fkey FOREIGN KEY (tx_id) REFERENCES eth.transaction_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- Name: state_accounts state_accounts_state_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.state_accounts
-    ADD CONSTRAINT state_accounts_state_id_fkey FOREIGN KEY (state_id) REFERENCES eth.state_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- Name: state_cids state_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.state_cids
-    ADD CONSTRAINT state_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES eth.header_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- Name: state_cids state_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.state_cids
-    ADD CONSTRAINT state_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- Name: storage_cids storage_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.storage_cids
-    ADD CONSTRAINT storage_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- Name: storage_cids storage_cids_state_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.storage_cids
-    ADD CONSTRAINT storage_cids_state_id_fkey FOREIGN KEY (state_id) REFERENCES eth.state_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- Name: transaction_cids transaction_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.transaction_cids
-    ADD CONSTRAINT transaction_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES eth.header_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- Name: transaction_cids transaction_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.transaction_cids
-    ADD CONSTRAINT transaction_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- Name: uncle_cids uncle_cids_header_id_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.uncle_cids
-    ADD CONSTRAINT uncle_cids_header_id_fkey FOREIGN KEY (header_id) REFERENCES eth.header_cids(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- Name: uncle_cids uncle_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
---
-
-ALTER TABLE ONLY eth.uncle_cids
-    ADD CONSTRAINT uncle_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
-
-
---
--- PostgreSQL database dump complete
---
-

docker-compose.yml

@@ -1,78 +0,0 @@
-version: '3.2'
-
-services:
-  dapptools:
-    restart: unless-stopped
-    image: vulcanize/dapptools:v0.29.0-statediff-0.0.2
-    ports:
-      - "127.0.0.1:8545:8545"
-      - "127.0.0.1:8546:8546"
-
-  db:
-    restart: always
-    image: postgres:10.12-alpine
-    environment:
-      POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_public"
-      POSTGRES_PASSWORD: "password"
-    volumes:
-     - vdb_db_eth_server:/var/lib/postgresql/data
-    ports:
-     - "127.0.0.1:8077:5432"
-
-  eth-indexer:
-    restart: unless-stopped
-    depends_on:
-      - db
-      - dapptools
-    image: vulcanize/ipld-eth-indexer:v0.3.0-alpha
-    environment:
-      DATABASE_NAME: vulcanize_public
-      DATABASE_HOSTNAME: db
-      DATABASE_PORT: 5432
-      DATABASE_USER: vdbm
-      DATABASE_PASSWORD: password
-      ETH_WS_PATH: "dapptools:8546"
-      ETH_HTTP_PATH: "dapptools:8545"
-      ETH_CHAIN_ID: 4
-      ETH_NETWORK_ID: 4
-      VDB_COMMAND: sync
-
-  eth-server:
-    depends_on:
-      - db
-    build:
-      context: ./
-      cache_from:
-       - alpine:latest
-       - golang:1.13-alpine
-    environment:
-      VDB_COMMAND: "serve"
-      DATABASE_NAME: "vulcanize_public"
-      DATABASE_HOSTNAME: "db"
-      DATABASE_PORT: 5432
-      DATABASE_USER: "vdbm"
-      DATABASE_PASSWORD: "password"
-      SERVER_WS_PATH: "0.0.0.0:8081"
-      SERVER_HTTP_PATH: "0.0.0.0:8082"
-    ports:
-     - "127.0.0.1:8080:8080"
-     - "127.0.0.1:8081:8081"
-
-  graphql:
-    restart: unless-stopped
-    depends_on:
-      - db
-    image: vulcanize/postgraphile:v1.0.1
-    environment:
-      - PG_HOST=db
-      - PG_PORT=5432
-      - PG_DATABASE=vulcanize_public
-      - PG_USER=vdbm
-      - PG_PASSWORD=password
-      - SCHEMA=public,eth
-    ports:
-      - "127.0.0.1:5000:5000"
-
-volumes:
-  vdb_db_eth_server:

documentation/apis.md

@@ -12,7 +12,7 @@ We can expose a number of different APIs for remote access to ipld-eth-server da
 ipld-eth-server stores all processed data in Postgres using PG-IPFS, this includes all of the IPLD objects.
 [Postgraphile](https://www.graphile.org/postgraphile/) can be used to expose GraphQL endpoints for the Postgres tables.
 
-e.g. 
+e.g.
 
 `postgraphile --plugins @graphile/pg-pubsub --subscriptions --simple-subscriptions -c postgres://localhost:5432/vulcanize_public?sslmode=disable -s public,btc,eth -a -j`
 
@@ -33,16 +33,16 @@ by ipld-eth-server to filter and return a requested subset of chain data to the
 An example of how to subscribe to a real-time Ethereum data feed from ipld-eth-server using the `Stream` RPC method is provided below
 
 ```go
-    package main 
+    package main
 
     import (
     	"github.com/ethereum/go-ethereum/rlp"
     	"github.com/ethereum/go-ethereum/rpc"
     	"github.com/spf13/viper"
-    	
-        "github.com/vulcanize/ipld-eth-server/pkg/client"
-        "github.com/vulcanize/ipld-eth-server/pkg/eth"
-        "github.com/vulcanize/ipld-eth-server/pkg/watch"
+
+        "github.com/cerc-io/ipld-eth-server/v5/pkg/client"
+        "github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+        "github.com/cerc-io/ipld-eth-server/v5/pkg/watch"
     )
 
     config, _ := eth.NewEthSubscriptionConfig()
@@ -153,16 +153,16 @@ the addresses in the `addresses` fields are pre-hashed ETH addresses.
 An example of how to subscribe to a real-time Bitcoin data feed from ipld-eth-server using the `Stream` RPC method is provided below
 
 ```go
-    package main 
+    package main
 
     import (
     	"github.com/ethereum/go-ethereum/rlp"
     	"github.com/ethereum/go-ethereum/rpc"
     	"github.com/spf13/viper"
-    	
-    	"github.com/vulcanize/ipld-eth-server/pkg/btc"
-    	"github.com/vulcanize/ipld-eth-server/pkg/client"
-    	"github.com/vulcanize/ipld-eth-server/pkg/watch"
+
+    	"github.com/vulcanize/ipld-eth-server/v5/pkg/btc"
+    	"github.com/vulcanize/ipld-eth-server/v5/pkg/client"
+    	"github.com/vulcanize/ipld-eth-server/v5/pkg/watch"
     )
 
     config, _ := btc.NewBtcSubscriptionConfig()

entrypoint.sh

@@ -1,28 +1,8 @@
 #!/bin/sh
-# Runs the db migrations and starts the watcher services
 
-# Construct the connection string for postgres
-VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
-
-# Run the DB migrations
-echo "Connecting with: $VDB_PG_CONNECT"
-echo "Running database migrations"
-./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
-rv=$?
-
-if [ $rv != 0 ]; then
-  echo "Could not run migrations. Are the database details correct?"
-  exit 1
+START_CMD="./ipld-eth-server"
+if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
+    START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec `pwd`/ipld-eth-server --continue --"
 fi
 
-
-echo "Beginning the ipld-eth-server process"
-
-echo running: ./ipld-eth-server ${VDB_COMMAND} --config=config.toml
-./ipld-eth-server ${VDB_COMMAND} --config=config.toml
-rv=$?
-
-if [ $rv != 0 ]; then
-  echo "ipld-eth-server startup failed"
-  exit 1
-fi
+exec $START_CMD ${VDB_COMMAND:-serve}

environments/example.toml

@@ -1,14 +1,33 @@
 [database]
-    name     = "vulcanize_public" # $DATABASE_NAME
+    name     = "cerc_testing" # $DATABASE_NAME
     hostname = "localhost" # $DATABASE_HOSTNAME
     port     = 5432 # $DATABASE_PORT
     user     = "postgres" # $DATABASE_USER
     password = "" # $DATABASE_PASSWORD
 
 [log]
-    level = "info" # $LOGRUS_LEVEL
+    level = "info" # $LOG_LEVEL
 
 [server]
+    ipc = false
     ipcPath = "~/.vulcanize/vulcanize.ipc" # $SERVER_IPC_PATH
-    wsPath = "127.0.0.1:8081" # $SERVER_WS_PATH
-    httpPath = "127.0.0.1:8082" # $SERVER_HTTP_PATH
+    ws = true
+    wsPath = "127.0.0.1:8080" # $SERVER_WS_PATH
+    http = true
+    httpPath = "127.0.0.1:8081" # $SERVER_HTTP_PATH
+    graphql = true # $SERVER_GRAPHQL
+    graphqlPath = "127.0.0.1:8082" # $SERVER_GRAPHQL_PATH
+
+[ethereum]
+    chainConfig = "./chain.json" # ETH_CHAIN_CONFIG
+    chainID = "1" # $ETH_CHAIN_ID
+    rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
+    httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF
+    stateDiffTimeout = "4m" # $ETH_STATEDIFF_TIMEOUT
+    forwardEthCalls = false # $ETH_FORWARD_ETH_CALLS
+    proxyOnError = true # $ETH_PROXY_ON_ERROR
+    nodeID = "arch1" # $ETH_NODE_ID
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
+    networkID = "1" # $ETH_NETWORK_ID

environments/subscribeExample.toml

@@ -1,30 +0,0 @@
-[watcher]
-    [watcher.ethSubscription]
-        historicalData = false
-        historicalDataOnly = false
-        startingBlock = 0
-        endingBlock = 0
-        wsPath = "ws://127.0.0.1:8080"
-        [watcher.ethSubscription.headerFilter]
-            off = false
-            uncles = false
-        [watcher.ethSubscription.txFilter]
-            off = false
-            src = []
-            dst = []
-        [watcher.ethSubscription.receiptFilter]
-            off = false
-            contracts = []
-            topic0s = []
-            topic1s = []
-            topic2s = []
-            topic3s = []
-        [watcher.ethSubscription.stateFilter]
-            off = false
-            addresses = []
-           intermediateNodes = false
-        [watcher.ethSubscription.storageFilter]
-            off = true
-            addresses = []
-            storageKeys = []
-            intermediateNodes = false

environments/testing.toml

@@ -1,5 +1,5 @@
 [database]
-  name     = "vulcanize_testing"
+  name     = "cerc_testing"
   hostname = "localhost"
   port     = 5432
 

go.mod

@@ -1,23 +1,296 @@
-module github.com/vulcanize/ipld-eth-server
+module github.com/cerc-io/ipld-eth-server/v5
 
-go 1.13
+go 1.21
 
 require (
-	github.com/ethereum/go-ethereum v1.9.11
-	github.com/ipfs/go-cid v0.0.5
-	github.com/ipfs/go-ipfs-blockstore v1.0.0
-	github.com/ipfs/go-ipfs-ds-help v1.0.0
-	github.com/ipfs/go-ipld-format v0.2.0
-	github.com/jmoiron/sqlx v1.2.0
-	github.com/lib/pq v1.5.2
-	github.com/multiformats/go-multihash v0.0.13
-	github.com/onsi/ginkgo v1.12.1
-	github.com/onsi/gomega v1.10.1
-	github.com/sirupsen/logrus v1.6.0
-	github.com/spf13/cobra v1.0.0
-	github.com/spf13/viper v1.7.0
-	github.com/vulcanize/ipld-eth-indexer v0.2.0-alpha
-	github.com/vulcanize/pg-ipfs-ethdb v0.0.1-alpha
+	github.com/cerc-io/eth-ipfs-state-validator/v5 v5.2.0-alpha
+	github.com/cerc-io/eth-iterator-utils v0.3.1
+	github.com/cerc-io/ipfs-ethdb/v5 v5.1.0-alpha
+	github.com/cerc-io/ipld-eth-statedb v0.1.1
+	github.com/cerc-io/plugeth-statediff v0.3.2
+	github.com/ethereum/go-ethereum v1.13.14
+	github.com/google/uuid v1.6.0
+	github.com/graph-gophers/graphql-go v1.3.0
+	github.com/holiman/uint256 v1.2.4
+	github.com/ipfs/go-cid v0.4.1
+	github.com/jmoiron/sqlx v1.3.5
+	github.com/joho/godotenv v1.4.0
+	github.com/lib/pq v1.10.9
+	github.com/machinebox/graphql v0.2.2
+	github.com/mailgun/groupcache/v2 v2.3.0
+	github.com/onsi/ginkgo/v2 v2.15.0
+	github.com/onsi/gomega v1.30.0
+	github.com/prometheus/client_golang v1.18.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.8.0
+	github.com/spf13/viper v1.18.2
+	gorm.io/driver/postgres v1.3.7
+	gorm.io/gorm v1.23.5
 )
 
-replace github.com/ethereum/go-ethereum v1.9.11 => github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5
+require (
+	bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect
+	github.com/DataDog/zstd v1.5.5 // indirect
+	github.com/Jorropo/jsync v1.0.1 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/VictoriaMetrics/fastcache v1.12.2 // indirect
+	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
+	github.com/benbjohnson/clock v1.3.5 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.10.0 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
+	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.10.0 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 // indirect
+	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
+	github.com/consensys/bavard v0.1.13 // indirect
+	github.com/consensys/gnark-crypto v0.12.1 // indirect
+	github.com/containerd/cgroups v1.1.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
+	github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
+	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
+	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
+	github.com/cskr/pubsub v1.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
+	github.com/deckarep/golang-set/v2 v2.3.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
+	github.com/deepmap/oapi-codegen v1.8.2 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/elastic/gosigar v0.14.2 // indirect
+	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
+	github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
+	github.com/ferranbt/fastssz v0.1.2 // indirect
+	github.com/fjl/memsize v0.0.2 // indirect
+	github.com/flynn/noise v1.1.0 // indirect
+	github.com/francoispqt/gojay v1.2.13 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
+	github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect
+	github.com/georgysavva/scany v0.2.9 // indirect
+	github.com/getsentry/sentry-go v0.22.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-bexpr v0.1.12 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 // indirect
+	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
+	github.com/huin/goupnp v1.3.0 // indirect
+	github.com/inconshreveable/log15 v2.16.0+incompatible // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
+	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c // indirect
+	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/boxo v0.19.0 // indirect
+	github.com/ipfs/go-bitfield v1.1.0 // indirect
+	github.com/ipfs/go-block-format v0.2.0 // indirect
+	github.com/ipfs/go-cidutil v0.1.0 // indirect
+	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-ds-measure v0.2.0 // indirect
+	github.com/ipfs/go-fs-lock v0.0.7 // indirect
+	github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
+	github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
+	github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.3 // indirect
+	github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
+	github.com/ipfs/go-ipld-format v0.6.0 // indirect
+	github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.5.1 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
+	github.com/ipfs/go-unixfsnode v1.9.0 // indirect
+	github.com/ipfs/kubo v0.27.0 // indirect
+	github.com/ipld/go-car/v2 v2.13.1 // indirect
+	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
+	github.com/ipld/go-ipld-prime v0.21.0 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.14.3 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.3 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgtype v1.14.0 // indirect
+	github.com/jackc/pgx/v4 v4.18.3 // indirect
+	github.com/jackc/puddle v1.3.0 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.4 // indirect
+	github.com/klauspost/compress v1.17.6 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/koron/go-ssdp v0.0.4 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/libp2p/go-cidranger v1.1.0 // indirect
+	github.com/libp2p/go-doh-resolver v0.4.0 // indirect
+	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
+	github.com/libp2p/go-libp2p v0.33.0 // indirect
+	github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
+	github.com/libp2p/go-libp2p-kad-dht v0.24.4 // indirect
+	github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect
+	github.com/libp2p/go-libp2p-pubsub v0.10.0 // indirect
+	github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
+	github.com/libp2p/go-libp2p-record v0.2.0 // indirect
+	github.com/libp2p/go-libp2p-routing-helpers v0.7.3 // indirect
+	github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
+	github.com/libp2p/go-msgio v0.3.0 // indirect
+	github.com/libp2p/go-nat v0.2.0 // indirect
+	github.com/libp2p/go-netroute v0.2.1 // indirect
+	github.com/libp2p/go-reuseport v0.4.0 // indirect
+	github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
+	github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
+	github.com/matryer/is v1.4.1 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/miekg/dns v1.1.58 // indirect
+	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
+	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/pointerstructure v1.2.1 // indirect
+	github.com/mmcloughlin/addchain v0.4.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr v0.12.2 // indirect
+	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.2.0 // indirect
+	github.com/multiformats/go-multicodec v0.9.0 // indirect
+	github.com/multiformats/go-multihash v0.2.3 // indirect
+	github.com/multiformats/go-multistream v0.5.0 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/opencontainers/runtime-spec v1.2.0 // indirect
+	github.com/openrelayxyz/plugeth-utils v1.5.0 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
+	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
+	github.com/pganalyze/pg_query_go/v4 v4.2.1 // indirect
+	github.com/pion/datachannel v1.5.5 // indirect
+	github.com/pion/dtls/v2 v2.2.8 // indirect
+	github.com/pion/ice/v2 v2.3.11 // indirect
+	github.com/pion/interceptor v0.1.25 // indirect
+	github.com/pion/logging v0.2.2 // indirect
+	github.com/pion/mdns v0.0.9 // indirect
+	github.com/pion/randutil v0.1.0 // indirect
+	github.com/pion/rtcp v1.2.13 // indirect
+	github.com/pion/rtp v1.8.3 // indirect
+	github.com/pion/sctp v1.8.9 // indirect
+	github.com/pion/sdp/v3 v3.0.6 // indirect
+	github.com/pion/srtp/v2 v2.0.18 // indirect
+	github.com/pion/stun v0.6.1 // indirect
+	github.com/pion/transport/v2 v2.2.4 // indirect
+	github.com/pion/turn/v2 v2.1.4 // indirect
+	github.com/pion/webrtc/v3 v3.2.23 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/polydawn/refmt v0.89.0 // indirect
+	github.com/prometheus/client_model v0.6.0 // indirect
+	github.com/prometheus/common v0.47.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/quic-go/qpack v0.4.0 // indirect
+	github.com/quic-go/quic-go v0.41.0 // indirect
+	github.com/quic-go/webtransport-go v0.6.0 // indirect
+	github.com/raulk/go-watchdog v1.3.0 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/rs/cors v1.9.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/samber/lo v1.39.0 // indirect
+	github.com/segmentio/fasthash v1.0.3 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/status-im/keycard-go v0.2.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/supranational/blst v0.3.11 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
+	github.com/thoas/go-funk v0.9.3 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
+	github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
+	github.com/urfave/cli/v2 v2.25.7 // indirect
+	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
+	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
+	github.com/whyrusleeping/cbor-gen v0.0.0-20240109153615-66e95c3e8a87 // indirect
+	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
+	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
+	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/otel v1.25.0 // indirect
+	go.opentelemetry.io/otel/metric v1.25.0 // indirect
+	go.opentelemetry.io/otel/trace v1.25.0 // indirect
+	go.uber.org/dig v1.17.1 // indirect
+	go.uber.org/fx v1.20.1 // indirect
+	go.uber.org/mock v0.4.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	go4.org v0.0.0-20230225012048-214862532bf5 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.20.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	gonum.org/v1/gonum v0.14.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	lukechampine.com/blake3 v1.2.2 // indirect
+	rsc.io/tmplfunc v0.0.3 // indirect
+)

integration/deploy_helpers.go

@@ -0,0 +1,72 @@
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/big"
+	"net/http"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+type ContractDeployed struct {
+	Address         common.Address `json:"address"`
+	TransactionHash common.Hash    `json:"txHash"`
+	BlockNumber     int64          `json:"blockNumber"`
+	BlockHash       common.Hash    `json:"blockHash"`
+}
+
+type ContractDestroyed struct {
+	BlockNumber int64 `json:"blockNumber"`
+}
+
+type Tx struct {
+	From            string   `json:"from"`
+	To              string   `json:"to"`
+	Value           *big.Int `json:"value"`
+	TransactionHash string   `json:"txHash"`
+	BlockNumber     int64    `json:"blockNumber"`
+	BlockHash       string   `json:"blockHash"`
+}
+
+type StorageKey struct {
+	Key string `json:"key"`
+}
+
+type CountIncremented struct {
+	BlockNumber *big.Int `json:"blockNumber"`
+}
+
+const ContractServerUrl = "http://localhost:3000"
+
+// Factory which creates endpoint functions
+func MakeGetAndDecodeFunc[R any](format string) func(...interface{}) (*R, error) {
+	return func(params ...interface{}) (*R, error) {
+		params = append([]interface{}{ContractServerUrl}, params...)
+		url := fmt.Sprintf(format, params...)
+		res, err := http.Get(url)
+		if err != nil {
+			return nil, err
+		}
+		defer res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("%s: %s", url, res.Status)
+		}
+
+		var data R
+		decoder := json.NewDecoder(res.Body)
+		return &data, decoder.Decode(&data)
+	}
+}
+
+var (
+	DeployContract     = MakeGetAndDecodeFunc[ContractDeployed]("%s/v1/deployContract")
+	DestroyContract    = MakeGetAndDecodeFunc[ContractDestroyed]("%s/v1/destroyContract?addr=%s")
+	DeploySLVContract  = MakeGetAndDecodeFunc[ContractDeployed]("%s/v1/deploySLVContract")
+	DestroySLVContract = MakeGetAndDecodeFunc[ContractDestroyed]("%s/v1/destroySLVContract?addr=%s")
+	SendEth            = MakeGetAndDecodeFunc[Tx]("%s/v1/sendEth?to=%s&value=%s")
+	GetStorageSlotKey  = MakeGetAndDecodeFunc[StorageKey]("%s/v1/getStorageKey?contract=%s&label=%s")
+	IncrementCount     = MakeGetAndDecodeFunc[CountIncremented]("%s/v1/incrementCount%s?addr=%s")
+	Create2Contract    = MakeGetAndDecodeFunc[ContractDeployed]("%s/v1/create2Contract?contract=%s&salt=%s")
+)

integration/direct_proxy_integration_test.go

@@ -0,0 +1,378 @@
+package integration_test
+
+import (
+	"context"
+	"math/big"
+	"math/rand"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/cerc-io/ipld-eth-server/v5/integration"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+)
+
+var _ = Describe("Direct proxy integration test", Label("proxy"), func() {
+	ctx := context.Background()
+
+	var contract *integration.ContractDeployed
+	var tx *integration.Tx
+	var contractErr error
+	var txErr error
+
+	Describe("Get Block", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("gets not existing block by number", func() {
+			blockNum := contract.BlockNumber + 100
+
+			gethBlock, err := gethClient.BlockByNumber(ctx, big.NewInt(blockNum))
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(gethBlock).To(BeZero())
+
+			ipldBlock, err := ipldClient.BlockByNumber(ctx, big.NewInt(blockNum))
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(ipldBlock).To(BeZero())
+		})
+
+		It("gets not existing block by hash", func() {
+			gethBlock, err := gethClient.BlockByHash(ctx, nonExistingBlockHash)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(gethBlock).To(BeZero())
+
+			ipldBlock, err := ipldClient.BlockByHash(ctx, nonExistingBlockHash)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(ipldBlock).To(BeZero())
+		})
+
+		It("gets block by number", func() {
+			blockNum := contract.BlockNumber
+
+			_, err := gethClient.BlockByNumber(ctx, big.NewInt(blockNum))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BlockByNumber(ctx, big.NewInt(blockNum))
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("gets block by hash", func() {
+			_, err := gethClient.BlockByHash(ctx, contract.BlockHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BlockByHash(ctx, contract.BlockHash)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("Transaction", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("Get tx by hash", func() {
+			_, _, err := gethClient.TransactionByHash(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, _, err = ipldClient.TransactionByHash(ctx, contract.TransactionHash)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("not found"))
+		})
+
+		It("Get tx by block hash and index", func() {
+			_, err := gethClient.TransactionInBlock(ctx, contract.BlockHash, 0)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.TransactionInBlock(ctx, contract.BlockHash, 0)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("not found"))
+		})
+	})
+
+	Describe("Receipt", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("Get tx receipt", func() {
+			_, err := gethClient.TransactionReceipt(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.TransactionReceipt(ctx, contract.TransactionHash)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("not found"))
+		})
+	})
+
+	Describe("FilterLogs", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("with blockhash", func() {
+			blockHash := contract.BlockHash
+			filterQuery := ethereum.FilterQuery{
+				//Addresses: addresses,
+				BlockHash: &blockHash,
+				Topics:    [][]common.Hash{},
+			}
+
+			gethLogs, err := gethClient.FilterLogs(ctx, filterQuery)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldLogs, err := ipldClient.FilterLogs(ctx, filterQuery)
+			Expect(err).ToNot(HaveOccurred())
+
+			// not empty list
+			Expect(gethLogs).ToNot(BeEmpty())
+			// empty list
+			Expect(ipldLogs).To(BeEmpty())
+		})
+	})
+
+	Describe("CodeAt", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("gets code of deployed contract with block number", func() {
+			_, err := gethClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldCode).To(BeEmpty())
+		})
+		It("gets code of contract that doesn't exist at this height", func() {
+			gethCode, err := gethClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethCode).To(BeEmpty())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+
+		It("gets code at non-existing address without block number", func() {
+			gethCode, err := gethClient.CodeAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethCode).To(BeEmpty())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+		It("gets code of deployed contract without block number", func() {
+			_, err := gethClient.CodeAt(ctx, contract.Address, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldCode).To(BeEmpty())
+		})
+	})
+
+	Describe("Get balance", func() {
+		var newAddress common.Address
+		rand.Read(newAddress[:])
+
+		BeforeEach(func() {
+			tx, txErr = integration.SendEth(newAddress, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+		})
+
+		It("gets balance for an account with eth without block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethBalance.String()).To(Equal("10000000000000000"))
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, newAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldBalance.String()).To(Equal("0"))
+		})
+		It("gets balance for an account with eth with block number", func() {
+			_, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber))
+			Expect(err).To(MatchError("header not found"))
+		})
+		It("gets historical balance for an account with eth with block number", func() {
+			_, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber-1))
+			Expect(err).To(MatchError("header not found"))
+		})
+
+		It("gets balance for a non-existing account without block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+		It("gets balance for an non-existing block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber+3))
+			Expect(err).To(MatchError("header not found"))
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, nonExistingAddress, big.NewInt(tx.BlockNumber+3))
+			Expect(err).To(MatchError("header not found"))
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+	})
+
+	Describe("Get Storage", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("gets ERC20 total supply (without block number)", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethStorage)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldStorage).To(Equal(make([]byte, 32)))
+		})
+
+		It("gets ERC20 total supply (with block number)", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethStorage)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			_, err = ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).To(MatchError("header not found"))
+		})
+
+		It("gets storage for non-existing account", func() {
+			_, err := gethClient.StorageAt(ctx, nonExistingAddress, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.StorageAt(ctx, nonExistingAddress, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).To(MatchError("header not found"))
+		})
+
+		It("gets storage for non-existing contract slot", func() {
+			_, err := gethClient.StorageAt(ctx, contract.Address, randomHash, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.StorageAt(ctx, contract.Address, randomHash, big.NewInt(contract.BlockNumber))
+			Expect(err).To(MatchError("header not found"))
+		})
+
+		It("gets storage for non-existing contract", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(0))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for non-existing block number", func() {
+			blockNum := contract.BlockNumber + 100
+
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(blockNum))
+			Expect(err).To(MatchError("header not found"))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(blockNum))
+			Expect(err).To(MatchError("header not found"))
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("get storage after self destruct", func() {
+			tx, err := integration.DestroyContract(contract.Address)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethStorage1, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+			gethStorage2, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethStorage1).NotTo(Equal(gethStorage2))
+			Expect(gethStorage2).To(Equal(eth.EmptyNodeValue))
+
+			_, err = ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber-1))
+			Expect(err).To(MatchError("header not found"))
+
+			_, err = ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber))
+			Expect(err).To(MatchError("header not found"))
+
+			// Query the current block
+			ipldStorage3, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(eth.EmptyNodeValue).To(Equal(ipldStorage3))
+		})
+	})
+
+	Describe("eth call", func() {
+		var msg ethereum.CallMsg
+
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			msg = ethereum.CallMsg{
+				To:   &contract.Address,
+				Data: common.Hex2Bytes("18160ddd"), // totalSupply()
+			}
+		})
+
+		It("calls totalSupply() without block number", func() {
+			gethResult, err := gethClient.CallContract(ctx, msg, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContract(ctx, msg, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+
+		It("calls totalSupply() with block number", func() {
+			gethResult, err := gethClient.CallContract(ctx, msg, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContract(ctx, msg, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+	})
+
+	Describe("Chain ID", func() {
+		It("Check chain id", func() {
+			_, err := gethClient.ChainID(ctx)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.ChainID(ctx)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+})

integration/integration_suite_test.go

@@ -0,0 +1,53 @@
+package integration_test
+
+import (
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestIntegration(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "integration test suite")
+}
+
+var (
+	gethHttpPath    = "http://127.0.0.1:8545"
+	ipldEthHttpPath = "http://127.0.0.1:8081"
+
+	gethClient    *ethclient.Client
+	ipldClient    *ethclient.Client
+	gethRPCClient *rpc.Client
+	ipldRPCClient *rpc.Client
+
+	testChainId int64 = 99
+)
+
+var _ = BeforeSuite(func() {
+	var err error
+
+	envChainID := os.Getenv("ETH_CHAIN_ID")
+	if len(envChainID) == 0 {
+		panic("ETH_CHAIN_ID must be set")
+	}
+	testChainId, err = strconv.ParseInt(envChainID, 10, 64)
+	Expect(err).ToNot(HaveOccurred())
+
+	if path := os.Getenv("ETH_HTTP_PATH"); len(path) != 0 {
+		gethHttpPath = "http://" + path
+	}
+	if path := os.Getenv("SERVER_HTTP_PATH"); len(path) != 0 {
+		ipldEthHttpPath = "http://" + path
+	}
+
+	gethClient, err = ethclient.Dial(gethHttpPath)
+	Expect(err).ToNot(HaveOccurred())
+
+	ipldClient, err = ethclient.Dial(ipldEthHttpPath)
+	Expect(err).ToNot(HaveOccurred())
+})

integration/integration_test.go

@@ -0,0 +1,532 @@
+package integration_test
+
+import (
+	"context"
+	"math/big"
+	"math/rand"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	integration "github.com/cerc-io/ipld-eth-server/v5/integration"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+)
+
+var (
+	nonExistingBlockHash = common.HexToHash("0x111111111111111111111111111111111111111111111111111111111111111")
+	nonExistingAddress   = common.HexToAddress("0x1111111111111111111111111111111111111111")
+	randomAddr           = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
+	randomHash           = crypto.Keccak256Hash(randomAddr.Bytes())
+
+	erc20TotalSupply, _ = new(big.Int).SetString("1000000000000000000000", 10)
+	ercTotalSupplyIndex = common.HexToHash("0x2")
+)
+
+var _ = Describe("Basic integration test", func() {
+	ctx := context.Background()
+
+	var contract *integration.ContractDeployed
+	var tx *integration.Tx
+	var contractErr error
+	var txErr error
+
+	Describe("Get Block", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			err := waitForBlock(ctx, ipldClient, contract.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("gets not existing block by number", func() {
+			blockNum := big.NewInt(contract.BlockNumber + 100)
+
+			gethBlock, err := gethClient.BlockByNumber(ctx, blockNum)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(gethBlock).To(BeZero())
+
+			ipldBlock, err := ipldClient.BlockByNumber(ctx, blockNum)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(ipldBlock).To(BeZero())
+		})
+
+		It("gets not existing block by hash", func() {
+			gethBlock, err := gethClient.BlockByHash(ctx, nonExistingBlockHash)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(gethBlock).To(BeZero())
+
+			ipldBlock, err := ipldClient.BlockByHash(ctx, nonExistingBlockHash)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(ipldBlock).To(BeZero())
+		})
+
+		It("gets block by number", func() {
+			blockNum := big.NewInt(contract.BlockNumber)
+
+			gethBlock, err := gethClient.BlockByNumber(ctx, blockNum)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBlock, err := ipldClient.BlockByNumber(ctx, blockNum)
+			Expect(err).ToNot(HaveOccurred())
+
+			// check headers are equals
+			Expect(gethBlock.Header()).To(Equal(ipldBlock.Header()))
+
+			gethTxs := gethBlock.Transactions()
+			ipldTxs := ipldBlock.Transactions()
+
+			Expect(gethTxs.Len()).To(Equal(ipldTxs.Len()))
+			Expect(types.TxDifference(gethTxs, ipldTxs).Len()).To(Equal(0))
+		})
+
+		It("gets block by hash", func() {
+			gethBlock, err := gethClient.BlockByHash(ctx, contract.BlockHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBlock, err := ipldClient.BlockByHash(ctx, contract.BlockHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			// check headers are equals
+			compareBlocks(gethBlock, ipldBlock)
+
+			gethTxs := gethBlock.Transactions()
+			ipldTxs := ipldBlock.Transactions()
+
+			Expect(gethTxs.Len()).To(Equal(ipldTxs.Len()))
+			Expect(types.TxDifference(gethTxs, ipldTxs).Len()).To(Equal(0))
+		})
+	})
+
+	Describe("Transaction", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("Get tx by hash", func() {
+			gethTx, _, err := gethClient.TransactionByHash(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldTx, _, err := ipldClient.TransactionByHash(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			compareTxs(gethTx, ipldTx)
+
+			Expect(gethTx.Hash()).To(Equal(ipldTx.Hash()))
+		})
+
+		It("Get tx by block hash and index", func() {
+			gethTx, err := gethClient.TransactionInBlock(ctx, contract.BlockHash, 0)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldTx, err := ipldClient.TransactionInBlock(ctx, contract.BlockHash, 0)
+			Expect(err).ToNot(HaveOccurred())
+
+			compareTxs(gethTx, ipldTx)
+		})
+	})
+
+	Describe("Receipt", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("Get tx receipt", func() {
+			gethReceipt, err := gethClient.TransactionReceipt(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldReceipt, err := ipldClient.TransactionReceipt(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethReceipt).To(Equal(ipldReceipt))
+
+			rlpGeth, err := gethReceipt.MarshalBinary()
+			Expect(err).ToNot(HaveOccurred())
+
+			rlpIpld, err := ipldReceipt.MarshalBinary()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(rlpGeth).To(Equal(rlpIpld))
+		})
+	})
+
+	Describe("FilterLogs", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("with blockhash", func() {
+			blockHash := contract.BlockHash
+			filterQuery := ethereum.FilterQuery{
+				//Addresses: addresses,
+				BlockHash: &blockHash,
+				Topics:    [][]common.Hash{},
+			}
+
+			gethLogs, err := gethClient.FilterLogs(ctx, filterQuery)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldLogs, err := ipldClient.FilterLogs(ctx, filterQuery)
+			Expect(err).ToNot(HaveOccurred())
+
+			// not empty list
+			Expect(gethLogs).ToNot(BeEmpty())
+
+			Expect(len(gethLogs)).To(Equal(len(ipldLogs)))
+			Expect(gethLogs).To(Equal(ipldLogs))
+		})
+	})
+
+	Describe("CodeAt", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("gets code at non-existing address without block number", func() {
+			gethCode, err := gethClient.CodeAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethCode).To(BeEmpty())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+		It("gets code of deployed contract without block number", func() {
+			gethCode, err := gethClient.CodeAt(ctx, contract.Address, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+		It("gets code of deployed contract with block number", func() {
+			gethCode, err := gethClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+		It("gets code of contract that doesn't exist at this height", func() {
+			gethCode, err := gethClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethCode).To(BeEmpty())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+	})
+
+	Describe("Get balance", func() {
+		var newAddress common.Address
+		rand.Read(newAddress[:])
+
+		BeforeEach(func() {
+			tx, txErr = integration.SendEth(newAddress, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+		})
+
+		It("gets balance for an account with eth with block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+		It("gets balance for an account with eth without block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, newAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+
+		It("gets historical balance for an account with eth with block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+		It("gets balance for a non-existing account without block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+		It("gets balance for an non-existing block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber+3))
+			Expect(err).To(MatchError("header not found"))
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber+3))
+			Expect(err).To(MatchError("header not found"))
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+	})
+
+	Describe("Get Storage", func() {
+		var contractSalt string
+		countAIndex := common.HexToHash("0x5")
+
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+			Expect(contract.BlockNumber).ToNot(BeZero())
+
+			contractSalt = common.Bytes2Hex(contract.BlockHash[:10])
+
+			err := waitForBlock(ctx, ipldClient, contract.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("gets ERC20 total supply (with block number)", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethStorage)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets ERC20 total supply (without block number)", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethStorage)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for non-existing account", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, nonExistingAddress, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, nonExistingAddress, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for non-existing contract slot", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, randomHash, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, randomHash, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for non-existing contract", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(0))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for non-existing block number", func() {
+			blockNum := contract.BlockNumber + 100
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(blockNum))
+			Expect(err).To(MatchError("header not found"))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(blockNum))
+			Expect(err).To(MatchError("header not found"))
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for SLV countA after tx", func() {
+			slvContract, contractErr := integration.Create2Contract("SLVToken", contractSalt)
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			gethStorage, err := gethClient.StorageAt(ctx, slvContract.Address, countAIndex, big.NewInt(slvContract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			slvCountA := new(big.Int).SetBytes(gethStorage)
+
+			err = waitForBlock(ctx, ipldClient, slvContract.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, slvContract.Address, countAIndex, big.NewInt(slvContract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCountA := new(big.Int).SetBytes(ipldStorage)
+			Expect(ipldCountA).To(Equal(slvCountA))
+
+			inc, err := integration.IncrementCount("A", slvContract.Address)
+			Expect(err).ToNot(HaveOccurred())
+			slvCountA.Add(slvCountA, big.NewInt(1))
+
+			ipldStorage, err = ipldClient.StorageAt(ctx, slvContract.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCountA = new(big.Int).SetBytes(ipldStorage)
+			Expect(ipldCountA).To(Equal(slvCountA))
+		})
+
+		It("gets storage after destruction", func() {
+			slvContract, contractErr := integration.Create2Contract("SLVToken", contractSalt)
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			tx, err := integration.DestroyContract(contract.Address)
+			Expect(err).ToNot(HaveOccurred())
+
+			slvTx, err := integration.DestroyContract(slvContract.Address)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethStorage1, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+			gethStorage2, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethStorage1).NotTo(Equal(gethStorage2))
+			Expect(gethStorage2).To(Equal(eth.EmptyNodeValue))
+
+			ipldStorage1, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+			ipldStorage2, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(ipldStorage1).To(Equal(gethStorage1))
+			Expect(ipldStorage2).To(Equal(gethStorage2))
+
+			// Query the current block
+			ipldStorage3, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(ipldStorage2).To(Equal(ipldStorage3))
+
+			// Check for SLV contract
+			gethStorage, err := gethClient.StorageAt(ctx, slvContract.Address, countAIndex, big.NewInt(slvTx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(eth.EmptyNodeValue))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, slvContract.Address, countAIndex, big.NewInt(slvTx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldStorage).To(Equal(gethStorage))
+		})
+	})
+
+	Describe("eth call", func() {
+		var msg ethereum.CallMsg
+
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			msg = ethereum.CallMsg{
+				To:   &contract.Address,
+				Data: common.Hex2Bytes("18160ddd"), // totalSupply()
+			}
+		})
+
+		It("calls totalSupply() without block number", func() {
+			gethResult, err := gethClient.CallContract(ctx, msg, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContract(ctx, msg, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+
+		It("calls totalSupply() with block number", func() {
+			gethResult, err := gethClient.CallContract(ctx, msg, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContract(ctx, msg, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+
+		It("calls totalSupply() with block hash", func() {
+			gethResult, err := gethClient.CallContractAtHash(ctx, msg, contract.BlockHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContractAtHash(ctx, msg, contract.BlockHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+	})
+
+	Describe("Chain ID", func() {
+		It("Check chain id", func() {
+			gethChainId, err := gethClient.ChainID(ctx)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldChainId, err := ipldClient.ChainID(ctx)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethChainId).To(Equal(ipldChainId))
+		})
+	})
+})
+
+func compareBlocks(block1 *types.Block, block2 *types.Block) {
+	GinkgoHelper()
+	Expect(block1.Header()).To(Equal(block2.Header()))
+	Expect(block1.Uncles()).To(Equal(block2.Uncles()))
+
+	txs1 := block1.Transactions()
+	txs2 := block2.Transactions()
+
+	Expect(len(txs1)).To(Equal(len(txs2)))
+	for i, tx := range txs1 {
+		compareTxs(tx, txs2[i])
+	}
+}
+
+func compareTxs(tx1 *types.Transaction, tx2 *types.Transaction) {
+	GinkgoHelper()
+	Expect(tx1.Data()).To(Equal(tx2.Data()))
+	Expect(tx1.Hash()).To(Equal(tx2.Hash()))
+	Expect(tx1.Size()).To(Equal(tx2.Size()))
+
+	signer := types.NewLondonSigner(big.NewInt(testChainId))
+
+	gethSender, err := types.Sender(signer, tx1)
+	Expect(err).ToNot(HaveOccurred())
+
+	ipldSender, err := types.Sender(signer, tx2)
+	Expect(err).ToNot(HaveOccurred())
+
+	Expect(gethSender).To(Equal(ipldSender))
+}

integration/util_test.go

@@ -0,0 +1,28 @@
+package integration_test
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+)
+
+func waitForBlock(ctx context.Context, client *ethclient.Client, target int64) error {
+	timeout := 10 * time.Second
+	for {
+		select {
+		case <-time.After(timeout):
+			return errors.New("timed out")
+		default:
+			latest, err := client.BlockNumber(ctx)
+			if err != nil {
+				return err
+			}
+			if uint64(target) <= latest {
+				return nil
+			}
+			time.Sleep(time.Second)
+		}
+	}
+}

integration/watch_address_integration_test.go

@@ -0,0 +1,661 @@
+package integration_test
+
+import (
+	"context"
+	"math/big"
+
+	sdtypes "github.com/cerc-io/plugeth-statediff/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rpc"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	integration "github.com/cerc-io/ipld-eth-server/v5/integration"
+)
+
+var (
+	gethMethod = "statediff_watchAddress"
+	ipldMethod = "vdb_watchAddress"
+)
+
+var _ = Describe("WatchAddress integration test", Ordered, func() {
+	var (
+		ctx = context.Background()
+
+		txErr       error
+		contractErr error
+		incErr      error
+
+		tx  *integration.Tx
+		inc *integration.CountIncremented
+
+		contract1 *integration.ContractDeployed
+		contract2 *integration.ContractDeployed
+		contract3 *integration.ContractDeployed
+
+		countAIndex common.Hash
+
+		prevBalance1 *big.Int
+		prevBalance2 *big.Int
+		prevBalance3 *big.Int
+
+		actualBalance1 *big.Int
+		actualBalance2 *big.Int
+		actualBalance3 *big.Int
+
+		prevCountA1 *big.Int
+		prevCountA2 *big.Int
+		prevCountA3 *big.Int
+
+		actualCountA1 *big.Int
+		actualCountA2 *big.Int
+		actualCountA3 *big.Int
+	)
+
+	BeforeAll(func() {
+		var err error
+
+		gethRPCClient, err = rpc.Dial(gethHttpPath)
+		Expect(err).ToNot(HaveOccurred())
+
+		ipldRPCClient, err = rpc.Dial(ipldEthHttpPath)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Deploy three contracts
+		contract1, contractErr = integration.DeploySLVContract()
+		Expect(contractErr).ToNot(HaveOccurred())
+
+		contract2, contractErr = integration.DeploySLVContract()
+		Expect(contractErr).ToNot(HaveOccurred())
+
+		contract3, contractErr = integration.DeploySLVContract()
+		Expect(contractErr).ToNot(HaveOccurred())
+
+		// Get the storage slot key
+		storageSlotAKey, err := integration.GetStorageSlotKey("SLVToken", "countA")
+		Expect(err).ToNot(HaveOccurred())
+		countAIndex = common.HexToHash(storageSlotAKey.Key)
+
+		// Clear out watched addresses
+		err = gethRPCClient.Call(nil, gethMethod, sdtypes.Clear, nil)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Get initial balances for all the contracts
+		// Contract 1
+		actualBalance1 = big.NewInt(0)
+		initBalance1, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(contract1.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(initBalance1.String()).To(Equal(actualBalance1.String()))
+		prevBalance1 = big.NewInt(0)
+
+		// Contract 2
+		actualBalance2 = big.NewInt(0)
+		initBalance2, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(contract2.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(initBalance2.String()).To(Equal(actualBalance2.String()))
+		prevBalance2 = big.NewInt(0)
+
+		// Contract 3
+		actualBalance3 = big.NewInt(0)
+		initBalance3, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(contract3.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(initBalance3.String()).To(Equal(actualBalance3.String()))
+		prevBalance3 = big.NewInt(0)
+
+		// Get initial storage values for the contracts
+		// Contract 1, countA
+		actualCountA1 = big.NewInt(0)
+		ipldCountA1Storage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, big.NewInt(contract1.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		ipldCountA1 := new(big.Int).SetBytes(ipldCountA1Storage)
+		Expect(ipldCountA1.String()).To(Equal(actualCountA1.String()))
+		prevCountA1 = big.NewInt(0)
+
+		// Contract 2, countA
+		actualCountA2 = big.NewInt(0)
+		ipldCountA2Storage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, big.NewInt(contract2.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		ipldCountA2 := new(big.Int).SetBytes(ipldCountA2Storage)
+		Expect(ipldCountA2.String()).To(Equal("0"))
+		prevCountA2 = big.NewInt(0)
+
+		// Contract 3, countA
+		actualCountA3 = big.NewInt(0)
+		ipldCountA3Storage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, big.NewInt(contract3.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		ipldCountA3 := new(big.Int).SetBytes(ipldCountA3Storage)
+		Expect(ipldCountA3.String()).To(Equal("0"))
+		prevCountA3 = big.NewInt(0)
+	})
+
+	AfterAll(func() {
+		// Clear out watched addresses
+		err := gethRPCClient.Call(nil, gethMethod, sdtypes.Clear, nil)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Context("no contracts being watched", func() {
+		It("indexes state for all the contracts", func() {
+			// WatchedAddresses = []
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(actualBalance1.String()))
+			prevBalance1.Set(actualBalance1)
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(actualBalance2.String()))
+			prevBalance2.Set(actualBalance2)
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(actualBalance3.String()))
+			prevBalance3.Set(actualBalance3)
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(actualCountA1.String()))
+			prevCountA1.Set(actualCountA1)
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(actualCountA2.String()))
+			prevCountA2.Set(actualCountA2)
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(actualCountA3.String()))
+			prevCountA3.Set(actualCountA3)
+		})
+	})
+
+	Context("one contract being watched", func() {
+		It("indexes state only for the watched contract", func() {
+			operation := sdtypes.Add
+			args := []sdtypes.WatchAddressArg{
+				{
+					Address:   contract1.Address.String(),
+					CreatedAt: uint64(contract1.BlockNumber),
+				},
+			}
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).ToNot(HaveOccurred())
+
+			// WatchedAddresses = [Contract1]
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(actualBalance1.String()))
+			prevBalance1.Set(actualBalance1)
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(prevBalance2.String()))
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(prevBalance3.String()))
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(actualCountA1.String()))
+			prevCountA1.Set(actualCountA1)
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(prevCountA2.String()))
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(prevCountA3.String()))
+		})
+	})
+
+	Context("contract added to a non-empty watch-list", func() {
+		It("indexes state only for the watched contracts", func() {
+			operation := sdtypes.Add
+			args := []sdtypes.WatchAddressArg{
+				{
+					Address:   contract2.Address.String(),
+					CreatedAt: uint64(contract2.BlockNumber),
+				},
+			}
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).ToNot(HaveOccurred())
+
+			// WatchedAddresses = [Contract1, Contract2]
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(actualBalance1.String()))
+			prevBalance1.Set(actualBalance1)
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(actualBalance2.String()))
+			prevBalance2.Set(actualBalance2)
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(prevBalance3.String()))
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(actualCountA1.String()))
+			prevCountA1.Set(actualCountA1)
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(actualCountA2.String()))
+			prevCountA2.Set(actualCountA2)
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(prevCountA3.String()))
+		})
+	})
+
+	Context("contract removed from the watch-list", func() {
+		It("indexes state only for the watched contract", func() {
+			operation := sdtypes.Remove
+			args := []sdtypes.WatchAddressArg{
+				{
+					Address:   contract1.Address.String(),
+					CreatedAt: uint64(contract1.BlockNumber),
+				},
+			}
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).ToNot(HaveOccurred())
+
+			// WatchedAddresses = [Contract2]
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(prevBalance1.String()))
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(actualBalance2.String()))
+			prevBalance2.Set(actualBalance2)
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(prevBalance3.String()))
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(prevCountA1.String()))
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(actualCountA2.String()))
+			prevCountA2.Set(actualCountA2)
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(prevCountA3.String()))
+		})
+	})
+
+	Context("list of watched addresses set", func() {
+		It("indexes state only for the watched contracts", func() {
+			operation := sdtypes.Set
+			args := []sdtypes.WatchAddressArg{
+				{
+					Address:   contract1.Address.String(),
+					CreatedAt: uint64(contract1.BlockNumber),
+				},
+				{
+					Address:   contract3.Address.String(),
+					CreatedAt: uint64(contract3.BlockNumber),
+				},
+			}
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).ToNot(HaveOccurred())
+
+			// WatchedAddresses = [Contract1, Contract3]
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(actualBalance1.String()))
+			prevBalance1.Set(actualBalance1)
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(prevBalance2.String()))
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(actualBalance3.String()))
+			prevBalance3.Set(actualBalance3)
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(actualCountA1.String()))
+			prevCountA1.Set(actualCountA1)
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(prevCountA2.String()))
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(actualCountA3.String()))
+			prevCountA3.Set(actualCountA3)
+		})
+	})
+
+	Context("list of watched addresses cleared", func() {
+		It("indexes state for all the contracts", func() {
+			operation := sdtypes.Clear
+			args := []sdtypes.WatchAddressArg{}
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).ToNot(HaveOccurred())
+
+			// WatchedAddresses = []
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(actualBalance1.String()))
+			prevBalance1.Set(actualBalance1)
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(actualBalance2.String()))
+			prevBalance2.Set(actualBalance2)
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+			waitForBlock(ctx, ipldClient, tx.BlockNumber)
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(actualBalance3.String()))
+			prevBalance3.Set(actualBalance3)
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(actualCountA1.String()))
+			prevCountA1.Set(actualCountA1)
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(actualCountA2.String()))
+			prevCountA2.Set(actualCountA2)
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+			waitForBlock(ctx, ipldClient, inc.BlockNumber.Int64())
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(actualCountA3.String()))
+			prevCountA3.Set(actualCountA3)
+		})
+	})
+
+	Context("with invalid args", func() {
+		It("returns an error on invalid operation arg", func() {
+			operation := "WrongOp"
+			args := []sdtypes.WatchAddressArg{}
+
+			gethErr := gethRPCClient.Call(nil, gethMethod, operation, args)
+			Expect(gethErr).To(HaveOccurred())
+			Expect(gethErr.Error()).To(ContainSubstring("unexpected operation"))
+
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).To(HaveOccurred())
+			Expect(ipldErr.Error()).To(ContainSubstring("unexpected operation"))
+
+			Expect(ipldErr).To(Equal(gethErr))
+		})
+
+		It("returns an error on args of invalid type", func() {
+			operation := sdtypes.Add
+			args := []string{"WrongArg"}
+
+			gethErr := gethRPCClient.Call(nil, gethMethod, operation, args)
+			Expect(gethErr).To(HaveOccurred())
+			Expect(gethErr.Error()).To(ContainSubstring("WatchAddressArg"))
+
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).To(HaveOccurred())
+			Expect(ipldErr.Error()).To(ContainSubstring("WatchAddressArg"))
+
+			Expect(ipldErr).To(Equal(gethErr))
+		})
+	})
+})

main.go

@@ -16,14 +16,9 @@
 package main
 
 import (
-	"github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/ipld-eth-server/cmd"
+	"github.com/cerc-io/ipld-eth-server/v5/cmd"
 )
 
 func main() {
-	logrus.SetFormatter(&logrus.TextFormatter{
-		FullTimestamp: true,
-	})
 	cmd.Execute()
 }

monitoring/grafana/dashboard_main.json

@@ -0,0 +1,353 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 4,
+  "links": [],
+  "panels": [
+    {
+      "datasource": null,
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "7.2.1",
+      "targets": [
+        {
+          "expr": "ipld_eth_server_ws_count",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Websocket Connection Count",
+      "type": "gauge"
+    },
+    {
+      "datasource": null,
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "7.2.1",
+      "targets": [
+        {
+          "expr": "ipld_eth_server_ipc_count",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "IPC Connection Count",
+      "type": "gauge"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "rate(ipld_eth_server_http_count[1m])",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "HTTP requests per second",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "hiddenSeries": false,
+      "id": 7,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.9, rate(ipld_eth_server_http_duration_bucket[1m]))",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "0.9",
+          "refId": "B"
+        },
+        {
+          "expr": "histogram_quantile(0.75, rate(ipld_eth_server_http_duration_bucket[1m]))",
+          "interval": "",
+          "legendFormat": "0.75",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.5, rate(ipld_eth_server_http_duration_bucket[1m]))",
+          "interval": "",
+          "legendFormat": "0.5",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "HTTP Requests duration (percentile)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 26,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "ipld-eth-server",
+  "uid": "lFVEvNtGk",
+  "version": 7
+}

monitoring/prometheus.yml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 10s
+
+scrape_configs:
+  - job_name: 'ipld-eth-server'
+    static_configs:
+    - targets: ['localhost:8090']

pkg/client/client.go

@@ -1,45 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-// Client is used by watchers to stream chain IPLD data from a vulcanizedb ipld-eth-server
-package client
-
-import (
-	"context"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-
-	"github.com/ethereum/go-ethereum/rpc"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/serve"
-)
-
-// Client is used to subscribe to the ipld-eth-server ipld data stream
-type Client struct {
-	c *rpc.Client
-}
-
-// NewClient creates a new Client
-func NewClient(c *rpc.Client) *Client {
-	return &Client{
-		c: c,
-	}
-}
-
-// Stream is the main loop for subscribing to iplds from an ipld-eth-server server
-func (c *Client) Stream(payloadChan chan serve.SubscriptionPayload, params eth.SubscriptionSettings) (*rpc.ClientSubscription, error) {
-	return c.c.Subscribe(context.Background(), "vdb", payloadChan, "stream", params)
-}

pkg/debug/backend.go

@@ -0,0 +1,51 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package debug
+
+import (
+	"context"
+	"errors"
+
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/eth/tracers"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+)
+
+var _ tracers.Backend = &Backend{}
+
+var (
+	errMethodNotSupported = errors.New("backend method not supported")
+)
+
+// Backend implements tracers.Backend interface
+type Backend struct {
+	eth.Backend
+}
+
+// StateAtBlock retrieves the state database associated with a certain block
+func (b *Backend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) {
+	return nil, func() {}, errMethodNotSupported
+}
+
+// StateAtTransaction returns the execution environment of a certain transaction
+func (b *Backend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (*core.Message, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) {
+	return nil, vm.BlockContext{}, nil, func() {}, errMethodNotSupported
+}

pkg/eth/api_test.go

@@ -1,583 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	"context"
-	"strconv"
-
-	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rpc"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-var (
-	expectedBlock = map[string]interface{}{
-		"number":           (*hexutil.Big)(mocks.MockBlock.Number()),
-		"hash":             mocks.MockBlock.Hash(),
-		"parentHash":       mocks.MockBlock.ParentHash(),
-		"nonce":            mocks.MockBlock.Header().Nonce,
-		"mixHash":          mocks.MockBlock.MixDigest(),
-		"sha3Uncles":       mocks.MockBlock.UncleHash(),
-		"logsBloom":        mocks.MockBlock.Bloom(),
-		"stateRoot":        mocks.MockBlock.Root(),
-		"miner":            mocks.MockBlock.Coinbase(),
-		"difficulty":       (*hexutil.Big)(mocks.MockBlock.Difficulty()),
-		"extraData":        hexutil.Bytes(mocks.MockBlock.Header().Extra),
-		"gasLimit":         hexutil.Uint64(mocks.MockBlock.GasLimit()),
-		"gasUsed":          hexutil.Uint64(mocks.MockBlock.GasUsed()),
-		"timestamp":        hexutil.Uint64(mocks.MockBlock.Time()),
-		"transactionsRoot": mocks.MockBlock.TxHash(),
-		"receiptsRoot":     mocks.MockBlock.ReceiptHash(),
-		"totalDifficulty":  (*hexutil.Big)(mocks.MockBlock.Difficulty()),
-		"size":             hexutil.Uint64(mocks.MockBlock.Size()),
-	}
-	expectedHeader = map[string]interface{}{
-		"number":           (*hexutil.Big)(mocks.MockBlock.Header().Number),
-		"hash":             mocks.MockBlock.Header().Hash(),
-		"parentHash":       mocks.MockBlock.Header().ParentHash,
-		"nonce":            mocks.MockBlock.Header().Nonce,
-		"mixHash":          mocks.MockBlock.Header().MixDigest,
-		"sha3Uncles":       mocks.MockBlock.Header().UncleHash,
-		"logsBloom":        mocks.MockBlock.Header().Bloom,
-		"stateRoot":        mocks.MockBlock.Header().Root,
-		"miner":            mocks.MockBlock.Header().Coinbase,
-		"difficulty":       (*hexutil.Big)(mocks.MockBlock.Header().Difficulty),
-		"extraData":        hexutil.Bytes(mocks.MockBlock.Header().Extra),
-		"size":             hexutil.Uint64(mocks.MockBlock.Header().Size()),
-		"gasLimit":         hexutil.Uint64(mocks.MockBlock.Header().GasLimit),
-		"gasUsed":          hexutil.Uint64(mocks.MockBlock.Header().GasUsed),
-		"timestamp":        hexutil.Uint64(mocks.MockBlock.Header().Time),
-		"transactionsRoot": mocks.MockBlock.Header().TxHash,
-		"receiptsRoot":     mocks.MockBlock.Header().ReceiptHash,
-		"totalDifficulty":  (*hexutil.Big)(mocks.MockBlock.Header().Difficulty),
-	}
-	expectedTransaction = eth.NewRPCTransaction(mocks.MockTransactions[0], mocks.MockBlock.Hash(), mocks.MockBlock.NumberU64(), 0)
-)
-
-var _ = Describe("API", func() {
-	var (
-		db                *postgres.DB
-		retriever         *eth.CIDRetriever
-		fetcher           *eth.IPLDFetcher
-		indexAndPublisher *eth2.IPLDPublisher
-		backend           *eth.Backend
-		api               *eth.PublicEthAPI
-	)
-	BeforeEach(func() {
-		var err error
-		db, err = shared.SetupDB()
-		Expect(err).ToNot(HaveOccurred())
-		retriever = eth.NewCIDRetriever(db)
-		fetcher = eth.NewIPLDFetcher(db)
-		indexAndPublisher = eth2.NewIPLDPublisher(db)
-		backend = &eth.Backend{
-			Retriever: retriever,
-			Fetcher:   fetcher,
-			DB:        db,
-		}
-		api = eth.NewPublicEthAPI(backend)
-		err = indexAndPublisher.Publish(mocks.MockConvertedPayload)
-		Expect(err).ToNot(HaveOccurred())
-		uncles := mocks.MockBlock.Uncles()
-		uncleHashes := make([]common.Hash, len(uncles))
-		for i, uncle := range uncles {
-			uncleHashes[i] = uncle.Hash()
-		}
-		expectedBlock["uncles"] = uncleHashes
-	})
-	AfterEach(func() {
-		eth.TearDownDB(db)
-	})
-	Describe("BlockNumber", func() {
-		It("Retrieves the head block number", func() {
-			bn := api.BlockNumber()
-			ubn := (uint64)(bn)
-			subn := strconv.FormatUint(ubn, 10)
-			Expect(subn).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockNumber))
-		})
-	})
-
-	Describe("GetTransactionByHash", func() {
-		It("Retrieves a transaction by hash", func() {
-			hash := mocks.MockTransactions[0].Hash()
-			tx, err := api.GetTransactionByHash(context.Background(), hash)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(tx).To(Equal(expectedTransaction))
-		})
-	})
-
-	Describe("GetBlockByNumber", func() {
-		It("Retrieves a block by number", func() {
-			// without full txs
-			number, err := strconv.ParseInt(mocks.MockCIDPayload.HeaderCID.BlockNumber, 10, 64)
-			Expect(err).ToNot(HaveOccurred())
-			block, err := api.GetBlockByNumber(context.Background(), rpc.BlockNumber(number), false)
-			Expect(err).ToNot(HaveOccurred())
-			transactionHashes := make([]interface{}, len(mocks.MockBlock.Transactions()))
-			for i, trx := range mocks.MockBlock.Transactions() {
-				transactionHashes[i] = trx.Hash()
-			}
-			expectedBlock["transactions"] = transactionHashes
-			for key, val := range expectedBlock {
-				Expect(val).To(Equal(block[key]))
-			}
-			// with full txs
-			block, err = api.GetBlockByNumber(context.Background(), rpc.BlockNumber(number), true)
-			Expect(err).ToNot(HaveOccurred())
-			transactions := make([]interface{}, len(mocks.MockBlock.Transactions()))
-			for i, trx := range mocks.MockBlock.Transactions() {
-				transactions[i] = eth.NewRPCTransactionFromBlockHash(mocks.MockBlock, trx.Hash())
-			}
-			expectedBlock["transactions"] = transactions
-			for key, val := range expectedBlock {
-				Expect(val).To(Equal(block[key]))
-			}
-		})
-	})
-
-	Describe("GetHeaderByNumber", func() {
-		It("Retrieves a header by number", func() {
-			number, err := strconv.ParseInt(mocks.MockCIDPayload.HeaderCID.BlockNumber, 10, 64)
-			Expect(err).ToNot(HaveOccurred())
-			header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(header).To(Equal(expectedHeader))
-		})
-
-		It("Throws an error if a header cannot be found", func() {
-			number, err := strconv.ParseInt(mocks.MockCIDPayload.HeaderCID.BlockNumber, 10, 64)
-			Expect(err).ToNot(HaveOccurred())
-			header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number+1))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("header at block %d is not available", number+1))
-			Expect(header).To(BeNil())
-			_, err = api.B.DB.Beginx()
-			Expect(err).ToNot(HaveOccurred())
-		})
-	})
-
-	Describe("GetBlockByHash", func() {
-		It("Retrieves a block by hash", func() {
-			// without full txs
-			block, err := api.GetBlockByHash(context.Background(), mocks.MockBlock.Hash(), false)
-			Expect(err).ToNot(HaveOccurred())
-			transactionHashes := make([]interface{}, len(mocks.MockBlock.Transactions()))
-			for i, trx := range mocks.MockBlock.Transactions() {
-				transactionHashes[i] = trx.Hash()
-			}
-			expectedBlock["transactions"] = transactionHashes
-			for key, val := range expectedBlock {
-				Expect(val).To(Equal(block[key]))
-			}
-			// with full txs
-			block, err = api.GetBlockByHash(context.Background(), mocks.MockBlock.Hash(), true)
-			Expect(err).ToNot(HaveOccurred())
-			transactions := make([]interface{}, len(mocks.MockBlock.Transactions()))
-			for i, trx := range mocks.MockBlock.Transactions() {
-				transactions[i] = eth.NewRPCTransactionFromBlockHash(mocks.MockBlock, trx.Hash())
-			}
-			expectedBlock["transactions"] = transactions
-			for key, val := range expectedBlock {
-				Expect(val).To(Equal(block[key]))
-			}
-		})
-	})
-
-	Describe("GetLogs", func() {
-		It("Retrieves receipt logs that match the provided topcis within the provided range", func() {
-			crit := ethereum.FilterQuery{
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-					},
-				},
-				FromBlock: mocks.MockBlock.Number(),
-				ToBlock:   mocks.MockBlock.Number(),
-			}
-			logs, err := api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
-
-			crit = ethereum.FilterQuery{
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-						common.HexToHash("0x05"),
-					},
-				},
-				FromBlock: mocks.MockBlock.Number(),
-				ToBlock:   mocks.MockBlock.Number(),
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
-
-			crit = ethereum.FilterQuery{
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-						common.HexToHash("0x06"),
-					},
-				},
-				FromBlock: mocks.MockBlock.Number(),
-				ToBlock:   mocks.MockBlock.Number(),
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
-
-			crit = ethereum.FilterQuery{
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-					},
-					{
-						common.HexToHash("0x07"),
-					},
-				},
-				FromBlock: mocks.MockBlock.Number(),
-				ToBlock:   mocks.MockBlock.Number(),
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(0))
-
-			crit = ethereum.FilterQuery{
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-					},
-					{
-						common.HexToHash("0x06"),
-					},
-				},
-				FromBlock: mocks.MockBlock.Number(),
-				ToBlock:   mocks.MockBlock.Number(),
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
-
-			crit = ethereum.FilterQuery{
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x05"),
-					},
-					{
-						common.HexToHash("0x07"),
-					},
-				},
-				FromBlock: mocks.MockBlock.Number(),
-				ToBlock:   mocks.MockBlock.Number(),
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
-
-			crit = ethereum.FilterQuery{
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x05"),
-					},
-					{
-						common.HexToHash("0x06"),
-						common.HexToHash("0x07"),
-					},
-				},
-				FromBlock: mocks.MockBlock.Number(),
-				ToBlock:   mocks.MockBlock.Number(),
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
-
-			crit = ethereum.FilterQuery{
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-						common.HexToHash("0x05"),
-					},
-					{
-						common.HexToHash("0x06"),
-						common.HexToHash("0x07"),
-					},
-				},
-				FromBlock: mocks.MockBlock.Number(),
-				ToBlock:   mocks.MockBlock.Number(),
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
-
-			crit = ethereum.FilterQuery{
-				Topics: [][]common.Hash{
-					{},
-					{
-						common.HexToHash("0x07"),
-					},
-				},
-				FromBlock: mocks.MockBlock.Number(),
-				ToBlock:   mocks.MockBlock.Number(),
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
-
-			crit = ethereum.FilterQuery{
-				Topics: [][]common.Hash{
-					{},
-					{
-						common.HexToHash("0x06"),
-					},
-				},
-				FromBlock: mocks.MockBlock.Number(),
-				ToBlock:   mocks.MockBlock.Number(),
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
-
-			crit = ethereum.FilterQuery{
-				Topics:    [][]common.Hash{},
-				FromBlock: mocks.MockBlock.Number(),
-				ToBlock:   mocks.MockBlock.Number(),
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
-		})
-
-		It("Uses the provided blockhash if one is provided", func() {
-			hash := mocks.MockBlock.Hash()
-			crit := ethereum.FilterQuery{
-				BlockHash: &hash,
-				Topics: [][]common.Hash{
-					{},
-					{
-						common.HexToHash("0x06"),
-					},
-				},
-			}
-			logs, err := api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
-
-			crit = ethereum.FilterQuery{
-				BlockHash: &hash,
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-					},
-					{
-						common.HexToHash("0x06"),
-					},
-				},
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
-
-			crit = ethereum.FilterQuery{
-				BlockHash: &hash,
-				Topics: [][]common.Hash{
-					{},
-					{
-						common.HexToHash("0x07"),
-					},
-				},
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
-
-			crit = ethereum.FilterQuery{
-				BlockHash: &hash,
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x05"),
-					},
-					{
-						common.HexToHash("0x07"),
-					},
-				},
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
-
-			crit = ethereum.FilterQuery{
-				BlockHash: &hash,
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-					},
-					{
-						common.HexToHash("0x07"),
-					},
-				},
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(0))
-
-			crit = ethereum.FilterQuery{
-				BlockHash: &hash,
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-						common.HexToHash("0x05"),
-					},
-					{
-						common.HexToHash("0x07"),
-					},
-				},
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
-
-			crit = ethereum.FilterQuery{
-				BlockHash: &hash,
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-						common.HexToHash("0x05"),
-					},
-				},
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
-
-			crit = ethereum.FilterQuery{
-				BlockHash: &hash,
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-						common.HexToHash("0x05"),
-					},
-					{
-						common.HexToHash("0x06"),
-						common.HexToHash("0x07"),
-					},
-				},
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
-
-			crit = ethereum.FilterQuery{
-				BlockHash: &hash,
-				Topics:    [][]common.Hash{},
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
-		})
-
-		It("Filters on contract address if any are provided", func() {
-			hash := mocks.MockBlock.Hash()
-			crit := ethereum.FilterQuery{
-				BlockHash: &hash,
-				Addresses: []common.Address{
-					mocks.Address,
-				},
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-						common.HexToHash("0x05"),
-					},
-					{
-						common.HexToHash("0x06"),
-						common.HexToHash("0x07"),
-					},
-				},
-			}
-			logs, err := api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(1))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
-
-			hash = mocks.MockBlock.Hash()
-			crit = ethereum.FilterQuery{
-				BlockHash: &hash,
-				Addresses: []common.Address{
-					mocks.Address,
-					mocks.AnotherAddress,
-				},
-				Topics: [][]common.Hash{
-					{
-						common.HexToHash("0x04"),
-						common.HexToHash("0x05"),
-					},
-					{
-						common.HexToHash("0x06"),
-						common.HexToHash("0x07"),
-					},
-				},
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
-
-			hash = mocks.MockBlock.Hash()
-			crit = ethereum.FilterQuery{
-				BlockHash: &hash,
-				Addresses: []common.Address{
-					mocks.Address,
-					mocks.AnotherAddress,
-				},
-			}
-			logs, err = api.GetLogs(context.Background(), crit)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
-			Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
-		})
-	})
-})

pkg/eth/api_test/eth_suite_test.go

@@ -14,24 +14,16 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package serve
+package eth_api_test
 
-import log "github.com/sirupsen/logrus"
+import (
+	"testing"
 
-func sendNonBlockingErr(sub Subscription, err error) {
-	log.Error(err)
-	select {
-	case sub.PayloadChan <- SubscriptionPayload{Data: nil, Err: err.Error(), Flag: EmptyFlag}:
-	default:
-		log.Infof("unable to send error to subscription %s", sub.ID)
-	}
-}
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
 
-func sendNonBlockingQuit(sub Subscription) {
-	select {
-	case sub.QuitChan <- true:
-		log.Infof("closing subscription %s", sub.ID)
-	default:
-		log.Infof("unable to close subscription %s; channel has no receiver", sub.ID)
-	}
+func TestETHSuite(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ipld-eth-server/pkg/eth/api_test")
 }

pkg/eth/backend_utils.go

@@ -0,0 +1,470 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"math/big"
+
+	nodeiter "github.com/cerc-io/eth-iterator-utils"
+	"github.com/cerc-io/plugeth-statediff/utils"
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/trie"
+
+	"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/state"
+)
+
+var nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
+var emptyCodeHash = crypto.Keccak256([]byte{})
+
+// These marshalling functions are from internal/ethapi so we have to make our own versions here:
+
+// RPCMarshalHeader converts the given header to the RPC output.
+func RPCMarshalHeader(head *types.Header) map[string]interface{} {
+	result := map[string]interface{}{
+		"number":           (*hexutil.Big)(head.Number),
+		"hash":             head.Hash(),
+		"parentHash":       head.ParentHash,
+		"nonce":            head.Nonce,
+		"mixHash":          head.MixDigest,
+		"sha3Uncles":       head.UncleHash,
+		"logsBloom":        head.Bloom,
+		"stateRoot":        head.Root,
+		"miner":            head.Coinbase,
+		"difficulty":       (*hexutil.Big)(head.Difficulty),
+		"extraData":        hexutil.Bytes(head.Extra),
+		"gasLimit":         hexutil.Uint64(head.GasLimit),
+		"gasUsed":          hexutil.Uint64(head.GasUsed),
+		"timestamp":        hexutil.Uint64(head.Time),
+		"transactionsRoot": head.TxHash,
+		"receiptsRoot":     head.ReceiptHash,
+	}
+	if head.BaseFee != nil {
+		result["baseFeePerGas"] = (*hexutil.Big)(head.BaseFee)
+	}
+	if head.WithdrawalsHash != nil {
+		result["withdrawalsRoot"] = head.WithdrawalsHash
+	}
+	if head.BlobGasUsed != nil {
+		result["blobGasUsed"] = hexutil.Uint64(*head.BlobGasUsed)
+	}
+	if head.ExcessBlobGas != nil {
+		result["excessBlobGas"] = hexutil.Uint64(*head.ExcessBlobGas)
+	}
+	if head.ParentBeaconRoot != nil {
+		result["parentBeaconBlockRoot"] = head.ParentBeaconRoot
+	}
+	return result
+}
+
+// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are
+// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain
+// transaction hashes.
+func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, config *params.ChainConfig) map[string]interface{} {
+	fields := RPCMarshalHeader(block.Header())
+	fields["size"] = hexutil.Uint64(block.Size())
+
+	if inclTx {
+		formatTx := func(idx int, tx *types.Transaction) interface{} {
+			return tx.Hash()
+		}
+		if fullTx {
+			formatTx = func(idx int, tx *types.Transaction) interface{} {
+				return newRPCTransactionFromBlockIndex(block, uint64(idx), config)
+			}
+		}
+		txs := block.Transactions()
+		transactions := make([]interface{}, len(txs))
+		for i, tx := range txs {
+			transactions[i] = formatTx(i, tx)
+		}
+		fields["transactions"] = transactions
+	}
+	uncles := block.Uncles()
+	uncleHashes := make([]common.Hash, len(uncles))
+	for i, uncle := range uncles {
+		uncleHashes[i] = uncle.Hash()
+	}
+	fields["uncles"] = uncleHashes
+	if block.Header().WithdrawalsHash != nil {
+		fields["withdrawals"] = block.Withdrawals()
+	}
+	return fields
+}
+
+// RPCMarshalBlockWithUncleHashes marshals the block with the provided uncle hashes
+func RPCMarshalBlockWithUncleHashes(block *types.Block, uncleHashes []common.Hash, inclTx bool, fullTx bool, config *params.ChainConfig) (map[string]interface{}, error) {
+	fields := RPCMarshalHeader(block.Header())
+	fields["size"] = hexutil.Uint64(block.Size())
+
+	if inclTx {
+		formatTx := func(tx *types.Transaction) (interface{}, error) {
+			return tx.Hash(), nil
+		}
+		if fullTx {
+			formatTx = func(tx *types.Transaction) (interface{}, error) {
+				return NewRPCTransactionFromBlockHash(block, tx.Hash(), config), nil
+			}
+		}
+		txs := block.Transactions()
+		transactions := make([]interface{}, len(txs))
+		var err error
+		for i, tx := range txs {
+			if transactions[i], err = formatTx(tx); err != nil {
+				return nil, err
+			}
+		}
+		fields["transactions"] = transactions
+	}
+
+	fields["uncles"] = uncleHashes
+	return fields, nil
+}
+
+// newRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
+func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash, config *params.ChainConfig) *RPCTransaction {
+	for idx, tx := range b.Transactions() {
+		if tx.Hash() == hash {
+			return newRPCTransactionFromBlockIndex(b, uint64(idx), config)
+		}
+	}
+	return nil
+}
+
+// SignerForTx returns an appropriate Signer for this Transaction
+func SignerForTx(tx *types.Transaction) types.Signer {
+	var signer types.Signer
+	if tx.Protected() {
+		signer = types.LatestSignerForChainID(tx.ChainId())
+	} else {
+		signer = types.HomesteadSigner{}
+	}
+	return signer
+}
+
+// NewRPCTransaction returns a transaction that will serialize to the RPC
+// representation, with the given location metadata set (if available).
+func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, blockTime uint64, index uint64, baseFee *big.Int, config *params.ChainConfig) *RPCTransaction {
+	signer := types.MakeSigner(config, new(big.Int).SetUint64(blockNumber), blockTime)
+	from, _ := types.Sender(signer, tx)
+	v, r, s := tx.RawSignatureValues()
+	result := &RPCTransaction{
+		Type:     hexutil.Uint64(tx.Type()),
+		From:     from,
+		Gas:      hexutil.Uint64(tx.Gas()),
+		GasPrice: (*hexutil.Big)(tx.GasPrice()),
+		Hash:     tx.Hash(),
+		Input:    hexutil.Bytes(tx.Data()), // somehow this is ending up `nil`
+		Nonce:    hexutil.Uint64(tx.Nonce()),
+		To:       tx.To(),
+		Value:    (*hexutil.Big)(tx.Value()),
+		V:        (*hexutil.Big)(v),
+		R:        (*hexutil.Big)(r),
+		S:        (*hexutil.Big)(s),
+	}
+	if blockHash != (common.Hash{}) {
+		result.BlockHash = &blockHash
+		result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber))
+		result.TransactionIndex = (*hexutil.Uint64)(&index)
+	}
+
+	switch tx.Type() {
+	case types.LegacyTxType:
+		// if a legacy transaction has an EIP-155 chain id, include it explicitly
+		if id := tx.ChainId(); id.Sign() != 0 {
+			result.ChainID = (*hexutil.Big)(id)
+		}
+
+	case types.AccessListTxType:
+		al := tx.AccessList()
+		yparity := hexutil.Uint64(v.Sign())
+		result.Accesses = &al
+		result.ChainID = (*hexutil.Big)(tx.ChainId())
+		result.YParity = &yparity
+
+	case types.DynamicFeeTxType:
+		al := tx.AccessList()
+		yparity := hexutil.Uint64(v.Sign())
+		result.Accesses = &al
+		result.ChainID = (*hexutil.Big)(tx.ChainId())
+		result.YParity = &yparity
+		result.GasFeeCap = (*hexutil.Big)(tx.GasFeeCap())
+		result.GasTipCap = (*hexutil.Big)(tx.GasTipCap())
+		// if the transaction has been mined, compute the effective gas price
+		if baseFee != nil && blockHash != (common.Hash{}) {
+			// price = min(gasTipCap + baseFee, gasFeeCap)
+			result.GasPrice = (*hexutil.Big)(effectiveGasPrice(tx, baseFee))
+		} else {
+			result.GasPrice = (*hexutil.Big)(tx.GasFeeCap())
+		}
+
+	case types.BlobTxType:
+		al := tx.AccessList()
+		yparity := hexutil.Uint64(v.Sign())
+		result.Accesses = &al
+		result.ChainID = (*hexutil.Big)(tx.ChainId())
+		result.YParity = &yparity
+		result.GasFeeCap = (*hexutil.Big)(tx.GasFeeCap())
+		result.GasTipCap = (*hexutil.Big)(tx.GasTipCap())
+		// if the transaction has been mined, compute the effective gas price
+		if baseFee != nil && blockHash != (common.Hash{}) {
+			result.GasPrice = (*hexutil.Big)(effectiveGasPrice(tx, baseFee))
+		} else {
+			result.GasPrice = (*hexutil.Big)(tx.GasFeeCap())
+		}
+		result.MaxFeePerBlobGas = (*hexutil.Big)(tx.BlobGasFeeCap())
+		result.BlobVersionedHashes = tx.BlobHashes()
+	}
+	return result
+}
+
+// effectiveGasPrice computes the transaction gas fee, based on the given basefee value.
+//
+//	price = min(gasTipCap + baseFee, gasFeeCap)
+func effectiveGasPrice(tx *types.Transaction, baseFee *big.Int) *big.Int {
+	fee := tx.GasTipCap()
+	fee = fee.Add(fee, baseFee)
+	if tx.GasFeeCapIntCmp(fee) < 0 {
+		return tx.GasFeeCap()
+	}
+	return fee
+}
+
+type rpcBlock struct {
+	Hash         common.Hash      `json:"hash"`
+	Transactions []rpcTransaction `json:"transactions"`
+	UncleHashes  []common.Hash    `json:"uncles"`
+}
+
+type rpcTransaction struct {
+	tx *types.Transaction
+	txExtraInfo
+}
+
+type txExtraInfo struct {
+	BlockNumber *string         `json:"blockNumber,omitempty"`
+	BlockHash   *common.Hash    `json:"blockHash,omitempty"`
+	From        *common.Address `json:"from,omitempty"`
+}
+
+func (tx *rpcTransaction) UnmarshalJSON(msg []byte) error {
+	if err := json.Unmarshal(msg, &tx.tx); err != nil {
+		return err
+	}
+	return json.Unmarshal(msg, &tx.txExtraInfo)
+}
+
+func getBlockAndUncleHashes(cli *rpc.Client, ctx context.Context, method string, args ...interface{}) (*types.Block, []common.Hash, error) {
+	var raw json.RawMessage
+	err := cli.CallContext(ctx, &raw, method, args...)
+	if err != nil {
+		return nil, nil, err
+	} else if len(raw) == 0 {
+		return nil, nil, ethereum.NotFound
+	}
+	// Decode header and transactions.
+	var head *types.Header
+	var body rpcBlock
+	if err := json.Unmarshal(raw, &head); err != nil {
+		return nil, nil, err
+	}
+	if err := json.Unmarshal(raw, &body); err != nil {
+		return nil, nil, err
+	}
+	// Quick-verify transaction and uncle lists. This mostly helps with debugging the server.
+	if head.UncleHash == types.EmptyUncleHash && len(body.UncleHashes) > 0 {
+		return nil, nil, fmt.Errorf("server returned non-empty uncle list but block header indicates no uncles")
+	}
+	if head.UncleHash != types.EmptyUncleHash && len(body.UncleHashes) == 0 {
+		return nil, nil, fmt.Errorf("server returned empty uncle list but block header indicates uncles")
+	}
+	if head.TxHash == types.EmptyRootHash && len(body.Transactions) > 0 {
+		return nil, nil, fmt.Errorf("server returned non-empty transaction list but block header indicates no transactions")
+	}
+	if head.TxHash != types.EmptyRootHash && len(body.Transactions) == 0 {
+		return nil, nil, fmt.Errorf("server returned empty transaction list but block header indicates transactions")
+	}
+	txs := make([]*types.Transaction, len(body.Transactions))
+	for i, tx := range body.Transactions {
+		txs[i] = tx.tx
+	}
+	return types.NewBlockWithHeader(head).WithBody(txs, nil), body.UncleHashes, nil
+}
+
+// newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation.
+func newRPCTransactionFromBlockIndex(b *types.Block, index uint64, config *params.ChainConfig) *RPCTransaction {
+	txs := b.Transactions()
+	if index >= uint64(len(txs)) {
+		return nil
+	}
+	return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), b.Time(), index, b.BaseFee(), config)
+}
+
+// newRPCRawTransactionFromBlockIndex returns the bytes of a transaction given a block and a transaction index.
+func newRPCRawTransactionFromBlockIndex(b *types.Block, index uint64) hexutil.Bytes {
+	txs := b.Transactions()
+	if index >= uint64(len(txs)) {
+		return nil
+	}
+	blob, _ := txs[index].MarshalBinary()
+	return blob
+}
+
+func toFilterArg(q ethereum.FilterQuery) (interface{}, error) {
+	arg := map[string]interface{}{
+		"address": q.Addresses,
+		"topics":  q.Topics,
+	}
+	if q.BlockHash != nil {
+		arg["blockHash"] = *q.BlockHash
+		if q.FromBlock != nil || q.ToBlock != nil {
+			return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock")
+		}
+	} else {
+		if q.FromBlock == nil {
+			arg["fromBlock"] = "0x0"
+		} else {
+			arg["fromBlock"] = toBlockNumArg(q.FromBlock)
+		}
+		arg["toBlock"] = toBlockNumArg(q.ToBlock)
+	}
+	return arg, nil
+}
+
+func toBlockNumArg(number *big.Int) string {
+	if number == nil {
+		return "latest"
+	}
+	return hexutil.EncodeBig(number)
+}
+
+func getIteratorAtPath(t state.Trie, startKey []byte) (trie.NodeIterator, int64, error) {
+	startTime := makeTimestamp()
+	var it trie.NodeIterator
+	var err error
+
+	if len(startKey)%2 != 0 {
+		// Zero-pad for odd-length keys, required by HexToKeyBytes()
+		startKey = append(startKey, 0)
+		it, err = t.NodeIterator(nodeiter.HexToKeyBytes(startKey))
+		if err != nil {
+			return nil, 0, err
+		}
+	} else {
+		it, err = t.NodeIterator(nodeiter.HexToKeyBytes(startKey))
+		if err != nil {
+			return nil, 0, err
+		}
+		// Step to the required node (not required if original startKey was odd-length)
+		it.Next(true)
+	}
+
+	return it, makeTimestamp() - startTime, nil
+}
+
+func fillSliceNodeData(
+	sdb state.Database,
+	nodesMap map[string]string,
+	leavesMap map[string]GetSliceResponseAccount,
+	node StateNode,
+	nodeElements []interface{},
+	storage bool,
+) (int64, error) {
+	// Populate the nodes map
+	nodeValHash := crypto.Keccak256Hash(node.NodeValue)
+	nodesMap[common.Bytes2Hex(nodeValHash.Bytes())] = common.Bytes2Hex(node.NodeValue)
+
+	// Extract account data if it's a Leaf node
+	leafStartTime := makeTimestamp()
+	if node.NodeType == Leaf && !storage {
+		stateLeafKey, storageRoot, code, err := extractContractAccountInfo(sdb, node, nodeElements)
+		if err != nil {
+			return 0, fmt.Errorf("GetSlice account lookup error: %w", err)
+		}
+
+		if len(code) > 0 {
+			// Populate the leaves map
+			leavesMap[stateLeafKey] = GetSliceResponseAccount{
+				StorageRoot: storageRoot,
+				EVMCode:     common.Bytes2Hex(code),
+			}
+		}
+	}
+
+	return makeTimestamp() - leafStartTime, nil
+}
+
+func extractContractAccountInfo(sdb state.Database, node StateNode, nodeElements []interface{}) (string, string, []byte, error) {
+	var account types.StateAccount
+	if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
+		return "", "", nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
+	}
+
+	if bytes.Equal(account.CodeHash, emptyCodeHash) {
+		return "", "", nil, nil
+	}
+
+	// Extract state leaf key
+	partialPath := utils.CompactToHex(nodeElements[0].([]byte))
+	valueNodePath := append(node.Path, partialPath...)
+	encodedPath := utils.HexToCompact(valueNodePath)
+	leafKey := encodedPath[1:]
+	stateLeafKeyString := common.BytesToHash(leafKey).String()
+
+	storageRootString := account.Root.String()
+
+	// Extract codeHash and get code
+	codeHash := common.BytesToHash(account.CodeHash)
+	codeBytes, err := sdb.ContractCode(common.Address{}, codeHash)
+	if err != nil {
+		return "", "", nil, err
+	}
+
+	return stateLeafKeyString, storageRootString, codeBytes, nil
+}
+
+// IsLeaf checks if the node we are at is a leaf
+func IsLeaf(elements []interface{}) (bool, error) {
+	if len(elements) > 2 {
+		return false, nil
+	}
+	if len(elements) < 2 {
+		return false, fmt.Errorf("node cannot be less than two elements in length")
+	}
+	switch elements[0].([]byte)[0] / 16 {
+	case '\x00':
+		return false, nil
+	case '\x01':
+		return false, nil
+	case '\x02':
+		return true, nil
+	case '\x03':
+		return true, nil
+	default:
+		return false, fmt.Errorf("unknown hex prefix")
+	}
+}

pkg/eth/cid_retriever.go

@@ -1,573 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"fmt"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/jmoiron/sqlx"
-	"github.com/lib/pq"
-	log "github.com/sirupsen/logrus"
-
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-// Retriever interface for substituting mocks in tests
-type Retriever interface {
-	RetrieveFirstBlockNumber() (int64, error)
-	RetrieveLastBlockNumber() (int64, error)
-	Retrieve(filter SubscriptionSettings, blockNumber int64) ([]eth2.CIDWrapper, bool, error)
-}
-
-// CIDRetriever satisfies the CIDRetriever interface for ethereum
-type CIDRetriever struct {
-	db *postgres.DB
-}
-
-// NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface
-func NewCIDRetriever(db *postgres.DB) *CIDRetriever {
-	return &CIDRetriever{
-		db: db,
-	}
-}
-
-// RetrieveFirstBlockNumber is used to retrieve the first block number in the db
-func (ecr *CIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
-	var blockNumber int64
-	err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number ASC LIMIT 1")
-	return blockNumber, err
-}
-
-// RetrieveLastBlockNumber is used to retrieve the latest block number in the db
-func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
-	var blockNumber int64
-	err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1 ")
-	return blockNumber, err
-}
-
-// Retrieve is used to retrieve all of the CIDs which conform to the passed StreamFilters
-func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64) ([]eth2.CIDWrapper, bool, error) {
-	log.Debug("retrieving cids")
-
-	// Begin new db tx
-	tx, err := ecr.db.Beginx()
-	if err != nil {
-		return nil, true, err
-	}
-	defer func() {
-		if p := recover(); p != nil {
-			shared.Rollback(tx)
-			panic(p)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = tx.Commit()
-		}
-	}()
-
-	// Retrieve cached header CIDs at this block height
-	headers, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
-	if err != nil {
-		log.Error("header cid retrieval error")
-		return nil, true, err
-	}
-	cws := make([]eth2.CIDWrapper, len(headers))
-	empty := true
-	for i, header := range headers {
-		cw := new(eth2.CIDWrapper)
-		cw.BlockNumber = big.NewInt(blockNumber)
-		if !filter.HeaderFilter.Off {
-			cw.Header = header
-			empty = false
-			if filter.HeaderFilter.Uncles {
-				// Retrieve uncle cids for this header id
-				uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, header.ID)
-				if err != nil {
-					log.Error("uncle cid retrieval error")
-					return nil, true, err
-				}
-				cw.Uncles = uncleCIDs
-			}
-		}
-		// Retrieve cached trx CIDs
-		if !filter.TxFilter.Off {
-			cw.Transactions, err = ecr.RetrieveTxCIDs(tx, filter.TxFilter, header.ID)
-			if err != nil {
-				log.Error("transaction cid retrieval error")
-				return nil, true, err
-			}
-			if len(cw.Transactions) > 0 {
-				empty = false
-			}
-		}
-		trxIds := make([]int64, len(cw.Transactions))
-		for j, tx := range cw.Transactions {
-			trxIds[j] = tx.ID
-		}
-		// Retrieve cached receipt CIDs
-		if !filter.ReceiptFilter.Off {
-			cw.Receipts, err = ecr.RetrieveRctCIDsByHeaderID(tx, filter.ReceiptFilter, header.ID, trxIds)
-			if err != nil {
-				log.Error("receipt cid retrieval error")
-				return nil, true, err
-			}
-			if len(cw.Receipts) > 0 {
-				empty = false
-			}
-		}
-		// Retrieve cached state CIDs
-		if !filter.StateFilter.Off {
-			cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, filter.StateFilter, header.ID)
-			if err != nil {
-				log.Error("state cid retrieval error")
-				return nil, true, err
-			}
-			if len(cw.StateNodes) > 0 {
-				empty = false
-			}
-		}
-		// Retrieve cached storage CIDs
-		if !filter.StorageFilter.Off {
-			cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, filter.StorageFilter, header.ID)
-			if err != nil {
-				log.Error("storage cid retrieval error")
-				return nil, true, err
-			}
-			if len(cw.StorageNodes) > 0 {
-				empty = false
-			}
-		}
-		cws[i] = *cw
-	}
-
-	return cws, empty, err
-}
-
-// RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
-func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]eth2.HeaderModel, error) {
-	log.Debug("retrieving header cids for block ", blockNumber)
-	headers := make([]eth2.HeaderModel, 0)
-	pgStr := `SELECT * FROM eth.header_cids
-				WHERE block_number = $1`
-	return headers, tx.Select(&headers, pgStr, blockNumber)
-}
-
-// RetrieveUncleCIDsByHeaderID retrieves and returns all of the uncle cids for the provided header
-func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth2.UncleModel, error) {
-	log.Debug("retrieving uncle cids for block id ", headerID)
-	headers := make([]eth2.UncleModel, 0)
-	pgStr := `SELECT * FROM eth.uncle_cids
-				WHERE header_id = $1`
-	return headers, tx.Select(&headers, pgStr, headerID)
-}
-
-// RetrieveTxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters
-// also returns the ids for the returned transaction cids
-func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]eth2.TxModel, error) {
-	log.Debug("retrieving transaction cids for header id ", headerID)
-	args := make([]interface{}, 0, 3)
-	results := make([]eth2.TxModel, 0)
-	id := 1
-	pgStr := fmt.Sprintf(`SELECT transaction_cids.id, transaction_cids.header_id,
- 			transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.mh_key,
- 			transaction_cids.dst, transaction_cids.src, transaction_cids.index,
-			transaction_cids.tx_data, transaction_cids.deployment
- 			FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-			WHERE header_cids.id = $%d`, id)
-	args = append(args, headerID)
-	id++
-	if len(txFilter.Dst) > 0 {
-		pgStr += fmt.Sprintf(` AND transaction_cids.dst = ANY($%d::VARCHAR(66)[])`, id)
-		args = append(args, pq.Array(txFilter.Dst))
-		id++
-	}
-	if len(txFilter.Src) > 0 {
-		pgStr += fmt.Sprintf(` AND transaction_cids.src = ANY($%d::VARCHAR(66)[])`, id)
-		args = append(args, pq.Array(txFilter.Src))
-	}
-	pgStr += ` ORDER BY transaction_cids.index`
-	return results, tx.Select(&results, pgStr, args...)
-}
-
-// RetrieveRctCIDsByHeaderID retrieves and returns all of the rct cids at the provided header ID that conform to the provided
-// filter parameters and correspond to the provided tx ids
-func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]eth2.ReceiptModel, error) {
-	log.Debug("retrieving receipt cids for header id ", headerID)
-	args := make([]interface{}, 0, 4)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
- 			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
-			WHERE receipt_cids.tx_id = transaction_cids.id 
-			AND transaction_cids.header_id = header_cids.id
-			AND header_cids.id = $1`
-	id := 2
-	args = append(args, headerID)
-	if len(rctFilter.LogAddresses) > 0 {
-		// Filter on log contract addresses if there are any
-		pgStr += fmt.Sprintf(` AND ((receipt_cids.log_contracts && $%d::VARCHAR(66)[]`, id)
-		args = append(args, pq.Array(rctFilter.LogAddresses))
-		id++
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND ("
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
-			pgStr += ")"
-		}
-		pgStr += ")"
-		// Filter on txIDs if there are any and we are matching txs
-		if rctFilter.MatchTxs && len(trxIds) > 0 {
-			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-		pgStr += ")"
-	} else { // If there are no contract addresses to filter on
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND (("
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
-			pgStr += ")"
-			// Filter on txIDs if there are any and we are matching txs
-			if rctFilter.MatchTxs && len(trxIds) > 0 {
-				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-				args = append(args, pq.Array(trxIds))
-			}
-			pgStr += ")"
-		} else if rctFilter.MatchTxs && len(trxIds) > 0 {
-			// If there are no contract addresses or topics to filter on,
-			// Filter on txIDs if there are any and we are matching txs
-			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-	}
-	pgStr += ` ORDER BY transaction_cids.index`
-	receiptCids := make([]eth2.ReceiptModel, 0)
-	return receiptCids, tx.Select(&receiptCids, pgStr, args...)
-}
-
-// RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight or block hash that conform to the provided
-// filter parameters and correspond to the provided tx ids
-func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]eth2.ReceiptModel, error) {
-	log.Debug("retrieving receipt cids for block ", blockNumber)
-	args := make([]interface{}, 0, 5)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
- 			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
-			WHERE receipt_cids.tx_id = transaction_cids.id 
-			AND transaction_cids.header_id = header_cids.id`
-	id := 1
-	if blockNumber > 0 {
-		pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id)
-		args = append(args, blockNumber)
-		id++
-	}
-	if blockHash != nil {
-		pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id)
-		args = append(args, blockHash.String())
-		id++
-	}
-	if len(rctFilter.LogAddresses) > 0 {
-		// Filter on log contract addresses if there are any
-		pgStr += fmt.Sprintf(` AND ((receipt_cids.log_contracts && $%d::VARCHAR(66)[]`, id)
-		args = append(args, pq.Array(rctFilter.LogAddresses))
-		id++
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND ("
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
-			pgStr += ")"
-		}
-		pgStr += ")"
-		// Filter on txIDs if there are any and we are matching txs
-		if rctFilter.MatchTxs && len(trxIds) > 0 {
-			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-		pgStr += ")"
-	} else { // If there are no contract addresses to filter on
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND (("
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
-			pgStr += ")"
-			// Filter on txIDs if there are any and we are matching txs
-			if rctFilter.MatchTxs && len(trxIds) > 0 {
-				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-				args = append(args, pq.Array(trxIds))
-			}
-			pgStr += ")"
-		} else if rctFilter.MatchTxs && len(trxIds) > 0 {
-			// If there are no contract addresses or topics to filter on,
-			// Filter on txIDs if there are any and we are matching txs
-			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-	}
-	pgStr += ` ORDER BY transaction_cids.index`
-	receiptCids := make([]eth2.ReceiptModel, 0)
-	return receiptCids, tx.Select(&receiptCids, pgStr, args...)
-}
-
-func hasTopics(topics [][]string) bool {
-	for _, topicSet := range topics {
-		if len(topicSet) > 0 {
-			return true
-		}
-	}
-	return false
-}
-
-// RetrieveStateCIDs retrieves and returns all of the state node cids at the provided header ID that conform to the provided filter parameters
-func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID int64) ([]eth2.StateNodeModel, error) {
-	log.Debug("retrieving state cids for header id ", headerID)
-	args := make([]interface{}, 0, 2)
-	pgStr := `SELECT state_cids.id, state_cids.header_id,
-			state_cids.state_leaf_key, state_cids.node_type, state_cids.cid, state_cids.mh_key, state_cids.state_path
-			FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-			WHERE header_cids.id = $1`
-	args = append(args, headerID)
-	addrLen := len(stateFilter.Addresses)
-	if addrLen > 0 {
-		keys := make([]string, addrLen)
-		for i, addr := range stateFilter.Addresses {
-			keys[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes()).String()
-		}
-		pgStr += ` AND state_cids.state_leaf_key = ANY($2::VARCHAR(66)[])`
-		args = append(args, pq.Array(keys))
-	}
-	if !stateFilter.IntermediateNodes {
-		pgStr += ` AND state_cids.node_type = 2`
-	}
-	stateNodeCIDs := make([]eth2.StateNodeModel, 0)
-	return stateNodeCIDs, tx.Select(&stateNodeCIDs, pgStr, args...)
-}
-
-// RetrieveStorageCIDs retrieves and returns all of the storage node cids at the provided header id that conform to the provided filter parameters
-func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID int64) ([]eth2.StorageNodeWithStateKeyModel, error) {
-	log.Debug("retrieving storage cids for header id ", headerID)
-	args := make([]interface{}, 0, 3)
-	pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_leaf_key, storage_cids.node_type,
- 			storage_cids.cid, storage_cids.mh_key, storage_cids.storage_path, state_cids.state_leaf_key
- 			FROM eth.storage_cids, eth.state_cids, eth.header_cids
-			WHERE storage_cids.state_id = state_cids.id 
-			AND state_cids.header_id = header_cids.id
-			AND header_cids.id = $1`
-	args = append(args, headerID)
-	id := 2
-	addrLen := len(storageFilter.Addresses)
-	if addrLen > 0 {
-		keys := make([]string, addrLen)
-		for i, addr := range storageFilter.Addresses {
-			keys[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes()).String()
-		}
-		pgStr += fmt.Sprintf(` AND state_cids.state_leaf_key = ANY($%d::VARCHAR(66)[])`, id)
-		args = append(args, pq.Array(keys))
-		id++
-	}
-	if len(storageFilter.StorageKeys) > 0 {
-		pgStr += fmt.Sprintf(` AND storage_cids.storage_leaf_key = ANY($%d::VARCHAR(66)[])`, id)
-		args = append(args, pq.Array(storageFilter.StorageKeys))
-	}
-	if !storageFilter.IntermediateNodes {
-		pgStr += ` AND storage_cids.node_type = 2`
-	}
-	storageNodeCIDs := make([]eth2.StorageNodeWithStateKeyModel, 0)
-	return storageNodeCIDs, tx.Select(&storageNodeCIDs, pgStr, args...)
-}
-
-// RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash
-func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth2.HeaderModel, []eth2.UncleModel, []eth2.TxModel, []eth2.ReceiptModel, error) {
-	log.Debug("retrieving block cids for block hash ", blockHash.String())
-
-	// Begin new db tx
-	tx, err := ecr.db.Beginx()
-	if err != nil {
-		return eth2.HeaderModel{}, nil, nil, nil, err
-	}
-	defer func() {
-		if p := recover(); p != nil {
-			shared.Rollback(tx)
-			panic(p)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = tx.Commit()
-		}
-	}()
-
-	headerCID, err := ecr.RetrieveHeaderCIDByHash(tx, blockHash)
-	if err != nil {
-		log.Error("header cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
-	}
-	uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.ID)
-	if err != nil {
-		log.Error("uncle cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
-	}
-	txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.ID)
-	if err != nil {
-		log.Error("tx cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
-	}
-	txIDs := make([]int64, len(txCIDs))
-	for i, txCID := range txCIDs {
-		txIDs[i] = txCID.ID
-	}
-	rctCIDs, err := ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
-	if err != nil {
-		log.Error("rct cid retrieval error")
-	}
-	return headerCID, uncleCIDs, txCIDs, rctCIDs, err
-}
-
-// RetrieveBlockByNumber returns all of the CIDs needed to compose an entire block, for a given block number
-func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth2.HeaderModel, []eth2.UncleModel, []eth2.TxModel, []eth2.ReceiptModel, error) {
-	log.Debug("retrieving block cids for block number ", blockNumber)
-
-	// Begin new db tx
-	tx, err := ecr.db.Beginx()
-	if err != nil {
-		return eth2.HeaderModel{}, nil, nil, nil, err
-	}
-	defer func() {
-		if p := recover(); p != nil {
-			shared.Rollback(tx)
-			panic(p)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = tx.Commit()
-		}
-	}()
-
-	headerCID, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
-	if err != nil {
-		log.Error("header cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
-	}
-	if len(headerCID) < 1 {
-		return eth2.HeaderModel{}, nil, nil, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber)
-	}
-	uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID[0].ID)
-	if err != nil {
-		log.Error("uncle cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
-	}
-	txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].ID)
-	if err != nil {
-		log.Error("tx cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
-	}
-	txIDs := make([]int64, len(txCIDs))
-	for i, txCID := range txCIDs {
-		txIDs[i] = txCID.ID
-	}
-	rctCIDs, err := ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
-	if err != nil {
-		log.Error("rct cid retrieval error")
-	}
-	return headerCID[0], uncleCIDs, txCIDs, rctCIDs, err
-}
-
-// RetrieveHeaderCIDByHash returns the header for the given block hash
-func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (eth2.HeaderModel, error) {
-	log.Debug("retrieving header cids for block hash ", blockHash.String())
-	pgStr := `SELECT * FROM eth.header_cids
-			WHERE block_hash = $1`
-	var headerCID eth2.HeaderModel
-	return headerCID, tx.Get(&headerCID, pgStr, blockHash.String())
-}
-
-// RetrieveTxCIDsByHeaderID retrieves all tx CIDs for the given header id
-func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth2.TxModel, error) {
-	log.Debug("retrieving tx cids for block id ", headerID)
-	pgStr := `SELECT * FROM eth.transaction_cids
-			WHERE header_id = $1
-			ORDER BY index`
-	var txCIDs []eth2.TxModel
-	return txCIDs, tx.Select(&txCIDs, pgStr, headerID)
-}
-
-// RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs
-func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]eth2.ReceiptModel, error) {
-	log.Debugf("retrieving receipt cids for tx ids %v", txIDs)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
-			FROM eth.receipt_cids, eth.transaction_cids
-			WHERE tx_id = ANY($1::INTEGER[])
-			AND receipt_cids.tx_id = transaction_cids.id
-			ORDER BY transaction_cids.index`
-	var rctCIDs []eth2.ReceiptModel
-	return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txIDs))
-}

pkg/eth/cid_retriever_test.go

@@ -1,474 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-var (
-	openFilter = eth.SubscriptionSettings{
-		Start:         big.NewInt(0),
-		End:           big.NewInt(1),
-		HeaderFilter:  eth.HeaderFilter{},
-		TxFilter:      eth.TxFilter{},
-		ReceiptFilter: eth.ReceiptFilter{},
-		StateFilter:   eth.StateFilter{},
-		StorageFilter: eth.StorageFilter{},
-	}
-	rctAddressFilter = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			LogAddresses: []string{mocks.Address.String()},
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctTopicsFilter = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			Topics: [][]string{{"0x0000000000000000000000000000000000000000000000000000000000000004"}},
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctTopicsAndAddressFilter = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			Topics: [][]string{
-				{"0x0000000000000000000000000000000000000000000000000000000000000004"},
-				{"0x0000000000000000000000000000000000000000000000000000000000000006"},
-			},
-			LogAddresses: []string{mocks.Address.String()},
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctTopicsAndAddressFilterFail = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			Topics: [][]string{
-				{"0x0000000000000000000000000000000000000000000000000000000000000004"},
-				{"0x0000000000000000000000000000000000000000000000000000000000000007"}, // This topic won't match on the mocks.Address.String() contract receipt
-			},
-			LogAddresses: []string{mocks.Address.String()},
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctAddressesAndTopicFilter = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			Topics:       [][]string{{"0x0000000000000000000000000000000000000000000000000000000000000005"}},
-			LogAddresses: []string{mocks.Address.String(), mocks.AnotherAddress.String()},
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctsForAllCollectedTrxs = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{}, // Trx filter open so we will collect all trxs, therefore we will also collect all corresponding rcts despite rct filter
-		ReceiptFilter: eth.ReceiptFilter{
-			MatchTxs:     true,
-			Topics:       [][]string{{"0x0000000000000000000000000000000000000000000000000000000000000006"}}, // Topic0 isn't one of the topic0s we have
-			LogAddresses: []string{"0x0000000000000000000000000000000000000002"},                             // Contract isn't one of the contracts we have
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctsForSelectCollectedTrxs = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Dst: []string{mocks.AnotherAddress.String()}, // We only filter for one of the trxs so we will only get the one corresponding receipt
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			MatchTxs:     true,
-			Topics:       [][]string{{"0x0000000000000000000000000000000000000000000000000000000000000006"}}, // Topic0 isn't one of the topic0s we have
-			LogAddresses: []string{"0x0000000000000000000000000000000000000002"},                             // Contract isn't one of the contracts we have
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	stateFilter = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			Off: true,
-		},
-		StateFilter: eth.StateFilter{
-			Addresses: []string{mocks.AccountAddresss.Hex()},
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-)
-
-var _ = Describe("Retriever", func() {
-	var (
-		db        *postgres.DB
-		repo      *eth2.IPLDPublisher
-		retriever *eth.CIDRetriever
-	)
-	BeforeEach(func() {
-		var err error
-		db, err = shared.SetupDB()
-		Expect(err).ToNot(HaveOccurred())
-		repo = eth2.NewIPLDPublisher(db)
-		retriever = eth.NewCIDRetriever(db)
-	})
-	AfterEach(func() {
-		eth.TearDownDB(db)
-	})
-
-	Describe("Retrieve", func() {
-		BeforeEach(func() {
-			err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-		})
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-			cids, empty, err := retriever.Retrieve(openFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids)).To(Equal(1))
-			Expect(cids[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
-			expectedHeaderCID := mocks.MockCIDWrapper.Header
-			expectedHeaderCID.ID = cids[0].Header.ID
-			expectedHeaderCID.NodeID = cids[0].Header.NodeID
-			Expect(cids[0].Header).To(Equal(expectedHeaderCID))
-			Expect(len(cids[0].Transactions)).To(Equal(3))
-			Expect(eth.TxModelsContainsCID(cids[0].Transactions, mocks.MockCIDWrapper.Transactions[0].CID)).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids[0].Transactions, mocks.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids[0].Transactions, mocks.MockCIDWrapper.Transactions[2].CID)).To(BeTrue())
-			Expect(len(cids[0].Receipts)).To(Equal(3))
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, mocks.MockCIDWrapper.Receipts[0].CID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, mocks.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, mocks.MockCIDWrapper.Receipts[2].CID)).To(BeTrue())
-			Expect(len(cids[0].StateNodes)).To(Equal(2))
-			for _, stateNode := range cids[0].StateNodes {
-				if stateNode.CID == mocks.State1CID.String() {
-					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
-					Expect(stateNode.NodeType).To(Equal(2))
-					Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
-				}
-				if stateNode.CID == mocks.State2CID.String() {
-					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
-					Expect(stateNode.NodeType).To(Equal(2))
-					Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
-				}
-			}
-			Expect(len(cids[0].StorageNodes)).To(Equal(1))
-			expectedStorageNodeCIDs := mocks.MockCIDWrapper.StorageNodes
-			expectedStorageNodeCIDs[0].ID = cids[0].StorageNodes[0].ID
-			expectedStorageNodeCIDs[0].StateID = cids[0].StorageNodes[0].StateID
-			Expect(cids[0].StorageNodes).To(Equal(expectedStorageNodeCIDs))
-		})
-
-		It("Applies filters from the provided config.Subscription", func() {
-			cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids1)).To(Equal(1))
-			Expect(cids1[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
-			Expect(cids1[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids1[0].Transactions)).To(Equal(0))
-			Expect(len(cids1[0].StateNodes)).To(Equal(0))
-			Expect(len(cids1[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids1[0].Receipts)).To(Equal(1))
-			expectedReceiptCID := mocks.MockCIDWrapper.Receipts[0]
-			expectedReceiptCID.ID = cids1[0].Receipts[0].ID
-			expectedReceiptCID.TxID = cids1[0].Receipts[0].TxID
-			Expect(cids1[0].Receipts[0]).To(Equal(expectedReceiptCID))
-
-			cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids2)).To(Equal(1))
-			Expect(cids2[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
-			Expect(cids2[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids2[0].Transactions)).To(Equal(0))
-			Expect(len(cids2[0].StateNodes)).To(Equal(0))
-			Expect(len(cids2[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids2[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[0]
-			expectedReceiptCID.ID = cids2[0].Receipts[0].ID
-			expectedReceiptCID.TxID = cids2[0].Receipts[0].TxID
-			Expect(cids2[0].Receipts[0]).To(Equal(expectedReceiptCID))
-
-			cids3, empty, err := retriever.Retrieve(rctTopicsAndAddressFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids3)).To(Equal(1))
-			Expect(cids3[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
-			Expect(cids3[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids3[0].Transactions)).To(Equal(0))
-			Expect(len(cids3[0].StateNodes)).To(Equal(0))
-			Expect(len(cids3[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids3[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[0]
-			expectedReceiptCID.ID = cids3[0].Receipts[0].ID
-			expectedReceiptCID.TxID = cids3[0].Receipts[0].TxID
-			Expect(cids3[0].Receipts[0]).To(Equal(expectedReceiptCID))
-
-			cids4, empty, err := retriever.Retrieve(rctAddressesAndTopicFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids4)).To(Equal(1))
-			Expect(cids4[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
-			Expect(cids4[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids4[0].Transactions)).To(Equal(0))
-			Expect(len(cids4[0].StateNodes)).To(Equal(0))
-			Expect(len(cids4[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids4[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[1]
-			expectedReceiptCID.ID = cids4[0].Receipts[0].ID
-			expectedReceiptCID.TxID = cids4[0].Receipts[0].TxID
-			Expect(cids4[0].Receipts[0]).To(Equal(expectedReceiptCID))
-
-			cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids5)).To(Equal(1))
-			Expect(cids5[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
-			Expect(cids5[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids5[0].Transactions)).To(Equal(3))
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks.Trx1CID.String())).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks.Trx2CID.String())).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks.Trx3CID.String())).To(BeTrue())
-			Expect(len(cids5[0].StateNodes)).To(Equal(0))
-			Expect(len(cids5[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids5[0].Receipts)).To(Equal(3))
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks.Rct1CID.String())).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks.Rct2CID.String())).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks.Rct3CID.String())).To(BeTrue())
-
-			cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids6)).To(Equal(1))
-			Expect(cids6[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
-			Expect(cids6[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids6[0].Transactions)).To(Equal(1))
-			expectedTxCID := mocks.MockCIDWrapper.Transactions[1]
-			expectedTxCID.ID = cids6[0].Transactions[0].ID
-			expectedTxCID.HeaderID = cids6[0].Transactions[0].HeaderID
-			Expect(cids6[0].Transactions[0]).To(Equal(expectedTxCID))
-			Expect(len(cids6[0].StateNodes)).To(Equal(0))
-			Expect(len(cids6[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids6[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[1]
-			expectedReceiptCID.ID = cids6[0].Receipts[0].ID
-			expectedReceiptCID.TxID = cids6[0].Receipts[0].TxID
-			Expect(cids6[0].Receipts[0]).To(Equal(expectedReceiptCID))
-
-			cids7, empty, err := retriever.Retrieve(stateFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids7)).To(Equal(1))
-			Expect(cids7[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
-			Expect(cids7[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids7[0].Transactions)).To(Equal(0))
-			Expect(len(cids7[0].Receipts)).To(Equal(0))
-			Expect(len(cids7[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids7[0].StateNodes)).To(Equal(1))
-			Expect(cids7[0].StateNodes[0]).To(Equal(eth2.StateNodeModel{
-				ID:       cids7[0].StateNodes[0].ID,
-				HeaderID: cids7[0].StateNodes[0].HeaderID,
-				NodeType: 2,
-				StateKey: common.BytesToHash(mocks.AccountLeafKey).Hex(),
-				CID:      mocks.State2CID.String(),
-				MhKey:    mocks.State2MhKey,
-				Path:     []byte{'\x0c'},
-			}))
-
-			_, empty, err = retriever.Retrieve(rctTopicsAndAddressFilterFail, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).To(BeTrue())
-		})
-	})
-
-	Describe("RetrieveFirstBlockNumber", func() {
-		It("Throws an error if there are no blocks in the database", func() {
-			_, err := retriever.RetrieveFirstBlockNumber()
-			Expect(err).To(HaveOccurred())
-		})
-		It("Gets the number of the first block that has data in the database", func() {
-			err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveFirstBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(1)))
-		})
-
-		It("Gets the number of the first block that has data in the database", func() {
-			payload := mocks.MockConvertedPayload
-			payload.Block = newMockBlock(1010101)
-			err := repo.Publish(payload)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveFirstBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(1010101)))
-		})
-
-		It("Gets the number of the first block that has data in the database", func() {
-			payload1 := mocks.MockConvertedPayload
-			payload1.Block = newMockBlock(1010101)
-			payload2 := payload1
-			payload2.Block = newMockBlock(5)
-			err := repo.Publish(payload1)
-			Expect(err).ToNot(HaveOccurred())
-			err = repo.Publish(payload2)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveFirstBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(5)))
-		})
-	})
-
-	Describe("RetrieveLastBlockNumber", func() {
-		It("Throws an error if there are no blocks in the database", func() {
-			_, err := retriever.RetrieveLastBlockNumber()
-			Expect(err).To(HaveOccurred())
-		})
-		It("Gets the number of the latest block that has data in the database", func() {
-			err := repo.Publish(mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveLastBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(1)))
-		})
-
-		It("Gets the number of the latest block that has data in the database", func() {
-			payload := mocks.MockConvertedPayload
-			payload.Block = newMockBlock(1010101)
-			err := repo.Publish(payload)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveLastBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(1010101)))
-		})
-
-		It("Gets the number of the latest block that has data in the database", func() {
-			payload1 := mocks.MockConvertedPayload
-			payload1.Block = newMockBlock(1010101)
-			payload2 := payload1
-			payload2.Block = newMockBlock(5)
-			err := repo.Publish(payload1)
-			Expect(err).ToNot(HaveOccurred())
-			err = repo.Publish(payload2)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveLastBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(1010101)))
-		})
-	})
-})
-
-func newMockBlock(blockNumber uint64) *types.Block {
-	header := mocks.MockHeader
-	header.Number.SetUint64(blockNumber)
-	return types.NewBlock(&mocks.MockHeader, mocks.MockTransactions, nil, mocks.MockReceipts)
-}

pkg/eth/errors.go

@@ -0,0 +1,17 @@
+package eth
+
+import "fmt"
+
+type RequiresProxyError struct {
+	method string
+}
+
+var _ error = RequiresProxyError{}
+
+func (e RequiresProxyError) SetMethod(method string) {
+	e.method = method
+}
+
+func (e RequiresProxyError) Error() string {
+	return fmt.Sprintf("%s requires a configured proxy geth node", e.method)
+}

pkg/eth/eth_suite_test.go

@@ -17,19 +17,13 @@
 package eth_test
 
 import (
-	"io/ioutil"
 	"testing"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"github.com/sirupsen/logrus"
 )
 
-func TestETHWatcher(t *testing.T) {
+func TestETHSuite(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "eth ipld server eth suite test")
+	RunSpecs(t, "ipld-eth-server/pkg/eth")
 }
-
-var _ = BeforeSuite(func() {
-	logrus.SetOutput(ioutil.Discard)
-})

pkg/eth/filterer.go

@@ -1,323 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"bytes"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/statediff"
-	"github.com/multiformats/go-multihash"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
-)
-
-// Filterer interface for substituing mocks in tests
-type Filterer interface {
-	Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*eth.IPLDs, error)
-}
-
-// ResponseFilterer satisfies the ResponseFilterer interface for ethereum
-type ResponseFilterer struct{}
-
-// NewResponseFilterer creates a new Filterer satisfying the ResponseFilterer interface
-func NewResponseFilterer() *ResponseFilterer {
-	return &ResponseFilterer{}
-}
-
-// Filter is used to filter through eth data to extract and package requested data into a Payload
-func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*eth.IPLDs, error) {
-	if checkRange(filter.Start.Int64(), filter.End.Int64(), payload.Block.Number().Int64()) {
-		response := new(eth.IPLDs)
-		response.TotalDifficulty = payload.TotalDifficulty
-		if err := s.filterHeaders(filter.HeaderFilter, response, payload); err != nil {
-			return nil, err
-		}
-		txHashes, err := s.filterTransactions(filter.TxFilter, response, payload)
-		if err != nil {
-			return nil, err
-		}
-		var filterTxs []common.Hash
-		if filter.ReceiptFilter.MatchTxs {
-			filterTxs = txHashes
-		}
-		if err := s.filerReceipts(filter.ReceiptFilter, response, payload, filterTxs); err != nil {
-			return nil, err
-		}
-		if err := s.filterStateAndStorage(filter.StateFilter, filter.StorageFilter, response, payload); err != nil {
-			return nil, err
-		}
-		response.BlockNumber = payload.Block.Number()
-		return response, nil
-	}
-	return nil, nil
-}
-
-func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *eth.IPLDs, payload eth.ConvertedPayload) error {
-	if !headerFilter.Off {
-		headerRLP, err := rlp.EncodeToBytes(payload.Block.Header())
-		if err != nil {
-			return err
-		}
-		cid, err := ipld.RawdataToCid(ipld.MEthHeader, headerRLP, multihash.KECCAK_256)
-		if err != nil {
-			return err
-		}
-		response.Header = ipfs.BlockModel{
-			Data: headerRLP,
-			CID:  cid.String(),
-		}
-		if headerFilter.Uncles {
-			response.Uncles = make([]ipfs.BlockModel, len(payload.Block.Body().Uncles))
-			for i, uncle := range payload.Block.Body().Uncles {
-				uncleRlp, err := rlp.EncodeToBytes(uncle)
-				if err != nil {
-					return err
-				}
-				cid, err := ipld.RawdataToCid(ipld.MEthHeader, uncleRlp, multihash.KECCAK_256)
-				if err != nil {
-					return err
-				}
-				response.Uncles[i] = ipfs.BlockModel{
-					Data: uncleRlp,
-					CID:  cid.String(),
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func checkRange(start, end, actual int64) bool {
-	if (end <= 0 || end >= actual) && start <= actual {
-		return true
-	}
-	return false
-}
-
-func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *eth.IPLDs, payload eth.ConvertedPayload) ([]common.Hash, error) {
-	var trxHashes []common.Hash
-	if !trxFilter.Off {
-		trxLen := len(payload.Block.Body().Transactions)
-		trxHashes = make([]common.Hash, 0, trxLen)
-		response.Transactions = make([]ipfs.BlockModel, 0, trxLen)
-		for i, trx := range payload.Block.Body().Transactions {
-			// TODO: check if want corresponding receipt and if we do we must include this transaction
-			if checkTransactionAddrs(trxFilter.Src, trxFilter.Dst, payload.TxMetaData[i].Src, payload.TxMetaData[i].Dst) {
-				trxBuffer := new(bytes.Buffer)
-				if err := trx.EncodeRLP(trxBuffer); err != nil {
-					return nil, err
-				}
-				data := trxBuffer.Bytes()
-				cid, err := ipld.RawdataToCid(ipld.MEthTx, data, multihash.KECCAK_256)
-				if err != nil {
-					return nil, err
-				}
-				response.Transactions = append(response.Transactions, ipfs.BlockModel{
-					Data: data,
-					CID:  cid.String(),
-				})
-				trxHashes = append(trxHashes, trx.Hash())
-			}
-		}
-	}
-	return trxHashes, nil
-}
-
-// checkTransactionAddrs returns true if either the transaction src and dst are one of the wanted src and dst addresses
-func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst string) bool {
-	// If we aren't filtering for any addresses, every transaction is a go
-	if len(wantedDst) == 0 && len(wantedSrc) == 0 {
-		return true
-	}
-	for _, src := range wantedSrc {
-		if src == actualSrc {
-			return true
-		}
-	}
-	for _, dst := range wantedDst {
-		if dst == actualDst {
-			return true
-		}
-	}
-	return false
-}
-
-func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *eth.IPLDs, payload eth.ConvertedPayload, trxHashes []common.Hash) error {
-	if !receiptFilter.Off {
-		response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
-		for i, receipt := range payload.Receipts {
-			// topics is always length 4
-			topics := [][]string{payload.ReceiptMetaData[i].Topic0s, payload.ReceiptMetaData[i].Topic1s, payload.ReceiptMetaData[i].Topic2s, payload.ReceiptMetaData[i].Topic3s}
-			if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, payload.ReceiptMetaData[i].LogContracts, trxHashes) {
-				receiptBuffer := new(bytes.Buffer)
-				if err := receipt.EncodeRLP(receiptBuffer); err != nil {
-					return err
-				}
-				data := receiptBuffer.Bytes()
-				cid, err := ipld.RawdataToCid(ipld.MEthTxReceipt, data, multihash.KECCAK_256)
-				if err != nil {
-					return err
-				}
-				response.Receipts = append(response.Receipts, ipfs.BlockModel{
-					Data: data,
-					CID:  cid.String(),
-				})
-			}
-		}
-	}
-	return nil
-}
-
-func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics [][]string, wantedAddresses []string, actualAddresses []string, wantedTrxHashes []common.Hash) bool {
-	// If we aren't filtering for any topics, contracts, or corresponding trxs then all receipts are a go
-	if len(wantedTopics) == 0 && len(wantedAddresses) == 0 && len(wantedTrxHashes) == 0 {
-		return true
-	}
-	// Keep receipts that are from watched txs
-	for _, wantedTrxHash := range wantedTrxHashes {
-		if bytes.Equal(wantedTrxHash.Bytes(), rct.TxHash.Bytes()) {
-			return true
-		}
-	}
-	// If there are no wanted contract addresses, we keep all receipts that match the topic filter
-	if len(wantedAddresses) == 0 {
-		if match := filterMatch(wantedTopics, actualTopics); match == true {
-			return true
-		}
-	}
-	// If there are wanted contract addresses to filter on
-	for _, wantedAddr := range wantedAddresses {
-		// and this is an address of interest
-		for _, actualAddr := range actualAddresses {
-			if wantedAddr == actualAddr {
-				// we keep the receipt if it matches on the topic filter
-				if match := filterMatch(wantedTopics, actualTopics); match == true {
-					return true
-				}
-			}
-		}
-	}
-	return false
-}
-
-// filterMatch returns true if the actualTopics conform to the wantedTopics filter
-func filterMatch(wantedTopics, actualTopics [][]string) bool {
-	// actualTopics should always be length 4, but the members can be nil slices
-	matches := 0
-	for i, actualTopicSet := range actualTopics {
-		if i < len(wantedTopics) && len(wantedTopics[i]) > 0 {
-			// If we have topics in this filter slot, count as a match if one of the topics matches
-			matches += slicesShareString(actualTopicSet, wantedTopics[i])
-		} else {
-			// Filter slot is either empty or doesn't exist => not matching any topics at this slot => counts as a match
-			matches++
-		}
-	}
-	if matches == 4 {
-		return true
-	}
-	return false
-}
-
-// returns 1 if the two slices have a string in common, 0 if they do not
-func slicesShareString(slice1, slice2 []string) int {
-	for _, str1 := range slice1 {
-		for _, str2 := range slice2 {
-			if str1 == str2 {
-				return 1
-			}
-		}
-	}
-	return 0
-}
-
-// filterStateAndStorage filters state and storage nodes into the response according to the provided filters
-func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *eth.IPLDs, payload eth.ConvertedPayload) error {
-	response.StateNodes = make([]eth.StateNode, 0, len(payload.StateNodes))
-	response.StorageNodes = make([]eth.StorageNode, 0)
-	stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses))
-	for i, addr := range stateFilter.Addresses {
-		stateAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
-	}
-	storageAddressFilters := make([]common.Hash, len(storageFilter.Addresses))
-	for i, addr := range storageFilter.Addresses {
-		storageAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
-	}
-	storageKeyFilters := make([]common.Hash, len(storageFilter.StorageKeys))
-	for i, store := range storageFilter.StorageKeys {
-		storageKeyFilters[i] = common.HexToHash(store)
-	}
-	for _, stateNode := range payload.StateNodes {
-		if !stateFilter.Off && checkNodeKeys(stateAddressFilters, stateNode.LeafKey) {
-			if stateNode.Type == statediff.Leaf || stateFilter.IntermediateNodes {
-				cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
-				if err != nil {
-					return err
-				}
-				response.StateNodes = append(response.StateNodes, eth.StateNode{
-					StateLeafKey: stateNode.LeafKey,
-					Path:         stateNode.Path,
-					IPLD: ipfs.BlockModel{
-						Data: stateNode.Value,
-						CID:  cid.String(),
-					},
-					Type: stateNode.Type,
-				})
-			}
-		}
-		if !storageFilter.Off && checkNodeKeys(storageAddressFilters, stateNode.LeafKey) {
-			for _, storageNode := range payload.StorageNodes[common.Bytes2Hex(stateNode.Path)] {
-				if checkNodeKeys(storageKeyFilters, storageNode.LeafKey) {
-					cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
-					if err != nil {
-						return err
-					}
-					response.StorageNodes = append(response.StorageNodes, eth.StorageNode{
-						StateLeafKey:   stateNode.LeafKey,
-						StorageLeafKey: storageNode.LeafKey,
-						IPLD: ipfs.BlockModel{
-							Data: storageNode.Value,
-							CID:  cid.String(),
-						},
-						Type: storageNode.Type,
-						Path: storageNode.Path,
-					})
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
-	// If we aren't filtering for any specific keys, all nodes are a go
-	if len(wantedKeys) == 0 {
-		return true
-	}
-	for _, key := range wantedKeys {
-		if bytes.Equal(key.Bytes(), actualKey.Bytes()) {
-			return true
-		}
-	}
-	return false
-}

pkg/eth/filterer_test.go

@@ -1,200 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	"bytes"
-
-	"github.com/ethereum/go-ethereum/statediff"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-var (
-	filterer *eth.ResponseFilterer
-)
-
-var _ = Describe("Filterer", func() {
-	Describe("FilterResponse", func() {
-		BeforeEach(func() {
-			filterer = eth.NewResponseFilterer()
-		})
-
-		It("Transcribes all the data from the IPLDPayload into the StreamPayload if given an open filter", func() {
-			iplds, err := filterer.Filter(openFilter, mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds).ToNot(BeNil())
-			Expect(iplds.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds.Header).To(Equal(mocks.MockIPLDs.Header))
-			var expectedEmptyUncles []ipfs.BlockModel
-			Expect(iplds.Uncles).To(Equal(expectedEmptyUncles))
-			Expect(len(iplds.Transactions)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(2))).To(BeTrue())
-			Expect(len(iplds.Receipts)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(2))).To(BeTrue())
-			Expect(len(iplds.StateNodes)).To(Equal(2))
-			for _, stateNode := range iplds.StateNodes {
-				Expect(stateNode.Type).To(Equal(statediff.Leaf))
-				if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.AccountLeafKey) {
-					Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
-						Data: mocks.State2IPLD.RawData(),
-						CID:  mocks.State2IPLD.Cid().String(),
-					}))
-				}
-				if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.ContractLeafKey) {
-					Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
-						Data: mocks.State1IPLD.RawData(),
-						CID:  mocks.State1IPLD.Cid().String(),
-					}))
-				}
-			}
-			Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
-		})
-
-		It("Applies filters from the provided config.Subscription", func() {
-			iplds1, err := filterer.Filter(rctAddressFilter, mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds1).ToNot(BeNil())
-			Expect(iplds1.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds1.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds1.Uncles)).To(Equal(0))
-			Expect(len(iplds1.Transactions)).To(Equal(0))
-			Expect(len(iplds1.StorageNodes)).To(Equal(0))
-			Expect(len(iplds1.StateNodes)).To(Equal(0))
-			Expect(len(iplds1.Receipts)).To(Equal(1))
-			Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct1IPLD.RawData(),
-				CID:  mocks.Rct1IPLD.Cid().String(),
-			}))
-
-			iplds2, err := filterer.Filter(rctTopicsFilter, mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds2).ToNot(BeNil())
-			Expect(iplds2.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds2.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds2.Uncles)).To(Equal(0))
-			Expect(len(iplds2.Transactions)).To(Equal(0))
-			Expect(len(iplds2.StorageNodes)).To(Equal(0))
-			Expect(len(iplds2.StateNodes)).To(Equal(0))
-			Expect(len(iplds2.Receipts)).To(Equal(1))
-			Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct1IPLD.RawData(),
-				CID:  mocks.Rct1IPLD.Cid().String(),
-			}))
-
-			iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds3).ToNot(BeNil())
-			Expect(iplds3.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds3.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds3.Uncles)).To(Equal(0))
-			Expect(len(iplds3.Transactions)).To(Equal(0))
-			Expect(len(iplds3.StorageNodes)).To(Equal(0))
-			Expect(len(iplds3.StateNodes)).To(Equal(0))
-			Expect(len(iplds3.Receipts)).To(Equal(1))
-			Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct1IPLD.RawData(),
-				CID:  mocks.Rct1IPLD.Cid().String(),
-			}))
-
-			iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds4).ToNot(BeNil())
-			Expect(iplds4.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds4.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds4.Uncles)).To(Equal(0))
-			Expect(len(iplds4.Transactions)).To(Equal(0))
-			Expect(len(iplds4.StorageNodes)).To(Equal(0))
-			Expect(len(iplds4.StateNodes)).To(Equal(0))
-			Expect(len(iplds4.Receipts)).To(Equal(1))
-			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct2IPLD.RawData(),
-				CID:  mocks.Rct2IPLD.Cid().String(),
-			}))
-
-			iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds5).ToNot(BeNil())
-			Expect(iplds5.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds5.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds5.Uncles)).To(Equal(0))
-			Expect(len(iplds5.Transactions)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(2))).To(BeTrue())
-			Expect(len(iplds5.StorageNodes)).To(Equal(0))
-			Expect(len(iplds5.StateNodes)).To(Equal(0))
-			Expect(len(iplds5.Receipts)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(2))).To(BeTrue())
-
-			iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds6).ToNot(BeNil())
-			Expect(iplds6.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds6.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds6.Uncles)).To(Equal(0))
-			Expect(len(iplds6.Transactions)).To(Equal(1))
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(len(iplds6.StorageNodes)).To(Equal(0))
-			Expect(len(iplds6.StateNodes)).To(Equal(0))
-			Expect(len(iplds6.Receipts)).To(Equal(1))
-			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct2IPLD.RawData(),
-				CID:  mocks.Rct2IPLD.Cid().String(),
-			}))
-
-			iplds7, err := filterer.Filter(stateFilter, mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds7).ToNot(BeNil())
-			Expect(iplds7.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds7.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds7.Uncles)).To(Equal(0))
-			Expect(len(iplds7.Transactions)).To(Equal(0))
-			Expect(len(iplds7.StorageNodes)).To(Equal(0))
-			Expect(len(iplds7.Receipts)).To(Equal(0))
-			Expect(len(iplds7.StateNodes)).To(Equal(1))
-			Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(mocks.AccountLeafKey))
-			Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
-				Data: mocks.State2IPLD.RawData(),
-				CID:  mocks.State2IPLD.Cid().String(),
-			}))
-
-			iplds8, err := filterer.Filter(rctTopicsAndAddressFilterFail, mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds8).ToNot(BeNil())
-			Expect(iplds8.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds8.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds8.Uncles)).To(Equal(0))
-			Expect(len(iplds8.Transactions)).To(Equal(0))
-			Expect(len(iplds8.StorageNodes)).To(Equal(0))
-			Expect(len(iplds8.StateNodes)).To(Equal(0))
-			Expect(len(iplds8.Receipts)).To(Equal(0))
-		})
-	})
-})

pkg/eth/helpers.go

@@ -17,54 +17,10 @@
 package eth
 
 import (
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/statediff"
+	"time"
 )
 
-func ResolveFromNodeType(nodeType statediff.NodeType) int {
-	switch nodeType {
-	case statediff.Branch:
-		return 0
-	case statediff.Extension:
-		return 1
-	case statediff.Leaf:
-		return 2
-	case statediff.Removed:
-		return 3
-	default:
-		return -1
-	}
-}
-
-func ResolveToNodeType(nodeType int) statediff.NodeType {
-	switch nodeType {
-	case 0:
-		return statediff.Branch
-	case 1:
-		return statediff.Extension
-	case 2:
-		return statediff.Leaf
-	case 3:
-		return statediff.Removed
-	default:
-		return statediff.Unknown
-	}
-}
-
-// ChainConfig returns the appropriate ethereum chain config for the provided chain id
-func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
-	switch chainID {
-	case 1:
-		return params.MainnetChainConfig, nil
-	case 3:
-		return params.TestnetChainConfig, nil // Ropsten
-	case 4:
-		return params.RinkebyChainConfig, nil
-	case 5:
-		return params.GoerliChainConfig, nil
-	default:
-		return nil, fmt.Errorf("chain config for chainid %d not available", chainID)
-	}
+// Timestamp in milliseconds
+func makeTimestamp() int64 {
+	return time.Now().UnixNano() / int64(time.Millisecond)
 }

pkg/eth/ipld_fetcher.go

@@ -1,219 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"errors"
-	"fmt"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/jmoiron/sqlx"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-// Fetcher interface for substituting mocks in tests
-type Fetcher interface {
-	Fetch(cids eth.CIDWrapper) (*eth.IPLDs, error)
-}
-
-// IPLDFetcher satisfies the IPLDFetcher interface for ethereum
-// It interfaces directly with PG-IPFS
-type IPLDFetcher struct {
-	db *postgres.DB
-}
-
-// NewIPLDFetcher creates a pointer to a new IPLDFetcher
-func NewIPLDFetcher(db *postgres.DB) *IPLDFetcher {
-	return &IPLDFetcher{
-		db: db,
-	}
-}
-
-// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
-func (f *IPLDFetcher) Fetch(cids eth.CIDWrapper) (*eth.IPLDs, error) {
-	log.Debug("fetching iplds")
-	iplds := new(eth.IPLDs)
-	var ok bool
-	iplds.TotalDifficulty, ok = new(big.Int).SetString(cids.Header.TotalDifficulty, 10)
-	if !ok {
-		return nil, errors.New("eth fetcher: unable to set total difficulty")
-	}
-	iplds.BlockNumber = cids.BlockNumber
-
-	tx, err := f.db.Beginx()
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		if p := recover(); p != nil {
-			shared.Rollback(tx)
-			panic(p)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = tx.Commit()
-		}
-	}()
-
-	iplds.Header, err = f.FetchHeader(tx, cids.Header)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: header fetching error: %s", err.Error())
-	}
-	iplds.Uncles, err = f.FetchUncles(tx, cids.Uncles)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: uncle fetching error: %s", err.Error())
-	}
-	iplds.Transactions, err = f.FetchTrxs(tx, cids.Transactions)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: transaction fetching error: %s", err.Error())
-	}
-	iplds.Receipts, err = f.FetchRcts(tx, cids.Receipts)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: receipt fetching error: %s", err.Error())
-	}
-	iplds.StateNodes, err = f.FetchState(tx, cids.StateNodes)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: state fetching error: %s", err.Error())
-	}
-	iplds.StorageNodes, err = f.FetchStorage(tx, cids.StorageNodes)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: storage fetching error: %s", err.Error())
-	}
-	return iplds, err
-}
-
-// FetchHeaders fetches headers
-func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c eth.HeaderModel) (ipfs.BlockModel, error) {
-	log.Debug("fetching header ipld")
-	headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-	if err != nil {
-		return ipfs.BlockModel{}, err
-	}
-	return ipfs.BlockModel{
-		Data: headerBytes,
-		CID:  c.CID,
-	}, nil
-}
-
-// FetchUncles fetches uncles
-func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []eth.UncleModel) ([]ipfs.BlockModel, error) {
-	log.Debug("fetching uncle iplds")
-	uncleIPLDs := make([]ipfs.BlockModel, len(cids))
-	for i, c := range cids {
-		uncleBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		uncleIPLDs[i] = ipfs.BlockModel{
-			Data: uncleBytes,
-			CID:  c.CID,
-		}
-	}
-	return uncleIPLDs, nil
-}
-
-// FetchTrxs fetches transactions
-func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []eth.TxModel) ([]ipfs.BlockModel, error) {
-	log.Debug("fetching transaction iplds")
-	trxIPLDs := make([]ipfs.BlockModel, len(cids))
-	for i, c := range cids {
-		txBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		trxIPLDs[i] = ipfs.BlockModel{
-			Data: txBytes,
-			CID:  c.CID,
-		}
-	}
-	return trxIPLDs, nil
-}
-
-// FetchRcts fetches receipts
-func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []eth.ReceiptModel) ([]ipfs.BlockModel, error) {
-	log.Debug("fetching receipt iplds")
-	rctIPLDs := make([]ipfs.BlockModel, len(cids))
-	for i, c := range cids {
-		rctBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		rctIPLDs[i] = ipfs.BlockModel{
-			Data: rctBytes,
-			CID:  c.CID,
-		}
-	}
-	return rctIPLDs, nil
-}
-
-// FetchState fetches state nodes
-func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]eth.StateNode, error) {
-	log.Debug("fetching state iplds")
-	stateNodes := make([]eth.StateNode, 0, len(cids))
-	for _, stateNode := range cids {
-		if stateNode.CID == "" {
-			continue
-		}
-		stateBytes, err := shared.FetchIPLDByMhKey(tx, stateNode.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		stateNodes = append(stateNodes, eth.StateNode{
-			IPLD: ipfs.BlockModel{
-				Data: stateBytes,
-				CID:  stateNode.CID,
-			},
-			StateLeafKey: common.HexToHash(stateNode.StateKey),
-			Type:         ResolveToNodeType(stateNode.NodeType),
-			Path:         stateNode.Path,
-		})
-	}
-	return stateNodes, nil
-}
-
-// FetchStorage fetches storage nodes
-func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateKeyModel) ([]eth.StorageNode, error) {
-	log.Debug("fetching storage iplds")
-	storageNodes := make([]eth.StorageNode, 0, len(cids))
-	for _, storageNode := range cids {
-		if storageNode.CID == "" || storageNode.StateKey == "" {
-			continue
-		}
-		storageBytes, err := shared.FetchIPLDByMhKey(tx, storageNode.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		storageNodes = append(storageNodes, eth.StorageNode{
-			IPLD: ipfs.BlockModel{
-				Data: storageBytes,
-				CID:  storageNode.CID,
-			},
-			StateLeafKey:   common.HexToHash(storageNode.StateKey),
-			StorageLeafKey: common.HexToHash(storageNode.StorageKey),
-			Type:           ResolveToNodeType(storageNode.NodeType),
-			Path:           storageNode.Path,
-		})
-	}
-	return storageNodes, nil
-}

pkg/eth/ipld_fetcher_test.go

@@ -1,66 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-var (
-	db            *postgres.DB
-	pubAndIndexer *eth2.IPLDPublisher
-	fetcher       *eth.IPLDFetcher
-)
-
-var _ = Describe("IPLDFetcher", func() {
-	Describe("Fetch", func() {
-		BeforeEach(func() {
-			var err error
-			db, err = shared.SetupDB()
-			Expect(err).ToNot(HaveOccurred())
-			pubAndIndexer = eth2.NewIPLDPublisher(db)
-			err = pubAndIndexer.Publish(mocks.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			fetcher = eth.NewIPLDFetcher(db)
-		})
-		AfterEach(func() {
-			eth.TearDownDB(db)
-		})
-
-		It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {
-			iplds, err := fetcher.Fetch(*mocks.MockCIDWrapper)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds).ToNot(BeNil())
-			Expect(iplds.TotalDifficulty).To(Equal(mocks.MockConvertedPayload.TotalDifficulty))
-			Expect(iplds.BlockNumber).To(Equal(mocks.MockConvertedPayload.Block.Number()))
-			Expect(iplds.Header).To(Equal(mocks.MockIPLDs.Header))
-			Expect(len(iplds.Uncles)).To(Equal(0))
-			Expect(iplds.Transactions).To(Equal(mocks.MockIPLDs.Transactions))
-			Expect(iplds.Receipts).To(Equal(mocks.MockIPLDs.Receipts))
-			Expect(iplds.StateNodes).To(Equal(mocks.MockIPLDs.StateNodes))
-			Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
-		})
-	})
-})

pkg/eth/node_types.go

@@ -0,0 +1,103 @@
+package eth
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/rlp"
+
+	"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
+	"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/triedb"
+)
+
+// NodeType for explicitly setting type of node
+type NodeType string
+
+const (
+	Unknown   NodeType = "Unknown"
+	Branch    NodeType = "Branch"
+	Extension NodeType = "Extension"
+	Leaf      NodeType = "Leaf"
+	Removed   NodeType = "Removed" // used to represent paths which have been emptied
+)
+
+func (n NodeType) Int() int {
+	switch n {
+	case Branch:
+		return 0
+	case Extension:
+		return 1
+	case Leaf:
+		return 2
+	case Removed:
+		return 3
+	default:
+		return -1
+	}
+}
+
+// CheckKeyType checks what type of key we have
+func CheckKeyType(elements []interface{}) (NodeType, error) {
+	if len(elements) > 2 {
+		return Branch, nil
+	}
+	if len(elements) < 2 {
+		return Unknown, fmt.Errorf("node cannot be less than two elements in length")
+	}
+	switch elements[0].([]byte)[0] / 16 {
+	case '\x00':
+		return Extension, nil
+	case '\x01':
+		return Extension, nil
+	case '\x02':
+		return Leaf, nil
+	case '\x03':
+		return Leaf, nil
+	default:
+		return Unknown, fmt.Errorf("unknown hex prefix")
+	}
+}
+
+// StateNode holds the data for a single state diff node
+type StateNode struct {
+	NodeType     NodeType      `json:"nodeType"        gencodec:"required"`
+	Path         []byte        `json:"path"            gencodec:"required"`
+	NodeValue    []byte        `json:"value"           gencodec:"required"`
+	StorageNodes []StorageNode `json:"storage"`
+	LeafKey      []byte        `json:"leafKey"`
+}
+
+// StorageNode holds the data for a single storage diff node
+type StorageNode struct {
+	NodeType  NodeType `json:"nodeType"        gencodec:"required"`
+	Path      []byte   `json:"path"            gencodec:"required"`
+	NodeValue []byte   `json:"value"           gencodec:"required"`
+	LeafKey   []byte   `json:"leafKey"`
+}
+
+func ResolveNode(path []byte, node []byte, trieDB *triedb.Database) (StateNode, []interface{}, error) {
+	var nodeElements []interface{}
+	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return StateNode{}, nil, err
+	}
+	ty, err := CheckKeyType(nodeElements)
+	if err != nil {
+		return StateNode{}, nil, err
+	}
+
+	nodePath := make([]byte, len(path))
+	copy(nodePath, path)
+	return StateNode{
+		NodeType:  ty,
+		Path:      nodePath,
+		NodeValue: node,
+	}, nodeElements, nil
+}
+
+// ResolveNodeIt return the state diff node pointed by the iterator.
+func ResolveNodeIt(it trie.NodeIterator, trieDB *triedb.Database) (StateNode, []interface{}, error) {
+	node, err := it.NodeBlob(), it.Error()
+	if err != nil {
+		return StateNode{}, nil, err
+	}
+	return ResolveNode(it.Path(), node, trieDB)
+}

pkg/eth/retriever.go

@@ -0,0 +1,573 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"bytes"
+	"fmt"
+	"math/big"
+	"strconv"
+
+	"github.com/cerc-io/plugeth-statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+)
+
+// Retriever is used for fetching
+type Retriever struct {
+	db     *sqlx.DB
+	gormDB *gorm.DB
+}
+
+type IPLDModelRecord struct {
+	models.IPLDModel
+}
+
+// TableName overrides the table name used by IPLD
+func (IPLDModelRecord) TableName() string {
+	return "ipld.blocks"
+}
+
+type HeaderCIDRecord struct {
+	CID             string `gorm:"column:cid"`
+	BlockHash       string `gorm:"primaryKey"`
+	BlockNumber     string `gorm:"primaryKey"`
+	ParentHash      string
+	Timestamp       uint64
+	StateRoot       string
+	TotalDifficulty string `gorm:"column:td"`
+	TxRoot          string
+	RctRoot         string `gorm:"column:receipt_root"`
+	UnclesHash      string
+	Bloom           []byte
+
+	// gorm doesn't check if foreign key exists in database.
+	// It is required to eager load relations using preload.
+	TransactionCIDs []TransactionCIDRecord `gorm:"foreignKey:HeaderID,BlockNumber;references:BlockHash,BlockNumber"`
+	IPLD            IPLDModelRecord        `gorm:"foreignKey:CID,BlockNumber;references:Key,BlockNumber"`
+}
+
+// TableName overrides the table name used by HeaderCIDRecord
+func (HeaderCIDRecord) TableName() string {
+	return "eth.header_cids"
+}
+
+type TransactionCIDRecord struct {
+	CID         string `gorm:"column:cid"`
+	TxHash      string `gorm:"primaryKey"`
+	BlockNumber string `gorm:"primaryKey"`
+	HeaderID    string `gorm:"column:header_id"`
+	Index       int64
+	Src         string
+	Dst         string
+	IPLD        IPLDModelRecord `gorm:"foreignKey:CID,BlockNumber;references:Key,BlockNumber"`
+}
+
+type StateAccountRecord struct {
+	Nonce    uint64 `db:"nonce"`
+	Balance  string `db:"balance"`
+	Root     string `db:"storage_root"`
+	CodeHash []byte `db:"code_hash"`
+	Removed  bool   `db:"removed"`
+}
+
+// TableName overrides the table name used by TransactionCIDRecord
+func (TransactionCIDRecord) TableName() string {
+	return "eth.transaction_cids"
+}
+
+// NewRetriever returns a pointer to a new Retriever which supports the Retriever interface
+func NewRetriever(db *sqlx.DB) *Retriever {
+	gormDB, err := gorm.Open(postgres.New(postgres.Config{
+		Conn: db,
+	}), &gorm.Config{})
+
+	if err != nil {
+		log.Error(err)
+		return nil
+	}
+
+	return &Retriever{
+		db:     db,
+		gormDB: gormDB,
+	}
+}
+
+// RetrieveFirstBlockNumber is used to retrieve the first block number in the db
+func (r *Retriever) RetrieveFirstBlockNumber() (int64, error) {
+	var blockNumber int64
+	err := r.db.Get(&blockNumber, "SELECT block_number FROM ipld.blocks ORDER BY block_number ASC LIMIT 1")
+	return blockNumber, err
+}
+
+// RetrieveLastBlockNumber is used to retrieve the latest block number in the db
+func (r *Retriever) RetrieveLastBlockNumber() (int64, error) {
+	var blockNumber int64
+	err := r.db.Get(&blockNumber, "SELECT block_number FROM ipld.blocks ORDER BY block_number DESC LIMIT 1")
+	return blockNumber, err
+}
+
+func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) {
+	for i, topicSet := range topics {
+		if len(topicSet) == 0 {
+			continue
+		}
+
+		if !first {
+			pgStr += " AND"
+		} else {
+			first = false
+		}
+		pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id)
+		args = append(args, pq.Array(topicSet))
+		*id++
+	}
+	return pgStr, args
+}
+
+func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) {
+	if len(rctFilter.LogAddresses) > 0 {
+		pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id)
+		args = append(args, pq.Array(rctFilter.LogAddresses))
+		*id++
+	}
+
+	// Filter on topics if there are any
+	if hasTopics(rctFilter.Topics) {
+		pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
+	}
+
+	return pgStr, args
+}
+
+func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) {
+	rctCond := " AND (receipt_cids.tx_id = ANY ( "
+	logQuery := "SELECT rct_id FROM eth.log_cids WHERE"
+	if len(rctFilter.LogAddresses) > 0 {
+		// Filter on log contract addresses if there are any
+		pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id)
+		args = append(args, pq.Array(rctFilter.LogAddresses))
+		*id++
+
+		// Filter on topics if there are any
+		if hasTopics(rctFilter.Topics) {
+			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
+		}
+
+		pgStr += ")"
+
+		// Filter on txHashes if there are any, and we are matching txs
+		if rctFilter.MatchTxs && len(txHashes) > 0 {
+			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
+			args = append(args, pq.Array(txHashes))
+		}
+		pgStr += ")"
+	} else { // If there are no contract addresses to filter on
+		// Filter on topics if there are any
+		if hasTopics(rctFilter.Topics) {
+			pgStr += rctCond + logQuery
+			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true)
+			pgStr += ")"
+			// Filter on txHashes if there are any, and we are matching txs
+			if rctFilter.MatchTxs && len(txHashes) > 0 {
+				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
+				args = append(args, pq.Array(txHashes))
+			}
+			pgStr += ")"
+		} else if rctFilter.MatchTxs && len(txHashes) > 0 {
+			// If there are no contract addresses or topics to filter on,
+			// Filter on txHashes if there are any, and we are matching txs
+			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id)
+			args = append(args, pq.Array(txHashes))
+		}
+	}
+
+	return pgStr, args
+}
+
+// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided
+// filter parameters.
+func (r *Retriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) {
+	log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String())
+	args := make([]interface{}, 0, 4)
+	id := 1
+	pgStr := RetrieveFilteredGQLLogs
+	args = append(args, blockHash.String())
+	id++
+
+	if blockNumber != nil {
+		pgStr += ` AND receipt_cids.block_number = $2`
+		id++
+		args = append(args, blockNumber.Int64())
+	}
+
+	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
+	pgStr += ` ORDER BY log_cids.block_number, log_cids.index`
+
+	logs := make([]LogResult, 0)
+	err := tx.Select(&logs, pgStr, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	return logs, nil
+}
+
+// RetrieveFilteredLogsForBlock retrieves and returns all the log CIDs for the block that conform to the provided filter parameters.
+func (r *Retriever) RetrieveFilteredLogsForBlock(db *sqlx.DB, rctFilter ReceiptFilter, blockHash *common.Hash) ([]LogResult, error) {
+	return r.retrieveFilteredLogs(db, rctFilter, -1, -1, blockHash)
+}
+
+// RetrieveFilteredLogsForBlockRange retrieves and returns all the log CIDs for the blocks in the range that conform
+// to the provided filter parameters.
+func (r *Retriever) RetrieveFilteredLogsForBlockRange(db *sqlx.DB, rctFilter ReceiptFilter, startBlockNumber int64, stopBlockNumber int64) ([]LogResult, error) {
+	return r.retrieveFilteredLogs(db, rctFilter, startBlockNumber, stopBlockNumber, nil)
+}
+
+// retrieveFilteredLogs retrieves all the log CIDs either for a single block (by hash) or range of blocks (by number) which
+// conform to the provided filter parameters.
+func (r *Retriever) retrieveFilteredLogs(db *sqlx.DB, rctFilter ReceiptFilter, startBlockNumber int64, stopBlockNumber int64, blockHash *common.Hash) ([]LogResult, error) {
+	log.Debug("retrieving log cids for receipt ids")
+	args := make([]interface{}, 0, 4)
+	var pgStr string
+	id := 1
+	if blockHash != nil {
+		pgStr = RetrieveFilteredLogsSingle
+		args = append(args, blockHash.String())
+		id++
+	} else {
+		pgStr = RetrieveFilteredLogsRange
+		args = append(args, startBlockNumber)
+		id++
+		args = append(args, stopBlockNumber)
+		id++
+	}
+
+	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
+	pgStr += ` ORDER BY eth.log_cids.block_number, eth.log_cids.index`
+
+	logs := make([]LogResult, 0)
+	err := db.Select(&logs, pgStr, args...)
+	if err != nil {
+		return nil, err
+	}
+	// decode logs and extract original contract Data
+	for i, log := range logs {
+		var buf []interface{}
+		r := bytes.NewReader(log.LogLeafData)
+		if err := rlp.Decode(r, &buf); err != nil {
+			return nil, err
+		}
+		logs[i].Data = buf[2].([]byte)
+	}
+
+	return logs, nil
+}
+
+func hasTopics(topics [][]string) bool {
+	for _, topicSet := range topics {
+		if len(topicSet) > 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// RetrieveBlockNumberByHash returns the block number for the given block hash
+func (r *Retriever) RetrieveBlockNumberByHash(tx *sqlx.Tx, blockHash common.Hash) (uint64, error) {
+	log.Debug("retrieving block number for block hash ", blockHash.String())
+	pgStr := `SELECT CAST(block_number as TEXT) FROM eth.header_cids WHERE block_hash = $1`
+	var blockNumberStr string
+	if err := tx.Get(&blockNumberStr, pgStr, blockHash.String()); err != nil {
+		return 0, err
+	}
+	return strconv.ParseUint(blockNumberStr, 10, 64)
+}
+
+// RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number
+func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCIDRecord, error) {
+	log.Debug("retrieving header cids and tx cids for block number ", blockNumber)
+
+	var headerCIDs []HeaderCIDRecord
+
+	// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
+	// Will use join for TransactionCIDs once preload for 1:N is supported.
+	err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
+		return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
+	}).Joins("IPLD").Find(&headerCIDs, "header_cids.block_number = ?", blockNumber).Error
+
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return nil, err
+	}
+
+	return headerCIDs, nil
+}
+
+// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number)
+func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) {
+	log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String())
+
+	var headerCIDs []HeaderCIDRecord
+
+	conditions := map[string]interface{}{"block_hash": blockHash.String()}
+	if blockNumber != nil {
+		conditions["header_cids.block_number"] = blockNumber.Int64()
+	}
+
+	// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
+	// Will use join for TransactionCIDs once preload for 1:N is supported.
+	err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
+		return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
+	}).Joins("IPLD").Find(&headerCIDs, conditions).Error
+
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return HeaderCIDRecord{}, err
+	}
+
+	if len(headerCIDs) == 0 {
+		return HeaderCIDRecord{}, errHeaderHashNotFound
+	} else if len(headerCIDs) > 1 {
+		return HeaderCIDRecord{}, errMultipleHeadersForHash
+	}
+
+	return headerCIDs[0], nil
+}
+
+// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number)
+func (r *Retriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) {
+	log.Debug("retrieving tx cid for tx hash ", txHash)
+
+	var txCIDs []TransactionCIDRecord
+
+	var err error
+	if blockNumber != nil {
+		err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error
+	} else {
+		err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error
+	}
+	if err != nil {
+		log.Error("tx retrieval error")
+		return TransactionCIDRecord{}, err
+	}
+
+	if len(txCIDs) == 0 {
+		return TransactionCIDRecord{}, errTxHashNotFound
+	} else if len(txCIDs) > 1 {
+		// a transaction can be part of a only one canonical block
+		return TransactionCIDRecord{}, errTxHashInMultipleBlocks
+	}
+
+	return txCIDs[0], nil
+}
+
+var EmptyNodeValue = make([]byte, common.HashLength)
+
+// RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash
+func (r *Retriever) RetrieveHeaderByHash(hash common.Hash) (string, []byte, error) {
+	headerResult := new(ipldResult)
+	return headerResult.CID, headerResult.Data, r.db.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
+}
+
+// RetrieveHeaderByHash2 returns the cid and rlp bytes for the header corresponding to the provided block hash
+// using a sqlx.Tx
+func (r *Retriever) RetrieveHeaderByHash2(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) {
+	headerResult := new(ipldResult)
+	return headerResult.CID, headerResult.Data, tx.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
+}
+
+// RetrieveUncles returns the cid and rlp bytes for the uncle list corresponding to the provided block hash, number (of non-omner root block)
+func (r *Retriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) (string, []byte, error) {
+	uncleResult := new(ipldResult)
+	if err := tx.Get(uncleResult, RetrieveUnclesPgStr, hash.Hex(), number); err != nil {
+		return "", nil, err
+	}
+	return uncleResult.CID, uncleResult.Data, nil
+}
+
+// RetrieveUnclesByBlockHash returns the cid and rlp bytes for the uncle list corresponding to the provided block hash (of non-omner root block)
+func (r *Retriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) {
+	uncleResult := new(ipldResult)
+	if err := tx.Get(uncleResult, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil {
+		return "", nil, err
+	}
+	return uncleResult.CID, uncleResult.Data, nil
+}
+
+// RetrieveTransactions returns the cids and rlp bytes for the transactions corresponding to the provided block hash, number
+func (r *Retriever) RetrieveTransactions(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) {
+	txResults := make([]ipldResult, 0)
+	if err := tx.Select(&txResults, RetrieveTransactionsPgStr, hash.Hex(), number); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(txResults))
+	txs := make([][]byte, len(txResults))
+	for i, res := range txResults {
+		cids[i] = res.CID
+		txs[i] = res.Data
+	}
+	return cids, txs, nil
+}
+
+// RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash
+func (r *Retriever) RetrieveTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) {
+	txResults := make([]ipldResult, 0)
+	if err := tx.Select(&txResults, RetrieveTransactionsByBlockHashPgStr, hash.Hex()); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(txResults))
+	txs := make([][]byte, len(txResults))
+	for i, res := range txResults {
+		cids[i] = res.CID
+		txs[i] = res.Data
+	}
+	return cids, txs, nil
+}
+
+// DecodeLeafNode decodes the leaf node data
+func DecodeLeafNode(node []byte) ([]byte, error) {
+	var nodeElements []interface{}
+	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return nil, err
+	}
+	ok, err := IsLeaf(nodeElements)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, fmt.Errorf("expected leaf node but found %v", nodeElements)
+	}
+	return nodeElements[1].([]byte), nil
+}
+
+// RetrieveReceipts returns the cids and rlp bytes for the receipts corresponding to the provided block hash, number.
+// cid returned corresponds to the leaf node data which contains the receipt.
+func (r *Retriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, []common.Hash, error) {
+	rctResults := make([]ipldResult, 0)
+	if err := tx.Select(&rctResults, RetrieveReceiptsPgStr, hash.Hex(), number); err != nil {
+		return nil, nil, nil, err
+	}
+	cids := make([]string, len(rctResults))
+	rcts := make([][]byte, len(rctResults))
+	txs := make([]common.Hash, len(rctResults))
+
+	for i, res := range rctResults {
+		cids[i] = res.CID
+		rcts[i] = res.Data
+		txs[i] = common.HexToHash(res.TxHash)
+	}
+
+	return cids, rcts, txs, nil
+}
+
+// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
+// cid returned corresponds to the leaf node data which contains the receipt.
+func (r *Retriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, []common.Hash, error) {
+	rctResults := make([]ipldResult, 0)
+	if err := tx.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
+		return nil, nil, nil, err
+	}
+	cids := make([]string, len(rctResults))
+	rcts := make([][]byte, len(rctResults))
+	txs := make([]common.Hash, len(rctResults))
+
+	for i, res := range rctResults {
+		cids[i] = res.CID
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		rcts[i] = nodeVal
+		txs[i] = common.HexToHash(res.TxHash)
+	}
+
+	return cids, rcts, txs, nil
+}
+
+// RetrieveWithdrawals returns the CIDs and RLP bytes for the withdrawals corresponding to the
+// provided block hash, number.  Returned CIDs correspond to the leaf node data which contains the
+// withdrawal object.
+func (r *Retriever) RetrieveWithdrawals(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) {
+	results := make([]ipldResult, 0)
+	if err := tx.Select(&results, RetrieveWithdrawalsPgStr, hash.Hex(), number); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(results))
+	withdrawals := make([][]byte, len(results))
+
+	for i, res := range results {
+		cids[i] = res.CID
+		withdrawals[i] = res.Data
+	}
+	return cids, withdrawals, nil
+}
+
+// RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
+// TODO: ensure this handles deleted accounts appropriately
+func (r *Retriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (StateAccountRecord, error) {
+	var accountResult StateAccountRecord
+	leafKey := crypto.Keccak256Hash(address.Bytes())
+	if err := r.db.Get(&accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil {
+		return StateAccountRecord{}, err
+	}
+
+	if accountResult.Removed {
+		return StateAccountRecord{}, nil
+	}
+	return accountResult, nil
+}
+
+// RetrieveStorageAtByAddressAndStorageSlotAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage slot, and block hash
+func (r *Retriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address common.Address, key, hash common.Hash) ([]byte, error) {
+	var storageResult nodeInfo
+	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
+	storageHash := crypto.Keccak256Hash(key.Bytes())
+	if err := r.db.Get(&storageResult,
+		RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr,
+		stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil {
+		return nil, err
+	}
+	if storageResult.StateLeafRemoved || storageResult.Removed {
+		return EmptyNodeValue, nil
+	}
+	return storageResult.Value, nil
+}
+
+// RetrieveStorageAndRLP returns the cid and rlp bytes for the storage value corresponding to the
+// provided address, storage slot, and block hash
+func (r *Retriever) RetrieveStorageAndRLP(address common.Address, key, hash common.Hash) (string, []byte, error) {
+	var storageResult nodeInfo
+	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
+	storageHash := crypto.Keccak256Hash(key.Bytes())
+	if err := r.db.Get(&storageResult,
+		RetrieveStorageAndRLPByAddressHashAndLeafKeyAndBlockHashPgStr,
+		stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil {
+		return "", nil, err
+	}
+	if storageResult.StateLeafRemoved || storageResult.Removed {
+		return "", EmptyNodeValue, nil
+	}
+	return storageResult.CID, storageResult.Data, nil
+}

pkg/eth/retriever_test.go

@@ -0,0 +1,182 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"context"
+
+	"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
+)
+
+var _ = Describe("Retriever", func() {
+	var (
+		db          *sqlx.DB
+		diffIndexer interfaces.StateDiffIndexer
+		retriever   *eth.Retriever
+		ctx         = context.Background()
+	)
+	BeforeEach(func() {
+		db = shared.SetupDB()
+		diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash())
+
+		retriever = eth.NewRetriever(db)
+	})
+	AfterEach(func() {
+		shared.TearDownDB(db)
+		db.Close()
+	})
+
+	It("Retrieve", func() {
+		tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+		Expect(err).ToNot(HaveOccurred())
+		defer tx.RollbackOnFailure(err)
+		for _, node := range test_helpers.MockStateNodes {
+			err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		err = tx.Submit()
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("RetrieveFirstBlockNumber", func() {
+		It("Throws an error if there are no blocks in the database", func() {
+			_, err := retriever.RetrieveFirstBlockNumber()
+			Expect(err).To(HaveOccurred())
+		})
+		It("Gets the number of the first block that has data in the database", func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			defer tx.RollbackOnFailure(err)
+
+			err = tx.Submit()
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveFirstBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1)))
+		})
+
+		It("Gets the number of the first block that has data in the database", func() {
+			payload := test_helpers.MockConvertedPayload
+			payload.Block = newMockBlock(1010101)
+			tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			defer tx.RollbackOnFailure(err)
+
+			err = tx.Submit()
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveFirstBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1010101)))
+		})
+
+		It("Gets the number of the first block that has data in the database", func() {
+			payload1 := test_helpers.MockConvertedPayload
+			payload1.Block = newMockBlock(1010101)
+			payload2 := payload1
+			payload2.Block = newMockBlock(5)
+			tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			defer tx.RollbackOnFailure(err)
+			err = tx.Submit()
+			Expect(err).ToNot(HaveOccurred())
+
+			tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			defer tx.RollbackOnFailure(err)
+			err = tx.Submit()
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveFirstBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(5)))
+		})
+	})
+
+	Describe("RetrieveLastBlockNumber", func() {
+		It("Throws an error if there are no blocks in the database", func() {
+			_, err := retriever.RetrieveLastBlockNumber()
+			Expect(err).To(HaveOccurred())
+		})
+		It("Gets the number of the latest block that has data in the database", func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			defer tx.RollbackOnFailure(err)
+			err = tx.Submit()
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveLastBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1)))
+		})
+
+		It("Gets the number of the latest block that has data in the database", func() {
+			payload := test_helpers.MockConvertedPayload
+			payload.Block = newMockBlock(1010101)
+			tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			defer tx.RollbackOnFailure(err)
+
+			err = tx.Submit()
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveLastBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1010101)))
+		})
+
+		It("Gets the number of the latest block that has data in the database", func() {
+			payload1 := test_helpers.MockConvertedPayload
+			payload1.Block = newMockBlock(1010101)
+			payload2 := payload1
+			payload2.Block = newMockBlock(5)
+			tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			defer tx.RollbackOnFailure(err)
+			err = tx.Submit()
+			Expect(err).ToNot(HaveOccurred())
+
+			tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			defer tx.RollbackOnFailure(err)
+			err = tx.Submit()
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveLastBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1010101)))
+		})
+	})
+})
+
+func newMockBlock(blockNumber uint64) *types.Block {
+	header := test_helpers.MockHeader
+	header.Number.SetUint64(blockNumber)
+	return types.NewBlock(&test_helpers.MockHeader, test_helpers.MockTransactions, nil, test_helpers.MockReceipts, trie.NewEmpty(nil))
+}

pkg/eth/sql.go

@@ -0,0 +1,292 @@
+package eth
+
+const (
+	RetrieveHeaderByHashPgStr = `
+SELECT header_cids.cid,
+	blocks.data
+FROM ipld.blocks,
+	eth.header_cids
+WHERE header_cids.block_hash = $1
+	AND header_cids.canonical
+	AND blocks.key = header_cids.cid
+	AND blocks.block_number = header_cids.block_number
+LIMIT 1
+`
+	RetrieveUnclesPgStr = `SELECT uncle_cids.cid, data
+										FROM eth.uncle_cids
+											INNER JOIN eth.header_cids ON (
+												uncle_cids.header_id = header_cids.block_hash
+												AND uncle_cids.block_number = header_cids.block_number
+											)
+											INNER JOIN ipld.blocks ON (
+												uncle_cids.cid = blocks.key
+												AND uncle_cids.block_number = blocks.block_number
+											)
+										WHERE header_cids.block_hash = $1
+										AND header_cids.block_number = $2
+										ORDER BY uncle_cids.parent_hash
+										LIMIT 1`
+	RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data
+										FROM eth.uncle_cids
+											INNER JOIN eth.header_cids ON (
+												uncle_cids.header_id = header_cids.block_hash
+												AND uncle_cids.block_number = header_cids.block_number
+											)
+											INNER JOIN ipld.blocks ON (
+												uncle_cids.cid = blocks.key
+												AND uncle_cids.block_number = blocks.block_number
+											)
+										WHERE header_cids.block_hash = $1
+										ORDER BY uncle_cids.parent_hash
+										LIMIT 1`
+	RetrieveTransactionsPgStr = `
+SELECT transaction_cids.cid,
+	blocks.data
+FROM eth.transaction_cids,
+	eth.header_cids,
+	ipld.blocks
+WHERE header_cids.block_hash = $1
+    AND header_cids.block_number = $2
+	AND header_cids.canonical
+	AND transaction_cids.block_number = header_cids.block_number
+	AND transaction_cids.header_id = header_cids.block_hash
+	AND blocks.block_number = header_cids.block_number
+	AND blocks.key = transaction_cids.cid
+ORDER BY eth.transaction_cids.index ASC
+`
+	RetrieveTransactionsByBlockHashPgStr = `
+SELECT transaction_cids.cid,
+	blocks.data
+FROM eth.transaction_cids,
+	eth.header_cids,
+	ipld.blocks
+WHERE header_cids.block_hash = $1
+	AND header_cids.canonical
+	AND transaction_cids.block_number = header_cids.block_number
+	AND transaction_cids.header_id = header_cids.block_hash
+	AND blocks.block_number = header_cids.block_number
+	AND blocks.key = transaction_cids.cid
+ORDER BY eth.transaction_cids.index ASC
+`
+	RetrieveReceiptsPgStr = `
+SELECT receipt_cids.cid,
+	blocks.data,
+	eth.transaction_cids.tx_hash
+FROM eth.receipt_cids,
+	eth.transaction_cids,
+	eth.header_cids,
+	ipld.blocks
+WHERE header_cids.block_hash = $1
+	AND header_cids.block_number = $2
+	AND header_cids.canonical
+	AND receipt_cids.block_number = header_cids.block_number
+	AND receipt_cids.header_id = header_cids.block_hash
+	AND receipt_cids.TX_ID = transaction_cids.TX_HASH
+	AND transaction_cids.block_number = header_cids.block_number
+	AND transaction_cids.header_id = header_cids.block_hash
+	AND blocks.block_number = header_cids.block_number
+	AND blocks.key = receipt_cids.cid
+ORDER BY eth.transaction_cids.index ASC
+`
+	RetrieveReceiptsByBlockHashPgStr = `
+SELECT receipt_cids.cid,
+	blocks.data,
+	eth.transaction_cids.tx_hash
+FROM eth.receipt_cids,
+	eth.transaction_cids,
+	eth.header_cids,
+	ipld.blocks
+WHERE header_cids.block_hash = $1
+	AND header_cids.canonical
+	AND receipt_cids.block_number = header_cids.block_number
+	AND receipt_cids.header_id = header_cids.block_hash
+	AND receipt_cids.TX_ID = transaction_cids.TX_HASH
+	AND transaction_cids.block_number = header_cids.block_number
+	AND transaction_cids.header_id = header_cids.block_hash
+	AND blocks.block_number = header_cids.block_number
+	AND blocks.key = receipt_cids.cid
+ORDER BY eth.transaction_cids.index ASC
+`
+	RetrieveWithdrawalsPgStr = `
+SELECT withdrawal_cids.cid,
+	   blocks.data
+  FROM eth.withdrawal_cids
+       JOIN eth.header_cids
+           ON header_cids.block_hash = $1
+           AND header_cids.block_number = $2
+           AND header_cids.canonical
+           AND withdrawal_cids.block_number = header_cids.block_number
+           AND withdrawal_cids.header_id = header_cids.block_hash
+       JOIN ipld.blocks
+           ON blocks.block_number = header_cids.block_number
+           AND blocks.key = withdrawal_cids.cid
+ ORDER BY eth.withdrawal_cids.index ASC`
+
+	RetrieveAccountByLeafKeyAndBlockHashPgStr = `
+SELECT state_cids.nonce,
+	state_cids.balance,
+	state_cids.storage_root,
+	state_cids.code_hash,
+	state_cids.removed
+FROM eth.state_cids,
+	eth.header_cids
+WHERE state_cids.state_leaf_key = $1
+	AND state_cids.block_number <=
+		(SELECT block_number
+			FROM eth.header_cids
+			WHERE block_hash = $2
+			LIMIT 1)
+	AND header_cids.canonical
+	AND state_cids.header_id = header_cids.block_hash
+	AND state_cids.block_number = header_cids.block_number
+ORDER BY state_cids.block_number DESC
+LIMIT 1
+`
+	RetrieveFilteredGQLLogs = `SELECT CAST(eth.log_cids.block_number as TEXT), eth.log_cids.header_id as block_hash,
+			eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address,
+			eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
+			data, eth.receipt_cids.cid AS rct_cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash
+				FROM eth.log_cids, eth.receipt_cids, ipld.blocks
+				WHERE eth.log_cids.rct_id = receipt_cids.tx_id
+				AND eth.log_cids.header_id = receipt_cids.header_id
+				AND eth.log_cids.block_number = receipt_cids.block_number
+				AND log_cids.cid = blocks.key
+				AND log_cids.block_number = blocks.block_number
+				AND receipt_cids.header_id = $1`
+	RetrieveFilteredLogsRange = `SELECT CAST(eth.log_cids.block_number as TEXT), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id,
+			eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
+			eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index,
+			ipld.blocks.data, eth.receipt_cids.cid AS rct_cid, eth.receipt_cids.post_status, log_cids.header_id AS block_hash
+							FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, ipld.blocks
+							WHERE eth.log_cids.block_number >= $1 AND eth.log_cids.block_number <= $2
+							AND eth.log_cids.header_id IN (SELECT block_hash from eth.header_cids where eth.header_cids.block_number >= $1 AND eth.header_cids.block_number <= $2 and eth.header_cids.canonical)
+							AND eth.transaction_cids.block_number = eth.log_cids.block_number
+							AND eth.transaction_cids.header_id = eth.log_cids.header_id
+							AND eth.receipt_cids.block_number = eth.log_cids.block_number
+							AND eth.receipt_cids.header_id = eth.log_cids.header_id
+							AND eth.receipt_cids.tx_id = eth.log_cids.rct_id
+							AND eth.receipt_cids.tx_id = eth.transaction_cids.tx_hash
+							AND ipld.blocks.block_number = eth.log_cids.block_number
+							AND ipld.blocks.key = eth.log_cids.cid`
+
+	RetrieveFilteredLogsSingle = `SELECT CAST(eth.log_cids.block_number as TEXT), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id,
+			eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
+			eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index,
+			ipld.blocks.data, eth.receipt_cids.cid AS rct_cid, eth.receipt_cids.post_status, log_cids.header_id AS block_hash
+							FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, ipld.blocks
+							WHERE eth.log_cids.header_id = $1
+							AND eth.transaction_cids.block_number = eth.log_cids.block_number
+							AND eth.transaction_cids.header_id = eth.log_cids.header_id
+							AND eth.receipt_cids.block_number = eth.log_cids.block_number
+							AND eth.receipt_cids.header_id = eth.log_cids.header_id
+							AND eth.receipt_cids.tx_id = eth.log_cids.rct_id
+							AND eth.receipt_cids.tx_id = eth.transaction_cids.tx_hash
+							AND ipld.blocks.block_number = eth.log_cids.block_number
+							AND ipld.blocks.key = eth.log_cids.cid`
+	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `
+SELECT storage_cids.cid,
+       storage_cids.val,
+       storage_cids.block_number,
+       storage_cids.removed,
+       was_state_leaf_removed_by_number(storage_cids.state_leaf_key, storage_cids.block_number) AS state_leaf_removed
+FROM eth.storage_cids, eth.header_cids
+WHERE header_cids.block_number <= (SELECT block_number from eth.header_cids where block_hash = $3 LIMIT 1)
+    AND header_cids.canonical
+    AND storage_cids.block_number = header_cids.block_number
+    AND storage_cids.header_id = header_cids.block_hash
+    AND storage_cids.storage_leaf_key = $2
+    AND storage_cids.state_leaf_key = $1
+ORDER BY storage_cids.block_number DESC LIMIT 1
+`
+	RetrieveStorageAndRLPByAddressHashAndLeafKeyAndBlockHashPgStr = `
+SELECT storage_cids.cid,
+       storage_cids.val,
+       storage_cids.block_number,
+       storage_cids.removed,
+       was_state_leaf_removed_by_number(storage_cids.state_leaf_key, storage_cids.block_number) AS state_leaf_removed,
+       blocks.data
+FROM eth.storage_cids, eth.header_cids, ipld.blocks
+WHERE header_cids.block_number <= (SELECT block_number from eth.header_cids where block_hash = $3 LIMIT 1)
+    AND header_cids.canonical
+    AND storage_cids.block_number = header_cids.block_number
+    AND storage_cids.header_id = header_cids.block_hash
+    AND storage_cids.storage_leaf_key = $2
+    AND storage_cids.state_leaf_key = $1
+    AND blocks.key = storage_cids.cid
+    AND blocks.block_number = storage_cids.block_number
+ORDER BY storage_cids.block_number DESC LIMIT 1
+`
+	RetrieveCanonicalBlockHashByNumber = `SELECT block_hash FROM eth.header_cids WHERE block_number = $1 AND canonical`
+	RetrieveCanonicalHeaderByNumber    = `
+SELECT header_cids.cid,
+	blocks.data
+FROM ipld.blocks,
+	eth.header_cids
+WHERE header_cids.block_number = $1
+	AND header_cids.canonical
+	AND blocks.key = header_cids.cid
+	AND blocks.block_number = header_cids.block_number
+LIMIT 1
+`
+	RetrieveCanonicalHeaderAndHashByNumber = `
+SELECT blocks.data,
+       header_cids.block_hash
+FROM ipld.blocks,
+	eth.header_cids
+WHERE header_cids.block_number = $1
+	AND header_cids.canonical
+	AND blocks.key = header_cids.cid
+	AND blocks.block_number = header_cids.block_number
+LIMIT 1
+`
+	RetrieveTD = `SELECT CAST(td as TEXT) FROM eth.header_cids
+			WHERE header_cids.block_hash = $1`
+	RetrieveRPCTransaction = `
+SELECT blocks.data,
+	transaction_cids.header_id,
+	transaction_cids.block_number,
+	transaction_cids.index
+FROM eth.transaction_cids,
+    ipld.blocks,
+	eth.header_cids
+WHERE transaction_cids.TX_HASH = $1
+	AND header_cids.block_hash = transaction_cids.header_id
+	AND header_cids.block_number = transaction_cids.block_number
+	AND header_cids.canonical
+        AND blocks.key = transaction_cids.cid
+	AND blocks.block_number = transaction_cids.block_number
+`
+	RetrieveCodeHashByLeafKeyAndBlockHash = `
+SELECT state_cids.code_hash
+FROM eth.state_cids,
+	eth.header_cids
+WHERE
+state_cids.state_leaf_key = $1
+	AND state_cids.block_number <=
+		(SELECT block_number
+			FROM eth.header_cids
+			WHERE block_hash = $2
+			LIMIT 1)
+	AND header_cids.canonical
+	AND state_cids.header_id = header_cids.block_hash
+	AND state_cids.block_number = header_cids.block_number
+ORDER BY state_cids.block_number DESC
+LIMIT 1
+`
+	RetrieveCodeByKey = `SELECT data FROM ipld.blocks WHERE key = $1`
+)
+
+type ipldResult struct {
+	CID    string `db:"cid"`
+	Data   []byte `db:"data"`
+	TxHash string `db:"tx_hash"`
+}
+
+type nodeInfo struct {
+	CID              string `db:"cid"`
+	Value            []byte `db:"val"`
+	BlockNumber      string `db:"block_number"`
+	Data             []byte `db:"data"`
+	Removed          bool   `db:"removed"`
+	StateLeafRemoved bool   `db:"state_leaf_removed"`
+}

pkg/eth/state_test/eth_suite_test.go

@@ -0,0 +1,29 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_state_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestETHSuite(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ipld-eth-server/pkg/eth/state_test")
+}

pkg/eth/state_test/helper_test.go

@@ -0,0 +1,71 @@
+package eth_state_test
+
+import (
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/format"
+	"github.com/onsi/gomega/types"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+)
+
+func CheckGetSliceResponse(sliceResponse eth.GetSliceResponse, expectedResponse eth.GetSliceResponse) {
+	Expect(sliceResponse.SliceID).To(Equal(expectedResponse.SliceID))
+	Expect(sliceResponse.TrieNodes).To(Equal(expectedResponse.TrieNodes))
+	Expect(sliceResponse.Leaves).To(Equal(expectedResponse.Leaves))
+	Expect(sliceResponse.MetaData.NodeStats).To(Equal(expectedResponse.MetaData.NodeStats))
+}
+
+// EqualBigInt compares a hexutil.Big for equality with a big.Int or hexutil.Big value.
+// It is an error for both actual and expected to be nil.  Use BeNil() instead.
+func EqualBigInt(expected *big.Int) types.GomegaMatcher {
+	return &BigIntEqualMatcher{
+		Expected: expected,
+	}
+}
+
+// EqualBigHex compares a hexutil.Big for equality with a big.Int or hexutil.Big value.
+// It is an error for both actual and expected to be nil.  Use BeNil() instead.
+func EqualBigHex(expected *hexutil.Big) types.GomegaMatcher {
+	return &BigIntEqualMatcher{
+		Expected: expected.ToInt(),
+	}
+}
+
+type BigIntEqualMatcher struct {
+	Expected *big.Int
+}
+
+func (matcher *BigIntEqualMatcher) Match(actual interface{}) (success bool, err error) {
+	if actual == nil && matcher.Expected == nil {
+		return false, fmt.Errorf("Refusing to compare <nil> to <nil>.\nBe explicit and use BeNil() instead.  This is to avoid mistakes where both sides of an assertion are erroneously uninitialized.")
+	}
+
+	var asInt *big.Int
+	switch casted := actual.(type) {
+	case *big.Int:
+		asInt = casted
+	case *hexutil.Big:
+		asInt = (*big.Int)(casted)
+	default:
+		return false, fmt.Errorf("BigIntEqualMatcher expects a hexutil.Big or big.Int. Got:\n%s", format.Object(actual, 1))
+	}
+	return matcher.Expected.Cmp(asInt) == 0, nil
+}
+
+func (matcher *BigIntEqualMatcher) FailureMessage(actual interface{}) (message string) {
+	actualString, actualOK := actual.(string)
+	expectedString := matcher.Expected.String()
+	if actualOK {
+		return format.MessageWithDiff(actualString, "to equal", expectedString)
+	}
+
+	return format.Message(actual, "to equal", matcher.Expected)
+}
+
+func (matcher *BigIntEqualMatcher) NegatedFailureMessage(actual interface{}) (message string) {
+	return format.Message(actual, "not to equal", matcher.Expected)
+}

pkg/eth/state_test/state_test.go

@@ -0,0 +1,836 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_state_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math/big"
+	"os"
+
+	"github.com/cerc-io/plugeth-statediff"
+	"github.com/cerc-io/plugeth-statediff/adapt"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
+)
+
+var (
+	parsedABI     abi.ABI
+	randomAddress = common.HexToAddress("0x9F4203bd7a11aCB94882050E6f1C3ab14BBaD3D9")
+	randomHash    = crypto.Keccak256Hash(randomAddress.Bytes())
+	number        = rpc.BlockNumber(test_helpers.BlockNumber1)
+
+	block1StateRoot    = common.HexToHash("0xa1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21")
+	rootDataHashBlock1 = "a1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21"
+	rootDataBlock1     = "f871a0577652b625b77bdb5bf77bc43f3125cad7464d679d1575565277d3611b8053e780808080a0fe889f10e5db8f2c2bf355928152a17f6e3bb99a9241ac6d84c77e6264509c798080808080808080a011db0cda34a896dabeb6839bb06a38f49514cfa486435984eb013b7df9ee85c58080"
+
+	block5StateRoot    = common.HexToHash("0x572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43")
+	rootDataHashBlock5 = "572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43"
+	rootDataBlock5     = "f8b1a0408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f280808080a0b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91a0180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e48080808080a0422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac995180a02d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba358080"
+
+	account1DataHash     = "180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e4"
+	account1Data         = "f869a03114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45b846f8440180a04bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0a0ba79854f3dbf6505fdbb085888e25fae8fa97288c5ce8fcd39aa589290d9a659"
+	account1StateLeafKey = "0x6114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"
+	account1Code         = "608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032"
+	account2DataHash     = "2d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba35"
+	account2Data         = "f871a03926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2b84ef84c02881bc16d674ec82710a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+	account3DataHash     = "408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f2"
+	account3Data         = "f86da030bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2ab84af848058405f5b608a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+	account4DataHash     = "422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac9951"
+	account4Data         = "f871a03957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45b84ef84c80883782dace9d9003e8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+	account5DataHash     = "b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91"
+	account5Data         = "f871a03380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312ab84ef84c80883782dace9d900000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+
+	contractStorageRootBlock5 = common.HexToHash("0x4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0")
+	storageRootDataHashBlock5 = "4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0"
+	storageRootDataBlock5     = "f838a120290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7"
+
+	contractStorageRootBlock4 = common.HexToHash("0x64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68")
+	storageRootDataHashBlock4 = "64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68"
+	storageRootDataBlock4     = "f8518080a08e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d558080808080808080a07b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af8080808080"
+
+	storageNode1DataHash = "7b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af"
+	storageNode1Data     = "e2a0310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf609"
+	storageNode2DataHash = "8e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d55"
+	storageNode2Data     = "f7a0390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7"
+)
+
+func init() {
+	// load abi
+	abiBytes, err := os.ReadFile("../test_helpers/abi.json")
+	if err != nil {
+		panic(err)
+	}
+	parsedABI, err = abi.JSON(bytes.NewReader(abiBytes))
+	if err != nil {
+		panic(err)
+	}
+}
+
+const chainLength = 5
+
+var (
+	blocks                  []*types.Block
+	receipts                []types.Receipts
+	chain                   *core.BlockChain
+	db                      *sqlx.DB
+	api                     *eth.PublicEthAPI
+	backend                 *eth.Backend
+	chainConfig             = &*params.TestChainConfig
+	mockTD                  = big.NewInt(1337)
+	expectedCanonicalHeader map[string]interface{}
+	ctx                     = context.Background()
+)
+
+var _ = BeforeSuite(func() {
+	chainConfig.LondonBlock = big.NewInt(100)
+
+	// db and type initializations
+	var err error
+	db = shared.SetupDB()
+
+	backend, err = eth.NewEthBackend(db, &eth.Config{
+		ChainConfig: chainConfig,
+		VMConfig:    vm.Config{},
+		RPCGasCap:   big.NewInt(10000000000), // Max gas capacity for a rpc call.
+		GroupCacheConfig: &shared.GroupCacheConfig{
+			StateDB: shared.GroupConfig{
+				Name:                   "eth_state_test",
+				CacheSizeInMB:          8,
+				CacheExpiryInMins:      60,
+				LogStatsIntervalInSecs: 0,
+			},
+		},
+	})
+	Expect(err).ToNot(HaveOccurred())
+	api, _ = eth.NewPublicEthAPI(backend, nil, eth.APIConfig{StateDiffTimeout: shared.DefaultStateDiffTimeout})
+
+	// make the test blockchain (and state)
+	blocks, receipts, chain = test_helpers.MakeChain(chainLength, test_helpers.Genesis, test_helpers.TestChainGen, chainConfig)
+
+	transformer := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
+	params := statediff.Params{}
+	canonicalHeader := blocks[1].Header()
+	expectedCanonicalHeader = map[string]interface{}{
+		"number":           (*hexutil.Big)(canonicalHeader.Number),
+		"hash":             canonicalHeader.Hash(),
+		"parentHash":       canonicalHeader.ParentHash,
+		"nonce":            canonicalHeader.Nonce,
+		"mixHash":          canonicalHeader.MixDigest,
+		"sha3Uncles":       canonicalHeader.UncleHash,
+		"logsBloom":        canonicalHeader.Bloom,
+		"stateRoot":        canonicalHeader.Root,
+		"miner":            canonicalHeader.Coinbase,
+		"difficulty":       (*hexutil.Big)(canonicalHeader.Difficulty),
+		"extraData":        hexutil.Bytes([]byte{}),
+		"gasLimit":         hexutil.Uint64(canonicalHeader.GasLimit),
+		"gasUsed":          hexutil.Uint64(canonicalHeader.GasUsed),
+		"timestamp":        hexutil.Uint64(canonicalHeader.Time),
+		"transactionsRoot": canonicalHeader.TxHash,
+		"receiptsRoot":     canonicalHeader.ReceiptHash,
+		"totalDifficulty":  (*hexutil.Big)(mockTD),
+	}
+
+	// Insert some non-canonical data into the database so that we test our ability to discern canonicity
+	// NOTE: Nan-canonical blocks must come first, because the statediffer will assume the most recent block it is
+	// provided at a certain height is canonical.  This is true inside geth, but not necessarily inside this test.
+	indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
+
+	tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+	Expect(err).ToNot(HaveOccurred())
+	defer tx.RollbackOnFailure(err)
+
+	err = tx.Submit()
+	Expect(err).ToNot(HaveOccurred())
+
+	// The non-canonical header has a child
+	tx, err = indexAndPublisher.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
+	Expect(err).ToNot(HaveOccurred())
+	defer tx.RollbackOnFailure(err)
+
+	err = tx.Submit()
+	Expect(err).ToNot(HaveOccurred())
+
+	// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
+	builder := statediff.NewBuilder(adapt.GethStateView(chain.StateCache()))
+	for i, block := range blocks {
+		var args statediff.Args
+		var rcts types.Receipts
+		if i == 0 {
+			args = statediff.Args{
+				OldStateRoot: common.Hash{},
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+		} else {
+			args = statediff.Args{
+				OldStateRoot: blocks[i-1].Root(),
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+			rcts = receipts[i-1]
+		}
+		diff, err := builder.BuildStateDiffObject(args, params)
+		Expect(err).ToNot(HaveOccurred())
+		tx, err := transformer.PushBlock(block, rcts, mockTD)
+		Expect(err).ToNot(HaveOccurred())
+		defer tx.RollbackOnFailure(err)
+
+		for _, node := range diff.Nodes {
+			err = transformer.PushStateNode(tx, node, block.Hash().String())
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		for _, ipld := range diff.IPLDs {
+			err = transformer.PushIPLD(tx, ipld)
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		err = tx.Submit()
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+})
+
+var _ = AfterSuite(func() {
+	shared.TearDownDB(db)
+	chain.Stop()
+})
+
+var _ = Describe("eth state reading tests", func() {
+
+	Describe("eth_call", func() {
+		It("Applies call args (tx data) on top of state, returning the result (e.g. a Getter method call)", func() {
+			data, err := parsedABI.Pack("data")
+			Expect(err).ToNot(HaveOccurred())
+			bdata := hexutil.Bytes(data)
+			callArgs := eth.CallArgs{
+				To:   &test_helpers.ContractAddr,
+				Data: &bdata,
+			}
+
+			// Before contract deployment, returns nil
+			res, err := api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(0), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(BeNil())
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(1), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(BeNil())
+
+			// After deployment
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(2), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes := hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(3), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(4), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(5), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000"))
+			Expect(res).To(Equal(expectedRes))
+		})
+	})
+
+	Describe("eth_getBalance", func() {
+		var (
+			expectedContractBalance   = (*hexutil.Big)(common.Big0)
+			expectedBankBalanceBlock0 = (*hexutil.Big)(test_helpers.TestBankFunds)
+
+			expectedAcct1BalanceBlock1 = (*hexutil.Big)(big.NewInt(10000))
+			expectedBankBalanceBlock1  = (*hexutil.Big)(new(big.Int).Sub(test_helpers.TestBankFunds, big.NewInt(10000)))
+
+			expectedAcct2BalanceBlock2 = (*hexutil.Big)(big.NewInt(1000))
+			expectedBankBalanceBlock2  = (*hexutil.Big)(new(big.Int).Sub(expectedBankBalanceBlock1.ToInt(), big.NewInt(1000)))
+
+			expectedAcct2BalanceBlock3 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock2.ToInt(), test_helpers.MiningReward))
+
+			expectedAcct2BalanceBlock4 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock3.ToInt(), test_helpers.MiningReward))
+
+			expectedAcct1BalanceBlock5 = (*hexutil.Big)(new(big.Int).Add(expectedAcct1BalanceBlock1.ToInt(), test_helpers.MiningReward))
+		)
+
+		It("Retrieves account balance by block number", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigInt((common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigInt((common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct2BalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct2BalanceBlock3))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct1BalanceBlock5))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock2))
+		})
+		It("Retrieves account balance by block hash", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigInt((common.Big0)))
+
+			_, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigInt((common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(bal).To(EqualBigHex(expectedAcct2BalanceBlock2))
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct2BalanceBlock3))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct1BalanceBlock5))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigHex(expectedBankBalanceBlock2))
+		})
+		It("Returns 0 if account balance not found by block number", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigInt(common.Big0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigInt(common.Big0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigInt(common.Big0))
+		})
+		It("Returns 0 if account balance not found by block hash", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigInt(common.Big0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigInt(common.Big0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(EqualBigInt(common.Big0))
+		})
+	})
+
+	Describe("eth_getCode", func() {
+		It("Retrieves the code for the provided contract address at the block with the provided number", func() {
+			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+
+			code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+		})
+		It("Retrieves the code for the provided contract address at the block with the provided hash", func() {
+			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+
+			code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+		})
+		It("Returns `nil` for an account it cannot find the code for", func() {
+			code, err := api.GetCode(ctx, randomAddress, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(BeEmpty())
+		})
+		It("Returns `nil`  for a contract that doesn't exist at this height", func() {
+			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(BeEmpty())
+		})
+	})
+
+	Describe("eth_getStorageAt", func() {
+		It("Returns empty slice if it tries to access a contract which does not exist", func() {
+			storage, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+
+			storage, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+		})
+		It("Returns empty slice if it tries to access a contract slot which does not exist", func() {
+			storage, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, randomHash.Hex(), rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+		})
+		It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() {
+			// After deployment
+			val, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes := hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+		})
+		It("Throws an error for a non-existing block hash", func() {
+			_, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithHash(randomHash, true))
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError("header for hash not found"))
+		})
+		It("Throws an error for a non-existing block number", func() {
+			_, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(chainLength+1))
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError("header not found"))
+		})
+	})
+
+	Describe("eth_getHeaderByNumber", func() {
+		It("Finds the canonical header based on the header's weight relative to others at the provided height", func() {
+			header, err := api.GetHeaderByNumber(ctx, number)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(header).To(Equal(expectedCanonicalHeader))
+		})
+	})
+
+	Describe("eth_getSlice", func() {
+		It("Retrieves the state slice for root path", func() {
+			path := "0x"
+			depth := 3
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "1",
+						"02-total-trie-nodes":    "6",
+						"03-leaves":              "5",
+						"04-smart-contracts":     "1",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						rootDataHashBlock5: rootDataBlock5,
+					},
+					Slice: map[string]string{
+						account1DataHash: account1Data,
+						account2DataHash: account2Data,
+						account3DataHash: account3Data,
+						account4DataHash: account4Data,
+						account5DataHash: account5Data,
+					},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{
+					account1StateLeafKey: {
+						StorageRoot: contractStorageRootBlock5.Hex(),
+						EVMCode:     account1Code,
+					},
+				},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the state slice for root path with 0 depth", func() {
+			path := "0x"
+			depth := 0
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "0",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						rootDataHashBlock5: rootDataBlock5,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the state slice for a path to an account", func() {
+			path := "0x06"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "2",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "2",
+						"03-leaves":              "1",
+						"04-smart-contracts":     "1",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{
+						rootDataHashBlock5: rootDataBlock5,
+					},
+					Head: map[string]string{
+						account1DataHash: account1Data,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{
+					account1StateLeafKey: {
+						StorageRoot: contractStorageRootBlock5.Hex(),
+						EVMCode:     account1Code,
+					},
+				},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the state slice for a path to a non-existing account", func() {
+			path := "0x06"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block1StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block1StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "0",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{
+						rootDataHashBlock1: rootDataBlock1,
+					},
+					Head:  map[string]string{},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+
+		It("Retrieves the storage slice for root path", func() {
+			path := "0x"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "1",
+						"02-total-trie-nodes":    "3",
+						"03-leaves":              "2",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						storageRootDataHashBlock4: storageRootDataBlock4,
+					},
+					Slice: map[string]string{
+						storageNode1DataHash: storageNode1Data,
+						storageNode2DataHash: storageNode2Data,
+					},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the storage slice for root path with 0 depth", func() {
+			path := "0x"
+			depth := 0
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "0",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						storageRootDataHashBlock4: storageRootDataBlock4,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the storage slice for root path with deleted nodes", func() {
+			path := "0x"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock5, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock5.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "1",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						storageRootDataHashBlock5: storageRootDataBlock5,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the storage slice for a path to a storage node", func() {
+			path := "0x0b"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "2",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "2",
+						"03-leaves":              "1",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{
+						storageRootDataHashBlock4: storageRootDataBlock4,
+					},
+					Head: map[string]string{
+						storageNode1DataHash: storageNode1Data,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+	})
+})

pkg/eth/subscription_config.go

@@ -16,38 +16,6 @@
 
 package eth
 
-import (
-	"math/big"
-
-	"github.com/spf13/viper"
-)
-
-// SubscriptionSettings config is used by a subscriber to specify what eth data to stream from the watcher
-type SubscriptionSettings struct {
-	BackFill      bool
-	BackFillOnly  bool
-	Start         *big.Int
-	End           *big.Int // set to 0 or a negative value to have no ending block
-	HeaderFilter  HeaderFilter
-	TxFilter      TxFilter
-	ReceiptFilter ReceiptFilter
-	StateFilter   StateFilter
-	StorageFilter StorageFilter
-}
-
-// HeaderFilter contains filter settings for headers
-type HeaderFilter struct {
-	Off    bool
-	Uncles bool
-}
-
-// TxFilter contains filter settings for txs
-type TxFilter struct {
-	Off bool
-	Src []string
-	Dst []string
-}
-
 // ReceiptFilter contains filter settings for receipts
 type ReceiptFilter struct {
 	Off bool
@@ -56,70 +24,3 @@ type ReceiptFilter struct {
 	LogAddresses []string // receipt contains logs from the provided addresses
 	Topics       [][]string
 }
-
-// StateFilter contains filter settings for state
-type StateFilter struct {
-	Off               bool
-	Addresses         []string // is converted to state key by taking its keccak256 hash
-	IntermediateNodes bool
-}
-
-// StorageFilter contains filter settings for storage
-type StorageFilter struct {
-	Off               bool
-	Addresses         []string
-	StorageKeys       []string // need to be the hashs key themselves not slot position
-	IntermediateNodes bool
-}
-
-// Init is used to initialize a EthSubscription struct with env variables
-func NewEthSubscriptionConfig() (*SubscriptionSettings, error) {
-	sc := new(SubscriptionSettings)
-	// Below default to false, which means we do not backfill by default
-	sc.BackFill = viper.GetBool("watcher.ethSubscription.historicalData")
-	sc.BackFillOnly = viper.GetBool("watcher.ethSubscription.historicalDataOnly")
-	// Below default to 0
-	// 0 start means we start at the beginning and 0 end means we continue indefinitely
-	sc.Start = big.NewInt(viper.GetInt64("watcher.ethSubscription.startingBlock"))
-	sc.End = big.NewInt(viper.GetInt64("watcher.ethSubscription.endingBlock"))
-	// Below default to false, which means we get all headers and no uncles by default
-	sc.HeaderFilter = HeaderFilter{
-		Off:    viper.GetBool("watcher.ethSubscription.headerFilter.off"),
-		Uncles: viper.GetBool("watcher.ethSubscription.headerFilter.uncles"),
-	}
-	// Below defaults to false and two slices of length 0
-	// Which means we get all transactions by default
-	sc.TxFilter = TxFilter{
-		Off: viper.GetBool("watcher.ethSubscription.txFilter.off"),
-		Src: viper.GetStringSlice("watcher.ethSubscription.txFilter.src"),
-		Dst: viper.GetStringSlice("watcher.ethSubscription.txFilter.dst"),
-	}
-	// By default all of the topic slices will be empty => match on any/all topics
-	topics := make([][]string, 4)
-	topics[0] = viper.GetStringSlice("watcher.ethSubscription.receiptFilter.topic0s")
-	topics[1] = viper.GetStringSlice("watcher.ethSubscription.receiptFilter.topic1s")
-	topics[2] = viper.GetStringSlice("watcher.ethSubscription.receiptFilter.topic2s")
-	topics[3] = viper.GetStringSlice("watcher.ethSubscription.receiptFilter.topic3s")
-	sc.ReceiptFilter = ReceiptFilter{
-		Off:          viper.GetBool("watcher.ethSubscription.receiptFilter.off"),
-		MatchTxs:     viper.GetBool("watcher.ethSubscription.receiptFilter.matchTxs"),
-		LogAddresses: viper.GetStringSlice("watcher.ethSubscription.receiptFilter.contracts"),
-		Topics:       topics,
-	}
-	// Below defaults to two false, and a slice of length 0
-	// Which means we get all state leafs by default, but no intermediate nodes
-	sc.StateFilter = StateFilter{
-		Off:               viper.GetBool("watcher.ethSubscription.stateFilter.off"),
-		IntermediateNodes: viper.GetBool("watcher.ethSubscription.stateFilter.intermediateNodes"),
-		Addresses:         viper.GetStringSlice("watcher.ethSubscription.stateFilter.addresses"),
-	}
-	// Below defaults to two false, and two slices of length 0
-	// Which means we get all storage leafs by default, but no intermediate nodes
-	sc.StorageFilter = StorageFilter{
-		Off:               viper.GetBool("watcher.ethSubscription.storageFilter.off"),
-		IntermediateNodes: viper.GetBool("watcher.ethSubscription.storageFilter.intermediateNodes"),
-		Addresses:         viper.GetStringSlice("watcher.ethSubscription.storageFilter.addresses"),
-		StorageKeys:       viper.GetStringSlice("watcher.ethSubscription.storageFilter.storageKeys"),
-	}
-	return sc, nil
-}

pkg/eth/test_helpers.go

@@ -1,66 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-)
-
-// TearDownDB is used to tear down the watcher dbs after tests
-func TearDownDB(db *postgres.DB) {
-	tx, err := db.Beginx()
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DELETE FROM eth.header_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.transaction_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.receipt_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.state_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.storage_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM blocks`)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = tx.Commit()
-	Expect(err).NotTo(HaveOccurred())
-}
-
-// TxModelsContainsCID used to check if a list of TxModels contains a specific cid string
-func TxModelsContainsCID(txs []eth.TxModel, cid string) bool {
-	for _, tx := range txs {
-		if tx.CID == cid {
-			return true
-		}
-	}
-	return false
-}
-
-// ListContainsBytes used to check if a list of byte arrays contains a particular byte array
-func ReceiptModelsContainsCID(rcts []eth.ReceiptModel, cid string) bool {
-	for _, rct := range rcts {
-		if rct.CID == cid {
-			return true
-		}
-	}
-	return false
-}

pkg/eth/test_helpers/abi.json

@@ -0,0 +1,47 @@
+[
+  {
+    "inputs": [],
+    "payable": false,
+    "stateMutability": "nonpayable",
+    "type": "constructor"
+  },
+  {
+    "constant": false,
+    "inputs": [
+      {
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      }
+    ],
+    "name": "Put",
+    "outputs": [],
+    "payable": false,
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "constant": false,
+    "inputs": [],
+    "name": "close",
+    "outputs": [],
+    "payable": false,
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "constant": true,
+    "inputs": [],
+    "name": "data",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "payable": false,
+    "stateMutability": "view",
+    "type": "function"
+  }
+]

pkg/eth/test_helpers/chain_maker.go

@@ -0,0 +1,110 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package test_helpers
+
+import (
+	"math/big"
+
+	"github.com/cerc-io/plugeth-statediff/test_helpers"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+// Test variables
+var (
+	Testdb          = rawdb.NewMemoryDatabase()
+	TestBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
+	TestBankFunds   = big.NewInt(100000000)
+
+	Genesis = test_helpers.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds, big.NewInt(params.InitialBaseFee), params.MaxGasLimit)
+
+	Account1Key, _       = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
+	Account2Key, _       = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
+	Account1Addr         = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
+	Account2Addr         = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e
+	DeploymentTxData     = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032")
+	ContractCode         = common.Hex2Bytes("608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032")
+	CodeHash             = crypto.Keccak256Hash(ContractCode)
+	ContractAddr         common.Address
+	IndexZero            = "0000000000000000000000000000000000000000000000000000000000000000"
+	IndexOne             = "0000000000000000000000000000000000000000000000000000000000000001"
+	ContractSlotPosition = common.FromHex(IndexOne)
+	ContractSlotKeyHash  = crypto.Keccak256Hash(ContractSlotPosition)
+	MiningReward         = big.NewInt(2000000000000000000)
+)
+
+/* test function signatures
+put function sig: 65f3c31a
+close function sig: 43d726d6
+data function sig: 73d4a13a
+*/
+
+// MakeChain creates a chain of n blocks starting at and including parent.
+// the returned hash chain is ordered head->parent.
+func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen), config *params.ChainConfig) ([]*types.Block, []types.Receipts, *core.BlockChain) {
+	blocks, receipts := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
+	chain, err := core.NewBlockChain(Testdb, nil, nil, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
+	if err != nil {
+		panic(err)
+	}
+	return append([]*types.Block{parent}, blocks...), receipts, chain
+}
+
+func TestChainGen(i int, block *core.BlockGen) {
+	signer := types.HomesteadSigner{}
+	switch i {
+	case 0:
+		// In block 1, the test bank sends account #1 some ether.
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey)
+		block.AddTx(tx)
+	case 1:
+		// In block 2, the test bank sends some more ether to account #1.
+		// Account1Addr passes it on to account #2.
+		// Account1Addr creates a test contract.
+		tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey)
+		nonce := block.TxNonce(Account1Addr)
+		tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key)
+		nonce++
+		tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), DeploymentTxData), signer, Account1Key)
+		ContractAddr = crypto.CreateAddress(Account1Addr, nonce)
+		block.AddTx(tx1)
+		block.AddTx(tx2)
+		block.AddTx(tx3)
+	case 2:
+		block.SetCoinbase(Account2Addr)
+		data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000003")
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+		block.AddTx(tx)
+	case 3:
+		block.SetCoinbase(Account2Addr)
+		data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000009")
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+		block.AddTx(tx)
+	case 4:
+		block.SetCoinbase(Account1Addr)
+		data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000000")
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+		block.AddTx(tx)
+	}
+}

pkg/eth/test_helpers/compiler_output.json

@@ -0,0 +1,6 @@
+{
+  "linkReferences": {},
+  "object": "608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032",
+  "opcodes": "PUSH1 0x80 PUSH1 0x40 MSTORE CALLVALUE DUP1 ISZERO PUSH2 0x10 JUMPI PUSH1 0x0 DUP1 REVERT JUMPDEST POP CALLER PUSH1 0x0 DUP1 PUSH2 0x100 EXP DUP2 SLOAD DUP2 PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF MUL NOT AND SWAP1 DUP4 PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND MUL OR SWAP1 SSTORE POP PUSH1 0x1 DUP1 DUP2 SWAP1 SSTORE POP PUSH2 0x1E2 DUP1 PUSH2 0x67 PUSH1 0x0 CODECOPY PUSH1 0x0 RETURN INVALID PUSH1 0x80 PUSH1 0x40 MSTORE CALLVALUE DUP1 ISZERO PUSH2 0x10 JUMPI PUSH1 0x0 DUP1 REVERT JUMPDEST POP PUSH1 0x4 CALLDATASIZE LT PUSH2 0x41 JUMPI PUSH1 0x0 CALLDATALOAD PUSH1 0xE0 SHR DUP1 PUSH4 0x43D726D6 EQ PUSH2 0x46 JUMPI DUP1 PUSH4 0x65F3C31A EQ PUSH2 0x50 JUMPI DUP1 PUSH4 0x73D4A13A EQ PUSH2 0x7E JUMPI JUMPDEST PUSH1 0x0 DUP1 REVERT JUMPDEST PUSH2 0x4E PUSH2 0x9C JUMP JUMPDEST STOP JUMPDEST PUSH2 0x7C PUSH1 0x4 DUP1 CALLDATASIZE SUB PUSH1 0x20 DUP2 LT ISZERO PUSH2 0x66 JUMPI PUSH1 0x0 DUP1 REVERT JUMPDEST DUP2 ADD SWAP1 DUP1 DUP1 CALLDATALOAD SWAP1 PUSH1 0x20 ADD SWAP1 SWAP3 SWAP2 SWAP1 POP POP POP PUSH2 0x17B JUMP JUMPDEST STOP JUMPDEST PUSH2 0x86 PUSH2 0x185 JUMP JUMPDEST PUSH1 0x40 MLOAD DUP1 DUP3 DUP2 MSTORE PUSH1 0x20 ADD SWAP2 POP POP PUSH1 0x40 MLOAD DUP1 SWAP2 SUB SWAP1 RETURN JUMPDEST PUSH1 0x0 DUP1 SWAP1 SLOAD SWAP1 PUSH2 0x100 EXP SWAP1 DIV PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND CALLER PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND EQ PUSH2 0x141 JUMPI PUSH1 0x40 MLOAD PUSH32 0x8C379A000000000000000000000000000000000000000000000000000000000 DUP2 MSTORE PUSH1 0x4 ADD DUP1 DUP1 PUSH1 0x20 ADD DUP3 DUP2 SUB DUP3 MSTORE PUSH1 0x22 DUP2 MSTORE PUSH1 0x20 ADD DUP1 PUSH2 0x18C PUSH1 0x22 SWAP2 CODECOPY PUSH1 0x40 ADD SWAP2 POP POP PUSH1 0x40 MLOAD DUP1 SWAP2 SUB SWAP1 REVERT JUMPDEST PUSH1 0x0 DUP1 SWAP1 SLOAD SWAP1 PUSH2 0x100 EXP SWAP1 DIV PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND SELFDESTRUCT JUMPDEST DUP1 PUSH1 0x1 DUP2 SWAP1 SSTORE POP POP JUMP JUMPDEST PUSH1 0x1 SLOAD DUP2 JUMP INVALID 0x4F PUSH15 0x6C79206F776E65722063616E206361 PUSH13 0x6C20746869732066756E637469 PUSH16 0x6E2EA265627A7A723158205BA9146612 SWAP16 GASLIMIT 0x28 0x5F MSTORE8 OR PUSH14 0x805117208C231EC6343D7896790E PUSH16 0xC4165B802B64736F6C63430005110032 ",
+  "sourceMap": "26:449:0:-;;;253:74;8:9:-1;5:2;;;30:1;27;20:12;5:2;253:74:0;292:10;284:5;;:18;;;;;;;;;;;;;;;;;;319:1;312:4;:8;;;;26:449;;;;;;"
+}

pkg/eth/test_helpers/test_contract.sol

@@ -0,0 +1,28 @@
+pragma solidity ^0.5.10;
+
+contract test {
+    address payable owner;
+
+    modifier onlyOwner {
+        require(
+            msg.sender == owner,
+            "Only owner can call this function."
+        );
+        _;
+    }
+
+    uint256 public data;
+
+    constructor() public {
+        owner = msg.sender;
+        data = 1;
+    }
+
+    function Put(uint256 value) public {
+        data = value;
+    }
+
+    function close() public onlyOwner {
+        selfdestruct(owner);
+    }
+}

pkg/eth/test_helpers/test_data.go

@@ -0,0 +1,469 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package test_helpers
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"math/big"
+
+	"github.com/cerc-io/plugeth-statediff/indexer/ipld"
+	"github.com/cerc-io/plugeth-statediff/indexer/models"
+	sdtypes "github.com/cerc-io/plugeth-statediff/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/holiman/uint256"
+	"github.com/ipfs/go-cid"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+)
+
+// Test variables
+var (
+	// block data
+	BlockNumber1 = int64(1)
+	BlockTime1   = uint64(0)
+	MockHeader   = types.Header{
+		Time:        0,
+		Number:      big.NewInt(BlockNumber1),
+		Root:        common.HexToHash("0x0"),
+		TxHash:      common.HexToHash("0x0"),
+		ReceiptHash: common.HexToHash("0x0"),
+		Difficulty:  big.NewInt(5000000),
+		Extra:       []byte{},
+	}
+	MockTransactions, MockReceipts, SenderAddr = createLegacyTransactionsAndReceipts()
+	MockWithdrawals                            = types.Withdrawals{
+		{Index: 0, Validator: 1, Address: Address, Amount: 1000000000},
+		{Index: 1, Validator: 5, Address: AnotherAddress, Amount: 2000000000},
+	}
+	MockUncles = []*types.Header{
+		{
+			Time:        1,
+			Number:      big.NewInt(BlockNumber1 + 1),
+			Root:        common.HexToHash("0x1"),
+			TxHash:      common.HexToHash("0x1"),
+			ReceiptHash: common.HexToHash("0x1"),
+			Difficulty:  big.NewInt(500001),
+			Extra:       []byte{},
+			ParentHash:  Genesis.Hash(),
+		},
+		{
+			Time:        2,
+			Number:      big.NewInt(BlockNumber1 + 2),
+			Root:        common.HexToHash("0x2"),
+			TxHash:      common.HexToHash("0x2"),
+			ReceiptHash: common.HexToHash("0x2"),
+			Difficulty:  big.NewInt(500002),
+			Extra:       []byte{},
+			ParentHash:  Genesis.Hash(),
+		},
+	}
+	MockBlock       = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, nil, trie.NewEmpty(nil))
+	MockChildHeader = types.Header{
+		Time:        0,
+		Number:      big.NewInt(BlockNumber1 + 1),
+		Root:        common.HexToHash("0x0"),
+		TxHash:      common.HexToHash("0x0"),
+		ReceiptHash: common.HexToHash("0x0"),
+		Difficulty:  big.NewInt(5000001),
+		Extra:       []byte{},
+		ParentHash:  MockBlock.Header().Hash(),
+	}
+	MockChild       = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts, trie.NewEmpty(nil))
+	Address         = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
+	AnotherAddress  = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	AnotherAddress1 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476594")
+	AnotherAddress2 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476596")
+	ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
+	mockTopic11     = common.HexToHash("0x04")
+	mockTopic12     = common.HexToHash("0x06")
+	mockTopic21     = common.HexToHash("0x05")
+	mockTopic22     = common.HexToHash("0x07")
+	mockTopic31     = common.HexToHash("0x08")
+	mockTopic41     = common.HexToHash("0x09")
+	mockTopic42     = common.HexToHash("0x0a")
+	mockTopic43     = common.HexToHash("0x0b")
+	mockTopic51     = common.HexToHash("0x0c")
+	mockTopic61     = common.HexToHash("0x0d")
+	MockLog1        = &types.Log{
+		Address:     Address,
+		Topics:      []common.Hash{mockTopic11, mockTopic12},
+		Data:        []byte{},
+		BlockNumber: uint64(BlockNumber1),
+		TxIndex:     0,
+		Index:       0,
+	}
+	MockLog2 = &types.Log{
+		Address:     AnotherAddress,
+		Topics:      []common.Hash{mockTopic21, mockTopic22},
+		Data:        []byte{},
+		BlockNumber: uint64(BlockNumber1),
+		TxIndex:     1,
+		Index:       1,
+	}
+	MockLog3 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic31},
+		Data:        []byte{},
+		BlockNumber: uint64(BlockNumber1),
+		TxIndex:     2,
+		Index:       2,
+	}
+
+	MockLog4 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic41, mockTopic42, mockTopic43},
+		Data:        []byte{},
+		BlockNumber: uint64(BlockNumber1),
+		TxIndex:     2,
+		Index:       3,
+	}
+	MockLog5 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic51},
+		Data:        []byte{},
+		BlockNumber: uint64(BlockNumber1),
+		TxIndex:     2,
+		Index:       4,
+	}
+	MockLog6 = &types.Log{
+		Address:     AnotherAddress2,
+		Topics:      []common.Hash{mockTopic61},
+		Data:        []byte{},
+		BlockNumber: uint64(BlockNumber1),
+		TxIndex:     3,
+		Index:       5,
+	}
+
+	rctCIDs, _  = getReceiptCIDs(MockReceipts)
+	Rct1CID     = rctCIDs[0]
+	Rct4CID     = rctCIDs[3]
+	MockTrxMeta = []models.TxModel{
+		{
+			CID:    "", // This is empty until we go to publish to ipfs
+			Src:    SenderAddr.Hex(),
+			Dst:    Address.String(),
+			Index:  0,
+			TxHash: MockTransactions[0].Hash().String(),
+		},
+		{
+			CID:    "",
+			Src:    SenderAddr.Hex(),
+			Dst:    AnotherAddress.String(),
+			Index:  1,
+			TxHash: MockTransactions[1].Hash().String(),
+		},
+		{
+			CID:    "",
+			Src:    SenderAddr.Hex(),
+			Dst:    "",
+			Index:  2,
+			TxHash: MockTransactions[2].Hash().String(),
+		},
+		{
+			CID:    "",
+			Src:    SenderAddr.Hex(),
+			Dst:    "",
+			Index:  3,
+			TxHash: MockTransactions[3].Hash().String(),
+		},
+	}
+	MockRctMeta = []models.ReceiptModel{
+		{
+			CID:      "",
+			Contract: "",
+		},
+		{
+			CID:      "",
+			Contract: "",
+		},
+		{
+			CID:      "",
+			Contract: ContractAddress.String(),
+		},
+		{
+			CID:      "",
+			Contract: "",
+		},
+	}
+
+	// statediff data
+	storageLocation    = common.HexToHash("0")
+	StorageLeafKey     = crypto.Keccak256Hash(storageLocation[:]).Bytes()
+	StorageValue       = crypto.Keccak256([]byte{1, 2, 3, 4, 5})
+	StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
+	StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
+		StoragePartialPath,
+		StorageValue,
+	})
+
+	ContractRoot    = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
+	contractPath    = common.Bytes2Hex([]byte{'\x06'})
+	ContractLeafKey = crypto.Keccak256(ContractAddress[:])
+	ContractAccount = types.StateAccount{
+		Nonce:    uint64(1),
+		Balance:  uint256.NewInt(0),
+		CodeHash: CodeHash.Bytes(),
+		Root:     common.HexToHash(ContractRoot),
+	}
+	ContractAccountRLP, _ = rlp.EncodeToBytes(&ContractAccount)
+	ContractPartialPath   = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
+	ContractLeafNode, _   = rlp.EncodeToBytes(&[]interface{}{
+		ContractPartialPath,
+		ContractAccountRLP,
+	})
+
+	nonce0          = uint64(0)
+	AccountBalance  = uint256.NewInt(1000)
+	AccountRoot     = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
+	AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
+	AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
+	AccountLeafKey  = crypto.Keccak256(AccountAddresss[:])
+	Account         = types.StateAccount{
+		Nonce:    nonce0,
+		Balance:  AccountBalance,
+		CodeHash: AccountCodeHash.Bytes(),
+		Root:     common.HexToHash(AccountRoot),
+	}
+	AccountRLP, _      = rlp.EncodeToBytes(&Account)
+	AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
+	AccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
+		AccountPartialPath,
+		AccountRLP,
+	})
+
+	MockStateNodes = []sdtypes.StateLeafNode{
+		{
+			AccountWrapper: sdtypes.AccountWrapper{
+				Account: &ContractAccount,
+				LeafKey: ContractLeafKey,
+				CID:     ipld.Keccak256ToCid(ipld.MEthStateTrie, crypto.Keccak256(ContractLeafNode)).String(),
+			},
+			StorageDiff: []sdtypes.StorageLeafNode{
+				{
+					LeafKey: StorageLeafKey,
+					Value:   StorageValue,
+					CID:     ipld.Keccak256ToCid(ipld.MEthStorageTrie, crypto.Keccak256(StorageLeafNode)).String(),
+				},
+			},
+		},
+		{
+			AccountWrapper: sdtypes.AccountWrapper{
+				Account: &Account,
+				LeafKey: AccountLeafKey,
+				CID:     ipld.Keccak256ToCid(ipld.MEthStateTrie, crypto.Keccak256(AccountLeafNode)).String(),
+			},
+		},
+	}
+	MockStorageNodes = map[string][]sdtypes.StorageLeafNode{
+		contractPath: {
+			{
+				LeafKey: StorageLeafKey,
+				Value:   StorageValue,
+				CID:     ipld.Keccak256ToCid(ipld.MEthStorageTrie, crypto.Keccak256(StorageLeafNode)).String(),
+			},
+		},
+	}
+
+	MockConvertedPayload = eth.ConvertedPayload{
+		TotalDifficulty: MockBlock.Difficulty(),
+		Block:           MockBlock,
+		Receipts:        MockReceipts,
+		TxMetaData:      MockTrxMeta,
+		ReceiptMetaData: MockRctMeta,
+		StorageNodes:    MockStorageNodes,
+		StateNodes:      MockStateNodes,
+	}
+
+	LondonBlockNum   = (BlockNumber1 + 2)
+	LondonBlockTime  = BlockTime1 + 1
+	MockLondonHeader = types.Header{
+		Time:       LondonBlockTime,
+		Number:     big.NewInt(LondonBlockNum),
+		Root:       common.HexToHash("0x00"),
+		Difficulty: big.NewInt(5000000),
+		Extra:      []byte{},
+		BaseFee:    big.NewInt(params.InitialBaseFee),
+	}
+
+	MockLondonTransactions, MockLondonReceipts, _ = createDynamicTransactionsAndReceipts(big.NewInt(LondonBlockNum), LondonBlockTime)
+	MockLondonUncles                              = []*types.Header{
+		{
+			Time:        1,
+			Number:      big.NewInt(BlockNumber1 + 1),
+			ParentHash:  common.HexToHash("0x2"),
+			Root:        common.HexToHash("0x1"),
+			TxHash:      common.HexToHash("0x1"),
+			ReceiptHash: common.HexToHash("0x1"),
+			Difficulty:  big.NewInt(500001),
+			Extra:       []byte{},
+		},
+		{
+			Time:        2,
+			Number:      big.NewInt(BlockNumber1 + 1),
+			ParentHash:  common.HexToHash("0x1"),
+			Root:        common.HexToHash("0x2"),
+			TxHash:      common.HexToHash("0x2"),
+			ReceiptHash: common.HexToHash("0x2"),
+			Difficulty:  big.NewInt(500002),
+			Extra:       []byte{},
+		},
+	}
+	MockLondonBlock = createNewBlock(&MockLondonHeader, MockLondonTransactions, MockLondonUncles, MockLondonReceipts, MockWithdrawals, trie.NewEmpty(nil))
+)
+
+func createNewBlock(header *types.Header, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, withdrawals types.Withdrawals, hasher types.TrieHasher) *types.Block {
+	block := types.NewBlockWithWithdrawals(header, txs, uncles, receipts, withdrawals, hasher)
+	bHash := block.Hash()
+	for _, r := range receipts {
+		for _, l := range r.Logs {
+			l.BlockHash = bHash
+		}
+	}
+	return block
+}
+
+// createDynamicTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createDynamicTransactionsAndReceipts(blockNumber *big.Int, blockTime uint64) (types.Transactions, types.Receipts, common.Address) {
+	// make transactions
+	config := *params.TestChainConfig
+	config.LondonBlock = blockNumber
+	trx1 := types.NewTx(&types.DynamicFeeTx{
+		ChainID:   config.ChainID,
+		Nonce:     1,
+		GasTipCap: big.NewInt(50),
+		GasFeeCap: big.NewInt(100),
+		Gas:       50,
+		To:        &Address,
+		Value:     big.NewInt(1000),
+		Data:      []byte{},
+	})
+
+	transactionSigner := types.MakeSigner(&config, blockNumber, blockTime)
+	mockCurve := elliptic.P256()
+	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+	signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	senderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	// make receipts
+	// TODO: Change the receipt type to DynamicFeeTxType once this PR is merged.
+	// https://github.com/ethereum/go-ethereum/pull/22806
+	mockReceipt1 := &types.Receipt{
+		Type:              types.DynamicFeeTxType,
+		PostState:         common.HexToHash("0x0").Bytes(),
+		Status:            types.ReceiptStatusSuccessful,
+		CumulativeGasUsed: 50,
+		Logs:              []*types.Log{},
+		TxHash:            signedTrx1.Hash(),
+	}
+
+	return types.Transactions{signedTrx1}, types.Receipts{mockReceipt1}, senderAddr
+}
+
+// createLegacyTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
+	// make transactions
+	trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
+	trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
+	trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), ContractCode)
+	trx4 := types.NewTransaction(3, AnotherAddress1, big.NewInt(2000), 100, big.NewInt(200), []byte{})
+	transactionSigner := types.MakeSigner(params.MainnetChainConfig, big.NewInt(BlockNumber1), 0)
+	mockCurve := elliptic.P256()
+	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx3, err := types.SignTx(trx3, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx4, err := types.SignTx(trx4, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	SenderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
+	if err != nil {
+		log.Fatal(err)
+	}
+	// make receipts
+	mockReceipt1 := types.NewReceipt(nil, false, 50)
+
+	hash1 := signedTrx1.Hash()
+	MockLog1.TxHash = hash1
+
+	mockReceipt1.Logs = []*types.Log{MockLog1}
+	mockReceipt1.TxHash = hash1
+	mockReceipt1.GasUsed = mockReceipt1.CumulativeGasUsed
+
+	mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
+	hash2 := signedTrx2.Hash()
+	MockLog2.TxHash = hash2
+
+	mockReceipt2.Logs = []*types.Log{MockLog2}
+	mockReceipt2.TxHash = hash2
+	mockReceipt2.GasUsed = mockReceipt2.CumulativeGasUsed - mockReceipt1.CumulativeGasUsed
+
+	mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 175)
+	mockReceipt3.Logs = []*types.Log{MockLog3, MockLog4, MockLog5}
+	mockReceipt3.TxHash = signedTrx3.Hash()
+	mockReceipt3.GasUsed = mockReceipt3.CumulativeGasUsed - mockReceipt2.CumulativeGasUsed
+
+	// Receipt with failed status.
+	mockReceipt4 := types.NewReceipt(nil, true, 250)
+	mockReceipt4.Logs = []*types.Log{MockLog6}
+	mockReceipt4.TxHash = signedTrx4.Hash()
+	mockReceipt4.GasUsed = mockReceipt4.CumulativeGasUsed - mockReceipt3.CumulativeGasUsed
+
+	return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4}, SenderAddr
+}
+
+func getReceiptCIDs(rcts []*types.Receipt) ([]cid.Cid, error) {
+	cids := make([]cid.Cid, len(rcts))
+	for i, rct := range rcts {
+		ethRct, err := ipld.EncodeReceipt(rct)
+		if err != nil {
+			return nil, err
+		}
+		cids[i] = ethRct.Cid()
+	}
+	return cids, nil
+}

pkg/eth/transaction_arguments.go

@@ -0,0 +1,49 @@
+package eth
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+// TransactionArgs represents the arguments to construct a new transaction
+// or a message call.
+type TransactionArgs struct {
+	From                 *common.Address `json:"from"`
+	To                   *common.Address `json:"to"`
+	Gas                  *hexutil.Uint64 `json:"gas"`
+	GasPrice             *hexutil.Big    `json:"gasPrice"`
+	MaxFeePerGas         *hexutil.Big    `json:"maxFeePerGas"`
+	MaxPriorityFeePerGas *hexutil.Big    `json:"maxPriorityFeePerGas"`
+	Value                *hexutil.Big    `json:"value"`
+	Nonce                *hexutil.Uint64 `json:"nonce"`
+
+	// We accept "data" and "input" for backwards-compatibility reasons.
+	// "input" is the newer name and should be preferred by clients.
+	// Issue detail: https://github.com/ethereum/go-ethereum/issues/15628
+	Data  *hexutil.Bytes `json:"data"`
+	Input *hexutil.Bytes `json:"input"`
+
+	// Introduced by AccessListTxType transaction.
+	AccessList *types.AccessList `json:"accessList,omitempty"`
+	ChainID    *hexutil.Big      `json:"chainId,omitempty"`
+}
+
+// from retrieves the transaction sender address.
+func (args *TransactionArgs) from() common.Address {
+	if args.From == nil {
+		return common.Address{}
+	}
+	return *args.From
+}
+
+// data retrieves the transaction calldata. Input field is preferred.
+func (args *TransactionArgs) data() []byte {
+	if args.Input != nil {
+		return *args.Input
+	}
+	if args.Data != nil {
+		return *args.Data
+	}
+	return nil
+}

pkg/eth/types.go

@@ -0,0 +1,304 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"errors"
+	"fmt"
+	"math/big"
+	"strconv"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/cerc-io/plugeth-statediff/indexer/models"
+	sdtypes "github.com/cerc-io/plugeth-statediff/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
+// Note: copied from go-ethereum/internal/ethapi
+type RPCTransaction struct {
+	BlockHash           *common.Hash      `json:"blockHash"`
+	BlockNumber         *hexutil.Big      `json:"blockNumber"`
+	From                common.Address    `json:"from"`
+	Gas                 hexutil.Uint64    `json:"gas"`
+	GasPrice            *hexutil.Big      `json:"gasPrice"`
+	GasFeeCap           *hexutil.Big      `json:"maxFeePerGas,omitempty"`
+	GasTipCap           *hexutil.Big      `json:"maxPriorityFeePerGas,omitempty"`
+	MaxFeePerBlobGas    *hexutil.Big      `json:"maxFeePerBlobGas,omitempty"`
+	Hash                common.Hash       `json:"hash"`
+	Input               hexutil.Bytes     `json:"input"`
+	Nonce               hexutil.Uint64    `json:"nonce"`
+	To                  *common.Address   `json:"to"`
+	TransactionIndex    *hexutil.Uint64   `json:"transactionIndex"`
+	Value               *hexutil.Big      `json:"value"`
+	Type                hexutil.Uint64    `json:"type"`
+	Accesses            *types.AccessList `json:"accessList,omitempty"`
+	ChainID             *hexutil.Big      `json:"chainId,omitempty"`
+	BlobVersionedHashes []common.Hash     `json:"blobVersionedHashes,omitempty"`
+	V                   *hexutil.Big      `json:"v"`
+	R                   *hexutil.Big      `json:"r"`
+	S                   *hexutil.Big      `json:"s"`
+	YParity             *hexutil.Uint64   `json:"yParity,omitempty"`
+}
+
+// RPCReceipt represents a receipt that will serialize to the RPC representation of a receipt
+type RPCReceipt struct {
+	BlockHash        *common.Hash    `json:"blockHash"`
+	BlockNumber      *hexutil.Big    `json:"blockNumber"`
+	TransactionHash  *common.Hash    `json:"transactionHash"`
+	TransactionIndex *hexutil.Uint64 `json:"transactionIndex"`
+	From             common.Address  `json:"from"`
+	To               *common.Address `json:"to"`
+	GasUsed          hexutil.Uint64  `json:"gasUsed"`
+	CumulativeGsUsed hexutil.Uint64  `json:"cumulativeGasUsed"`
+	ContractAddress  *common.Address `json:"contractAddress"`
+	Logs             []*types.Log    `json:"logs"`
+	Bloom            types.Bloom     `json:"logsBloom"`
+	Root             []byte          `json:"root"`
+	Status           uint64          `json:"status"`
+}
+
+// AccountResult struct for GetProof
+type AccountResult struct {
+	Address      common.Address  `json:"address"`
+	AccountProof []string        `json:"accountProof"`
+	Balance      *hexutil.Big    `json:"balance"`
+	CodeHash     common.Hash     `json:"codeHash"`
+	Nonce        hexutil.Uint64  `json:"nonce"`
+	StorageHash  common.Hash     `json:"storageHash"`
+	StorageProof []StorageResult `json:"storageProof"`
+}
+
+// StorageResult for GetProof
+type StorageResult struct {
+	Key   string       `json:"key"`
+	Value *hexutil.Big `json:"value"`
+	Proof []string     `json:"proof"`
+}
+
+// CallArgs represents the arguments for a call.
+type CallArgs struct {
+	From                 *common.Address   `json:"from"`
+	To                   *common.Address   `json:"to"`
+	Gas                  *hexutil.Uint64   `json:"gas"`
+	GasPrice             *hexutil.Big      `json:"gasPrice"`
+	MaxFeePerGas         *hexutil.Big      `json:"maxFeePerGas"`
+	MaxPriorityFeePerGas *hexutil.Big      `json:"maxPriorityFeePerGas"`
+	Value                *hexutil.Big      `json:"value"`
+	Data                 *hexutil.Bytes    `json:"data"`
+	AccessList           *types.AccessList `json:"accessList,omitempty"`
+	Input                *hexutil.Bytes    `json:"input"`
+}
+
+// from retrieves the transaction sender address.
+func (arg *CallArgs) from() common.Address {
+	if arg.From == nil {
+		return common.Address{}
+	}
+	return *arg.From
+}
+
+// data retrieves the transaction calldata. Input field is preferred.
+func (arg *CallArgs) data() []byte {
+	if arg.Input != nil {
+		return *arg.Input
+	}
+	if arg.Data != nil {
+		return *arg.Data
+	}
+	return nil
+}
+
+// ToMessage converts the transaction arguments to the Message type used by the
+// core evm. This method is used in calls and traces that do not require a real
+// live transaction.
+func (arg *CallArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (*core.Message, error) {
+	// Reject invalid combinations of pre- and post-1559 fee styles
+	if arg.GasPrice != nil && (arg.MaxFeePerGas != nil || arg.MaxPriorityFeePerGas != nil) {
+		return nil, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified")
+	}
+	// Set sender address or use zero address if none specified.
+	addr := arg.from()
+
+	// Set default gas & gas price if none were set
+	gas := globalGasCap
+	if gas == 0 {
+		gas = uint64(math.MaxUint64 / 2)
+	}
+	if arg.Gas != nil {
+		gas = uint64(*arg.Gas)
+	}
+	if globalGasCap != 0 && globalGasCap < gas {
+		log.Warn("Caller gas above allowance, capping", "requested", gas, "cap", globalGasCap)
+		gas = globalGasCap
+	}
+	var (
+		gasPrice  *big.Int
+		gasFeeCap *big.Int
+		gasTipCap *big.Int
+	)
+	if baseFee == nil {
+		// If there's no basefee, then it must be a non-1559 execution
+		gasPrice = new(big.Int)
+		if arg.GasPrice != nil {
+			gasPrice = arg.GasPrice.ToInt()
+		}
+		gasFeeCap, gasTipCap = gasPrice, gasPrice
+	} else {
+		// A basefee is provided, necessitating 1559-type execution
+		if arg.GasPrice != nil {
+			// User specified the legacy gas field, convert to 1559 gas typing
+			gasPrice = arg.GasPrice.ToInt()
+			gasFeeCap, gasTipCap = gasPrice, gasPrice
+		} else {
+			// User specified 1559 gas feilds (or none), use those
+			gasFeeCap = new(big.Int)
+			if arg.MaxFeePerGas != nil {
+				gasFeeCap = arg.MaxFeePerGas.ToInt()
+			}
+			gasTipCap = new(big.Int)
+			if arg.MaxPriorityFeePerGas != nil {
+				gasTipCap = arg.MaxPriorityFeePerGas.ToInt()
+			}
+			// Backfill the legacy gasPrice for EVM execution, unless we're all zeroes
+			gasPrice = new(big.Int)
+			if gasFeeCap.BitLen() > 0 || gasTipCap.BitLen() > 0 {
+				gasPrice = math.BigMin(new(big.Int).Add(gasTipCap, baseFee), gasFeeCap)
+			}
+		}
+	}
+	value := new(big.Int)
+	if arg.Value != nil {
+		value = arg.Value.ToInt()
+	}
+	data := arg.data()
+	var accessList types.AccessList
+	if arg.AccessList != nil {
+		accessList = *arg.AccessList
+	}
+	msg := &core.Message{
+		Nonce:             0,
+		GasLimit:          gas,
+		GasPrice:          gasPrice,
+		GasFeeCap:         gasFeeCap,
+		GasTipCap:         gasTipCap,
+		To:                arg.To,
+		Value:             value,
+		Data:              data,
+		AccessList:        accessList,
+		SkipAccountChecks: true,
+		From:              addr,
+	}
+	return msg, nil
+}
+
+// ConvertedPayload is a custom type which packages raw ETH data for publishing to IPFS and filtering to subscribers
+// Returned by PayloadConverter
+// Passed to IPLDPublisher and ResponseFilterer
+type ConvertedPayload struct {
+	TotalDifficulty *big.Int
+	Block           *types.Block
+	TxMetaData      []models.TxModel
+	Receipts        types.Receipts
+	ReceiptMetaData []models.ReceiptModel
+	StateNodes      []sdtypes.StateLeafNode
+	StorageNodes    map[string][]sdtypes.StorageLeafNode
+}
+
+// LogResult represent a log.
+type LogResult struct {
+	LeafCID     string `db:"cid"`
+	ReceiptID   string `db:"rct_id"`
+	Address     string `db:"address"`
+	Index       int64  `db:"index"`
+	Data        []byte `db:"log_data"`
+	Topic0      string `db:"topic0"`
+	Topic1      string `db:"topic1"`
+	Topic2      string `db:"topic2"`
+	Topic3      string `db:"topic3"`
+	LogLeafData []byte `db:"data"`
+	RctCID      string `db:"rct_cid"`
+	RctStatus   uint64 `db:"post_status"`
+	BlockNumber string `db:"block_number"`
+	BlockHash   string `db:"block_hash"`
+	TxnIndex    int64  `db:"txn_index"`
+	TxHash      string `db:"tx_hash"`
+}
+
+// GetSliceResponse holds response for the eth_getSlice method
+type GetSliceResponse struct {
+	SliceID   string                             `json:"sliceId"`
+	MetaData  GetSliceResponseMetadata           `json:"metadata"`
+	TrieNodes GetSliceResponseTrieNodes          `json:"trieNodes"`
+	Leaves    map[string]GetSliceResponseAccount `json:"leaves"` // key: Keccak256Hash(address) in hex (leafKey)
+}
+
+func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) {
+	sr.SliceID = fmt.Sprintf("%s-%d-%s", path, depth, root.String())
+	sr.MetaData = GetSliceResponseMetadata{
+		NodeStats: make(map[string]string, 0),
+		TimeStats: make(map[string]string, 0),
+	}
+	sr.Leaves = make(map[string]GetSliceResponseAccount)
+	sr.TrieNodes = GetSliceResponseTrieNodes{
+		Stem:  make(map[string]string),
+		Head:  make(map[string]string),
+		Slice: make(map[string]string),
+	}
+}
+
+func (sr *GetSliceResponse) populateMetaData(metaData metaDataFields) {
+	sr.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head))
+	sr.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(metaData.maxDepth)
+	sr.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head) + len(sr.TrieNodes.Slice))
+	sr.MetaData.NodeStats["03-leaves"] = strconv.Itoa(metaData.leafCount)
+	sr.MetaData.NodeStats["04-smart-contracts"] = strconv.Itoa(len(sr.Leaves))
+
+	sr.MetaData.TimeStats["00-trie-loading"] = strconv.FormatInt(metaData.trieLoadingTime, 10)
+	sr.MetaData.TimeStats["01-fetch-stem-keys"] = strconv.FormatInt(metaData.stemNodesFetchTime, 10)
+	sr.MetaData.TimeStats["02-fetch-slice-keys"] = strconv.FormatInt(metaData.sliceNodesFetchTime, 10)
+	sr.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.FormatInt(metaData.leavesFetchTime, 10)
+}
+
+type GetSliceResponseMetadata struct {
+	TimeStats map[string]string `json:"timeStats"` // stem, state, storage (one by one)
+	NodeStats map[string]string `json:"nodeStats"` // total, leaves, smart contracts
+}
+
+type GetSliceResponseTrieNodes struct {
+	Stem  map[string]string `json:"stem"` // key: Keccak256Hash(data) in hex, value: trie node data in hex
+	Head  map[string]string `json:"head"`
+	Slice map[string]string `json:"sliceNodes"`
+}
+
+type GetSliceResponseAccount struct {
+	StorageRoot string `json:"storageRoot"`
+	EVMCode     string `json:"evmCode"`
+}
+
+type metaDataFields struct {
+	maxDepth            int
+	leafCount           int
+	trieLoadingTime     int64
+	stemNodesFetchTime  int64
+	sliceNodesFetchTime int64
+	leavesFetchTime     int64
+}

pkg/graphql/client.go

@@ -0,0 +1,274 @@
+package graphql
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	gqlclient "github.com/machinebox/graphql"
+)
+
+type StorageResponse struct {
+	CID       string        `json:"cid"`
+	Value     common.Hash   `json:"value"`
+	IpldBlock hexutil.Bytes `json:"ipldBlock"`
+}
+
+type GetStorageAt struct {
+	Response StorageResponse `json:"getStorageAt"`
+}
+
+type LogResponse struct {
+	Topics      []common.Hash       `json:"topics"`
+	Data        hexutil.Bytes       `json:"data"`
+	Transaction TransactionResponse `json:"transaction"`
+	ReceiptCID  string              `json:"receiptCID"`
+	Status      int32               `json:"status"`
+}
+
+type TransactionResponse struct {
+	Hash common.Hash `json:"hash"`
+}
+
+type GetLogs struct {
+	Responses []LogResponse `json:"getLogs"`
+}
+
+type IPFSBlockResponse struct {
+	Key  string `json:"key"`
+	Data string `json:"data"`
+}
+
+type EthTransactionCIDResponse struct {
+	CID        string            `json:"cid"`
+	TxHash     string            `json:"txHash"`
+	Index      int32             `json:"index"`
+	Src        string            `json:"src"`
+	Dst        string            `json:"dst"`
+	BlockByCid IPFSBlockResponse `json:"blockByCid"`
+}
+
+type EthTransactionCIDByTxHash struct {
+	Response EthTransactionCIDResponse `json:"ethTransactionCidByTxHash"`
+}
+
+type EthTransactionCIDsByHeaderIdResponse struct {
+	Nodes []EthTransactionCIDResponse `json:"nodes"`
+}
+
+type EthHeaderCIDResponse struct {
+	CID                          string                               `json:"cid"`
+	BlockNumber                  BigInt                               `json:"blockNumber"`
+	BlockHash                    string                               `json:"blockHash"`
+	ParentHash                   string                               `json:"parentHash"`
+	Timestamp                    BigInt                               `json:"timestamp"`
+	StateRoot                    string                               `json:"stateRoot"`
+	Td                           BigInt                               `json:"td"`
+	TxRoot                       string                               `json:"txRoot"`
+	ReceiptRoot                  string                               `json:"receiptRoot"`
+	UncleRoot                    string                               `json:"uncleRoot"`
+	Bloom                        string                               `json:"bloom"`
+	EthTransactionCIDsByHeaderId EthTransactionCIDsByHeaderIdResponse `json:"ethTransactionCidsByHeaderId"`
+	BlockByCid                   IPFSBlockResponse                    `json:"blockByCid"`
+}
+
+type AllEthHeaderCIDsResponse struct {
+	Nodes []EthHeaderCIDResponse `json:"nodes"`
+}
+
+type AllEthHeaderCIDs struct {
+	Response AllEthHeaderCIDsResponse `json:"allEthHeaderCids"`
+}
+
+type Client struct {
+	client *gqlclient.Client
+}
+
+func NewClient(endpoint string) *Client {
+	client := gqlclient.NewClient(endpoint)
+	return &Client{client: client}
+}
+
+func (c *Client) GetLogs(ctx context.Context, hash common.Hash, addresses []common.Address) ([]LogResponse, error) {
+	params := fmt.Sprintf(`blockHash: "%s"`, hash.String())
+
+	if addresses != nil {
+		addressStrings := make([]string, len(addresses))
+		for i, address := range addresses {
+			addressStrings[i] = fmt.Sprintf(`"%s"`, address.String())
+		}
+
+		params += fmt.Sprintf(`, addresses: [%s]`, strings.Join(addressStrings, ","))
+	}
+
+	getLogsQuery := fmt.Sprintf(`query{
+			getLogs(%s) {
+				data
+				topics
+				transaction {
+					hash
+				}
+				status
+				receiptCID
+			}
+		}`, params)
+
+	req := gqlclient.NewRequest(getLogsQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var logs GetLogs
+	err = json.Unmarshal(jsonStr, &logs)
+	if err != nil {
+		return nil, err
+	}
+	return logs.Responses, nil
+}
+
+func (c *Client) GetStorageAt(ctx context.Context, hash common.Hash, address common.Address, slot string) (*StorageResponse, error) {
+	getLogsQuery := fmt.Sprintf(`
+		query{
+			getStorageAt(blockHash: "%s", contract: "%s",slot: "%s") {
+				cid
+				value
+				ipldBlock
+			}
+		}
+	`, hash.String(), address.String(), common.HexToHash(slot))
+
+	req := gqlclient.NewRequest(getLogsQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var storageAt GetStorageAt
+	err = json.Unmarshal(jsonStr, &storageAt)
+	if err != nil {
+		return nil, err
+	}
+	return &storageAt.Response, nil
+}
+
+func (c *Client) AllEthHeaderCIDs(ctx context.Context, condition EthHeaderCIDCondition) (*AllEthHeaderCIDsResponse, error) {
+	var params string
+	if condition.BlockHash != nil {
+		params = fmt.Sprintf(`blockHash: "%s"`, *condition.BlockHash)
+	}
+	if condition.BlockNumber != nil {
+		params += fmt.Sprintf(`blockNumber: "%s"`, condition.BlockNumber.String())
+	}
+
+	getHeadersQuery := fmt.Sprintf(`
+		query{
+			allEthHeaderCids(condition: { %s }) {
+				nodes {
+					cid
+					blockNumber
+					blockHash
+					parentHash
+					timestamp
+					stateRoot
+					td
+					txRoot
+					receiptRoot
+					uncleRoot
+					bloom
+					blockByCid {
+						key
+						data
+					}
+					ethTransactionCidsByHeaderId {
+						nodes {
+							cid
+							txHash
+							index
+							src
+							dst
+						}
+					}
+				}
+			}
+		}
+	`, params)
+
+	req := gqlclient.NewRequest(getHeadersQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var allEthHeaderCIDs AllEthHeaderCIDs
+	err = json.Unmarshal(jsonStr, &allEthHeaderCIDs)
+	if err != nil {
+		return nil, err
+	}
+	return &allEthHeaderCIDs.Response, nil
+}
+
+func (c *Client) EthTransactionCIDByTxHash(ctx context.Context, txHash string) (*EthTransactionCIDResponse, error) {
+	getTxQuery := fmt.Sprintf(`
+		query{
+			ethTransactionCidByTxHash(txHash: "%s") {
+				cid
+				txHash
+				index
+				src
+				dst
+				blockByCid {
+					data
+				}
+			}
+		}
+	`, txHash)
+
+	req := gqlclient.NewRequest(getTxQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var ethTxCID EthTransactionCIDByTxHash
+	err = json.Unmarshal(jsonStr, &ethTxCID)
+	if err != nil {
+		return nil, err
+	}
+	return &ethTxCID.Response, nil
+}

pkg/graphql/graphiql.go

@@ -0,0 +1,114 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+)
+
+// GraphiQL is an in-browser IDE for exploring GraphiQL APIs.
+// This handler returns GraphiQL when requested.
+//
+// For more information, see https://github.com/graphql/graphiql.
+type GraphiQL struct{}
+
+func respond(w http.ResponseWriter, body []byte, code int) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.WriteHeader(code)
+	_, _ = w.Write(body)
+}
+
+func errorJSON(msg string) []byte {
+	buf := bytes.Buffer{}
+	fmt.Fprintf(&buf, `{"error": "%s"}`, msg)
+	return buf.Bytes()
+}
+
+func (h GraphiQL) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		respond(w, errorJSON("only GET requests are supported"), http.StatusMethodNotAllowed)
+		return
+	}
+	w.Header().Set("Content-Type", "text/html")
+	w.Write(graphiql)
+}
+
+var graphiql = []byte(`
+<!DOCTYPE html>
+<html>
+	<head>
+		<link
+                rel="icon"
+                type="image/png"
+                href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAAlwSFlzAAALEwAACxMBAJqcGAAAActpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx4bXA6Q3JlYXRvclRvb2w+QWRvYmUgSW1hZ2VSZWFkeTwveG1wOkNyZWF0b3JUb29sPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaWVudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KKS7NPQAAB5FJREFUWAm1FmtsnEdxdr/vfC8/mpgEfHYa6gaUJqAihfhVO7UprSokHsn5jKgKiKLGIIEEbSlpJdQLIJw+UFFQUSuBWir1z9nnpEmgCUnkcxPSmDRCgkKpGoJpfXdxHSc4ftzr2x1m9rvPPQdDDSgrfd/O7szOe2YX4H8cGEtY3tFK2Nu7pjMCChbgzVfD11h4XLKAibahL6dbBv+SaRl6LUsw78XBxTG80mEsWSkxu1oM9qmJlkR7UPhPWSDJCzSISw5zXZGxvpMezUp5GmtWQszunpiAKiPPZ20KyCqY1/ncgs4v+IUPwLJvYhzTVIbmvXgvqwAxkImKJHt1yzM+AQLXvdKXy3QevB4R+3O6wIYHSUCIlABEtTO9bf86pmFa7B6xPeHMi3l668p5SQjInbRGQQw0E3FMH4FHaFPoP8USVaveEo9aaH3LsdRh2vsYKqwhMhRBKw82vGbNQbcC9ePL1+PDmwf7iix0N+xmPoafq4TgDDaRYxmLCrBwD5HpSK4vKRVeP9b3ZyaaaE18UaL4KYE5x5afsWxoBgefFfX+jX6pMH9RvSnX2v1YxPP4D3UAHG2hgm80vRp7ns9nWxOb8kIt3HD6C+O8rpRVoYCxHDOtQwOg4QHS1kIb9oHGVQJlN0h8qPF07FFmkG4byouAjEdSO/bwOntr8kGt8EeNJ3uN27O37fse5PT3lVIjUsrL6MB2IVCThMcbx3ofIt7sZeMFExeTubSR3Zq4tVoEdhHSJs30WqjbIS1Zk6/VqzzhmdbBpyn5p1g4W8LMGkajj9GUSfcM/4IVaji+/QdOa7hehKz69xEPsllLkFZY+HdlWhOdLNxrXm5iTK1xPSHEeo4KxTFPzEsFLHH8D914rG+GGWe2Dd9UJav6ZbW1k9ep7rgF3SnTEUXA3hko2fdkowc2M27dk3deomgfLBIPYlJytC4QLzKLZdAoy3QzNTVqksT2y6Oz+YVL1TK4Oo9FYAVIkRFzgH8F/bOiD0cjv4m+hEA9IdXn8HaC4Mjxzx7OdCZH8R14mra6eB9sfUKTj4SCQLUvCHMqN235rKMGV5ZpPCAoSzGOcs2JaFZYVuc8FF5XQl8uCHV75FT0ZT6Q6Ry+02fZ3b7agLF+MGbYmF/Mg+vE14NY1Xnhjv2fZkTkWO+R2VXqc1BrLczp/OtULV0fOLXjHS5LlvkuhzL05oZf+xnMbtv3BLXZIwyPQNx4iRLvrXRXci/vcV/guXJ4dZ/elnwqfctQlnFxoGyhkY2+eCbTlnyCYU8GwzzcHHBhmKl7261X1CEBaIT0QNxJdyQfpLRdHblt4wNMeuhsVpWPvDulqAXQKH5i9f0Ut7pMT/LhOEWc96hfkBEYYnhDU3DJ2SUKMAEPIagRoTSJObF9uF5oHAC/uF/ENxeRrPcai0vt/k1mE+6GeE9eVIlvQwF+yGfL/KiNuMpUnmF4WQUYwX3AEEzjXmqi5yOp6DO8hrM7TeIZ+Orf2X6DY1oU+FeY1D8xJLh8G2bcsgpQ3vqoAU1P3nWouQaDd8mQdS8Tj1B/Z0sZXm6QyxbvAFlj3Us95e7Jbx6/EYScpnP/kjfMwy3DMre6mXVGIVTqiqi1mtVk8blZR78UOdGbQqDLheLMjWc54Yt7KSAaUvRwTyrdMXREvFF6VtRZfgrALNOcm8ixZxe9uOgBLsMPnftUIdM+tBFKcLtwxCeJ7GbdHDJlJ6DHYetX8gHfSTTEB4P9WNBb5JRq0VrfwbxZRuVN61pMt56ICz3elWxAB18OS//Nep4MKeowTOU/zMwo8RaV5fVKhs4WN1DzCjkzJV1jBT9K1TB6oWN4bR89arDMz7iTa1ikepxsy+CXqmXol1fUfJ4qwUfeptsXL1JNTFNWXkfmO5ydi8KXBIMWvCYnmbOWmKXr5zpZhHotSbQGp9YO+qkb3h05E3vBk+nmwJopw5SSdVxRsOjiCGhEXSMCMFdTrAdbPikul35PvWAN1adPgqAGz8Kk1FLTX2hlCyF9pHSIQlwnp+x6/yb1t9zu8LgFszJHt5v0K+TakuPmbFnmog2cXBzfbFtyj1b6O4SQ4BP76Zr1k1Etwoe7Ir+N/dwcfo8f3QnbsYR7yAO/kxICdAH1En+km/WxhtPRXZ4sZrOoQBk2npjcmmwu2ipMz6s/MlG6JflVqrC9pN8VqLK+1nhix4u8/3Z7YjXPRHeJ52z3vm7Mq6eISa0UeF/DK7FB3r/w8eGP0Htg4f1noud5TXgy1g1lpQIGQelGyLjbQk3J7TZr8yT7uxzwSfu+oiwdIL//gTKc+4MUltxL/lpPFn+ebvqByFhswAjid+VgTLNnXcGcyHGuY7PmvWUHZ2hlqXgXDRNfbD/YSE+2MeeWYzjZMmw+p+MYpnuSJy/FjtZ5DCvPuI9SFv5/DI4buZxfwZBuH7pnpu0QprcOztM3N9v2K8x2DH+FcZktB/nSWeJZ3v93Y8VasRubmqBoGKF4g6oBwjIQoi/MMDrqHOMamnMFmv6ziw0T97diTb0zHB7OEe4ZlCjf5X2U8vGm09HnKrPbo78mMwu6mjFn9tV713TtvWpZSCX83wr9J1EKd8CrhC26AAAAAElFTkSuQmCC"
+        />
+        <link
+                rel="stylesheet"
+                href="https://cdnjs.cloudflare.com/ajax/libs/graphiql/0.13.0/graphiql.css"
+                integrity="sha384-Qua2xoKBxcHOg1ivsKWo98zSI5KD/UuBpzMIg8coBd4/jGYoxeozCYFI9fesatT0"
+                crossorigin="anonymous"
+        />
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/fetch/3.0.0/fetch.min.js"
+                integrity="sha384-5B8/4F9AQqp/HCHReGLSOWbyAOwnJsPrvx6C0+VPUr44Olzi99zYT1xbVh+ZanQJ"
+                crossorigin="anonymous"
+        ></script>
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/react/16.8.5/umd/react.production.min.js"
+                integrity="sha384-dOCiLz3nZfHiJj//EWxjwSKSC6Z1IJtyIEK/b/xlHVNdVLXDYSesoxiZb94bbuGE"
+                crossorigin="anonymous"
+        ></script>
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/react-dom/16.8.5/umd/react-dom.production.min.js"
+                integrity="sha384-QI+ql5f+khgo3mMdCktQ3E7wUKbIpuQo8S5rA/3i1jg2rMsloCNyiZclI7sFQUGN"
+                crossorigin="anonymous"
+        ></script>
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/graphiql/0.13.0/graphiql.min.js"
+                integrity="sha384-roSmzNmO4zJK9X4lwggDi4/oVy+9V4nlS1+MN8Taj7tftJy1GvMWyAhTNXdC/fFR"
+                crossorigin="anonymous"
+        ></script>
+	</head>
+	<body style="width: 100%; height: 100%; margin: 0; overflow: hidden;">
+		<div id="graphiql" style="height: 100vh;">Loading...</div>
+		<script>
+			function fetchGQL(params) {
+				return fetch("/graphql", {
+					method: "post",
+					body: JSON.stringify(params),
+					credentials: "include",
+				}).then(function (resp) {
+					return resp.text();
+				}).then(function (body) {
+					try {
+						return JSON.parse(body);
+					} catch (error) {
+						return body;
+					}
+				});
+			}
+			ReactDOM.render(
+				React.createElement(GraphiQL, {fetcher: fetchGQL}),
+				document.getElementById("graphiql")
+			)
+		</script>
+	</body>
+</html>
+`)

pkg/graphql/graphql_suite_test.go

@@ -0,0 +1,29 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestGraphQL(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "graphql test suite")
+}

pkg/graphql/graphql_test.go

@@ -0,0 +1,359 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql_test
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"strconv"
+
+	statediff "github.com/cerc-io/plugeth-statediff"
+	"github.com/cerc-io/plugeth-statediff/adapt"
+	"github.com/cerc-io/plugeth-statediff/indexer/ipld"
+	sdtypes "github.com/cerc-io/plugeth-statediff/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/graphql"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
+)
+
+const (
+	gqlEndPoint = "127.0.0.1:8083"
+)
+
+var (
+	randomAddr              = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
+	randomHash              = crypto.Keccak256Hash(randomAddr.Bytes())
+	blocks                  []*types.Block
+	receipts                []types.Receipts
+	chain                   *core.BlockChain
+	db                      *sqlx.DB
+	backend                 *eth.Backend
+	graphQLServer           *graphql.Service
+	chainConfig             = &*params.TestChainConfig
+	client                  = graphql.NewClient(fmt.Sprintf("http://%s/graphql", gqlEndPoint))
+	mockTD                  = big.NewInt(1337)
+	ctx                     = context.Background()
+	nonCanonBlockHash       common.Hash
+	nonCanonContractAddress common.Address
+)
+
+var _ = BeforeSuite(func() {
+	var err error
+	db = shared.SetupDB()
+
+	backend, err = eth.NewEthBackend(db, &eth.Config{
+		ChainConfig: chainConfig,
+		VMConfig:    vm.Config{},
+		RPCGasCap:   big.NewInt(10000000000),
+		GroupCacheConfig: &shared.GroupCacheConfig{
+			StateDB: shared.GroupConfig{
+				Name:                   "graphql_test",
+				CacheSizeInMB:          8,
+				CacheExpiryInMins:      60,
+				LogStatsIntervalInSecs: 0,
+			},
+		},
+	})
+	Expect(err).ToNot(HaveOccurred())
+
+	// make the test blockchain (and state)
+	chainConfig.LondonBlock = big.NewInt(100)
+	blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen, chainConfig)
+	indexer := shared.SetupTestStateDiffIndexer(context.Background(), chainConfig, test_helpers.Genesis.Hash())
+	builder := statediff.NewBuilder(adapt.GethStateView(chain.StateCache()))
+
+	// Insert some non-canonical data into the database so that we test our ability to discern canonicity
+	nonCanonBlockHash = test_helpers.MockBlock.Hash()
+	nonCanonContractAddress = test_helpers.ContractAddr
+
+	tx, err := indexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+	Expect(err).ToNot(HaveOccurred())
+	defer tx.RollbackOnFailure(err)
+
+	err = tx.Submit()
+	Expect(err).ToNot(HaveOccurred())
+
+	// The non-canonical header has a child
+	tx, err = indexer.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
+	Expect(err).ToNot(HaveOccurred())
+	defer tx.RollbackOnFailure(err)
+
+	ipld := sdtypes.IPLD{
+		CID:     ipld.Keccak256ToCid(ipld.RawBinary, test_helpers.CodeHash.Bytes()).String(),
+		Content: test_helpers.ContractCode,
+	}
+	err = indexer.PushIPLD(tx, ipld)
+	Expect(err).ToNot(HaveOccurred())
+
+	err = tx.Submit()
+	Expect(err).ToNot(HaveOccurred())
+
+	// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
+	for i, block := range blocks {
+		var args statediff.Args
+		var rcts types.Receipts
+		if i == 0 {
+			args = statediff.Args{
+				OldStateRoot: common.Hash{},
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+		} else {
+			args = statediff.Args{
+				OldStateRoot: blocks[i-1].Root(),
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+			rcts = receipts[i-1]
+		}
+
+		tx, err = indexer.PushBlock(block, rcts, mockTD)
+		Expect(err).ToNot(HaveOccurred())
+		defer tx.RollbackOnFailure(err)
+
+		diff, err := builder.BuildStateDiffObject(args, statediff.Params{})
+		Expect(err).ToNot(HaveOccurred())
+
+		for _, node := range diff.Nodes {
+			err = indexer.PushStateNode(tx, node, block.Hash().String())
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		for _, ipld := range diff.IPLDs {
+			err = indexer.PushIPLD(tx, ipld)
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		err = tx.Submit()
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	graphQLServer, err = graphql.New(backend, gqlEndPoint, nil, []string{"*"}, rpc.HTTPTimeouts{})
+	Expect(err).ToNot(HaveOccurred())
+
+	err = graphQLServer.Start(nil)
+	Expect(err).ToNot(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+	err := graphQLServer.Stop()
+	Expect(err).ToNot(HaveOccurred())
+	shared.TearDownDB(db)
+	chain.Stop()
+})
+
+var _ = Describe("GraphQL", func() {
+	Describe("eth_getLogs", func() {
+		It("Retrieves logs that matches the provided blockHash and contract address", func() {
+			logs, err := client.GetLogs(ctx, nonCanonBlockHash, []common.Address{nonCanonContractAddress})
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedLogs := []graphql.LogResponse{
+				{
+					Topics:      test_helpers.MockLog1.Topics,
+					Data:        hexutil.Bytes(test_helpers.MockLog1.Data),
+					Transaction: graphql.TransactionResponse{Hash: test_helpers.MockTransactions[0].Hash()},
+					ReceiptCID:  test_helpers.Rct1CID.String(),
+					Status:      int32(test_helpers.MockReceipts[0].Status),
+				},
+			}
+
+			Expect(logs).To(Equal(expectedLogs))
+		})
+
+		It("Retrieves logs for the failed receipt status that matches the provided blockHash and another contract address", func() {
+			logs, err := client.GetLogs(ctx, nonCanonBlockHash, []common.Address{test_helpers.AnotherAddress2})
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedLogs := []graphql.LogResponse{
+				{
+					Topics:      test_helpers.MockLog6.Topics,
+					Data:        hexutil.Bytes(test_helpers.MockLog6.Data),
+					Transaction: graphql.TransactionResponse{Hash: test_helpers.MockTransactions[3].Hash()},
+					ReceiptCID:  test_helpers.Rct4CID.String(),
+					Status:      int32(test_helpers.MockReceipts[3].Status),
+				},
+			}
+
+			Expect(logs).To(Equal(expectedLogs))
+		})
+
+		It("Retrieves logs that matches the provided blockHash and multiple contract addresses", func() {
+			logs, err := client.GetLogs(ctx, nonCanonBlockHash, []common.Address{nonCanonContractAddress, test_helpers.AnotherAddress2})
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedLogs := []graphql.LogResponse{
+				{
+					Topics:      test_helpers.MockLog1.Topics,
+					Data:        hexutil.Bytes(test_helpers.MockLog1.Data),
+					Transaction: graphql.TransactionResponse{Hash: test_helpers.MockTransactions[0].Hash()},
+					ReceiptCID:  test_helpers.Rct1CID.String(),
+					Status:      int32(test_helpers.MockReceipts[0].Status),
+				},
+				{
+					Topics:      test_helpers.MockLog6.Topics,
+					Data:        hexutil.Bytes(test_helpers.MockLog6.Data),
+					Transaction: graphql.TransactionResponse{Hash: test_helpers.MockTransactions[3].Hash()},
+					ReceiptCID:  test_helpers.Rct4CID.String(),
+					Status:      int32(test_helpers.MockReceipts[3].Status),
+				},
+			}
+
+			Expect(logs).To(Equal(expectedLogs))
+		})
+
+		It("Retrieves all the logs for the receipt that matches the provided blockHash and nil contract address", func() {
+			logs, err := client.GetLogs(ctx, nonCanonBlockHash, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(6))
+		})
+
+		It("Retrieves logs with random hash", func() {
+			logs, err := client.GetLogs(ctx, randomHash, []common.Address{nonCanonContractAddress})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(0))
+		})
+	})
+
+	Describe("eth_getStorageAt", func() {
+		It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash", func() {
+			storageRes, err := client.GetStorageAt(ctx, blocks[2].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.HexToHash("01")))
+
+			storageRes, err = client.GetStorageAt(ctx, blocks[3].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.HexToHash("03")))
+
+			storageRes, err = client.GetStorageAt(ctx, blocks[4].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.HexToHash("09")))
+		})
+
+		It("Retrieves empty data if it tries to access a contract at a blockHash which does not exist", func() {
+			storageRes, err := client.GetStorageAt(ctx, blocks[0].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.Hash{}))
+
+			storageRes, err = client.GetStorageAt(ctx, blocks[1].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.Hash{}))
+		})
+
+		It("Retrieves empty data if it tries to access a contract slot which does not exist", func() {
+			storageRes, err := client.GetStorageAt(ctx, blocks[3].Hash(), nonCanonContractAddress, randomHash.Hex())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.Hash{}))
+		})
+	})
+
+	Describe("allEthHeaderCids", func() {
+		It("Retrieves header_cids that matches the provided blockNumber", func() {
+			allEthHeaderCIDsResp, err := client.AllEthHeaderCIDs(ctx, graphql.EthHeaderCIDCondition{BlockNumber: new(graphql.BigInt).SetUint64(2)})
+			Expect(err).ToNot(HaveOccurred())
+
+			headerCIDs, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockNumber(2)
+			Expect(err).ToNot(HaveOccurred())
+
+			for idx, headerCID := range headerCIDs {
+				ethHeaderCID := allEthHeaderCIDsResp.Nodes[idx]
+
+				compareEthHeaderCID(ethHeaderCID, headerCID)
+			}
+		})
+
+		It("Retrieves header_cids that matches the provided blockHash", func() {
+			blockHash := blocks[1].Hash().String()
+			allEthHeaderCIDsResp, err := client.AllEthHeaderCIDs(ctx, graphql.EthHeaderCIDCondition{BlockHash: &blockHash})
+			Expect(err).ToNot(HaveOccurred())
+
+			headerCID, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(blocks[1].Hash(), nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(allEthHeaderCIDsResp.Nodes)).To(Equal(1))
+			ethHeaderCID := allEthHeaderCIDsResp.Nodes[0]
+			compareEthHeaderCID(ethHeaderCID, headerCID)
+		})
+	})
+
+	Describe("ethTransactionCidByTxHash", func() {
+		It("Retrieves tx_cid that matches the provided txHash", func() {
+			txHash := blocks[2].Transactions()[0].Hash().String()
+			ethTransactionCIDResp, err := client.EthTransactionCIDByTxHash(ctx, txHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			txCID, err := backend.Retriever.RetrieveTxCIDByHash(txHash, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			compareEthTxCID(*ethTransactionCIDResp, txCID)
+
+			Expect(ethTransactionCIDResp.BlockByCid.Data).To(Equal(graphql.Bytes(txCID.IPLD.Data).String()))
+		})
+	})
+})
+
+func compareEthHeaderCID(ethHeaderCID graphql.EthHeaderCIDResponse, headerCID eth.HeaderCIDRecord) {
+	blockNumber, err := strconv.ParseInt(headerCID.BlockNumber, 10, 64)
+	Expect(err).ToNot(HaveOccurred())
+
+	td, err := strconv.ParseInt(headerCID.TotalDifficulty, 10, 64)
+	Expect(err).ToNot(HaveOccurred())
+
+	Expect(ethHeaderCID.CID).To(Equal(headerCID.CID))
+	Expect(ethHeaderCID.BlockNumber).To(Equal(*new(graphql.BigInt).SetUint64(uint64(blockNumber))))
+	Expect(ethHeaderCID.BlockHash).To(Equal(headerCID.BlockHash))
+	Expect(ethHeaderCID.ParentHash).To(Equal(headerCID.ParentHash))
+	Expect(ethHeaderCID.Timestamp).To(Equal(*new(graphql.BigInt).SetUint64(headerCID.Timestamp)))
+	Expect(ethHeaderCID.StateRoot).To(Equal(headerCID.StateRoot))
+	Expect(ethHeaderCID.Td).To(Equal(*new(graphql.BigInt).SetUint64(uint64(td))))
+	Expect(ethHeaderCID.TxRoot).To(Equal(headerCID.TxRoot))
+	Expect(ethHeaderCID.ReceiptRoot).To(Equal(headerCID.RctRoot))
+	Expect(ethHeaderCID.UncleRoot).To(Equal(headerCID.UnclesHash))
+	Expect(ethHeaderCID.Bloom).To(Equal(graphql.Bytes(headerCID.Bloom).String()))
+
+	for tIdx, txCID := range headerCID.TransactionCIDs {
+		ethTxCID := ethHeaderCID.EthTransactionCIDsByHeaderId.Nodes[tIdx]
+		compareEthTxCID(ethTxCID, txCID)
+	}
+
+	Expect(ethHeaderCID.BlockByCid.Data).To(Equal(graphql.Bytes(headerCID.IPLD.Data).String()))
+	Expect(ethHeaderCID.BlockByCid.Key).To(Equal(headerCID.IPLD.Key))
+}
+
+func compareEthTxCID(ethTxCID graphql.EthTransactionCIDResponse, txCID eth.TransactionCIDRecord) {
+	Expect(ethTxCID.CID).To(Equal(txCID.CID))
+	Expect(ethTxCID.TxHash).To(Equal(txCID.TxHash))
+	Expect(ethTxCID.Index).To(Equal(int32(txCID.Index)))
+	Expect(ethTxCID.Src).To(Equal(txCID.Src))
+	Expect(ethTxCID.Dst).To(Equal(txCID.Dst))
+}

pkg/graphql/schema.go

@@ -0,0 +1,354 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+const schema string = `
+    # Bytes32 is a 32 byte binary string, represented as 0x-prefixed hexadecimal.
+    scalar Bytes32
+    # Address is a 20 byte Ethereum address, represented as 0x-prefixed hexadecimal.
+    scalar Address
+    # Bytes is an arbitrary length binary string, represented as 0x-prefixed hexadecimal.
+    # An empty byte string is represented as '0x'. Byte strings must have an even number of hexadecimal nybbles.
+    scalar Bytes
+    # BigInt is a large integer. Input is accepted as either a JSON number or as a string.
+    # Input and output strings may be either decimal or 0x-prefixed hexadecimal depending upon the resolver implementation.
+    scalar BigInt
+    # Long is a 64 bit unsigned integer.
+    scalar Long
+
+    schema {
+        query: Query
+    }
+
+    # Account is an Ethereum account at a particular block.
+    type Account {
+        # Address is the address owning the account.
+        address: Address!
+        # Balance is the balance of the account, in wei.
+        balance: BigInt!
+        # TransactionCount is the number of transactions sent from this account,
+        # or in the case of a contract, the number of contracts created. Otherwise
+        # known as the nonce.
+        transactionCount: Long!
+        # Code contains the smart contract code for this account, if the account
+        # is a (non-self-destructed) contract.
+        code: Bytes!
+        # Storage provides access to the storage of a contract account, indexed
+        # by its 32 byte slot identifier.
+        storage(slot: Bytes32!): Bytes32!
+    }
+
+    # Log is an Ethereum event log.
+    type Log {
+        # Index is the index of this log in the block.
+        index: Int!
+        # Account is the account which generated this log - this will always
+        # be a contract account.
+        account(block: Long): Account!
+        # Topics is a list of 0-4 indexed topics for the log.
+        topics: [Bytes32!]!
+        # Data is unindexed data for this log.
+        data: Bytes!
+        # Transaction is the transaction that generated this log entry.
+        transaction: Transaction
+
+        # CID for the leaf node IPLD block of the log.
+        cid: String!
+
+        # ReceiptCID for the Receipt IPLD block this Log exists in.
+        receiptCID: String!
+
+        # IPLD block data for the Log Leaf node.
+        ipldBlock: Bytes!
+
+        # Status of the Receipt IPLD block this Log exists in.
+        status: Int!
+    }
+
+    # Transaction is an Ethereum transaction.
+    type Transaction {
+        # Hash is the hash of this transaction.
+        hash: Bytes32!
+        # Nonce is the nonce of the account this transaction was generated with.
+        nonce: Long!
+        # Index is the index of this transaction in the parent block. This will
+        # be null if the transaction has not yet been mined.
+        index: Int
+        # From is the account that sent this transaction - this will always be
+        # an externally owned account.
+        from(block: Long): Account!
+        # To is the account the transaction was sent to. This is null for
+        # contract-creating transactions.
+        to(block: Long): Account
+        # Value is the value, in wei, sent along with this transaction.
+        value: BigInt!
+        # GasPrice is the price offered to miners for gas, in wei per unit.
+        gasPrice: BigInt!
+        # Gas is the maximum amount of gas this transaction can consume.
+        gas: Long!
+        # InputData is the data supplied to the target of the transaction.
+        inputData: Bytes!
+        # Block is the block this transaction was mined in. This will be null if
+        # the transaction has not yet been mined.
+        block: Block
+
+        # Status is the return status of the transaction. This will be 1 if the
+        # transaction succeeded, or 0 if it failed (due to a revert, or due to
+        # running out of gas). If the transaction has not yet been mined, this
+        # field will be null.
+        status: Long
+        # GasUsed is the amount of gas that was used processing this transaction.
+        # If the transaction has not yet been mined, this field will be null.
+        gasUsed: Long
+        # CumulativeGasUsed is the total gas used in the block up to and including
+        # this transaction. If the transaction has not yet been mined, this field
+        # will be null.
+        cumulativeGasUsed: Long
+        # CreatedContract is the account that was created by a contract creation
+        # transaction. If the transaction was not a contract creation transaction,
+        # or it has not yet been mined, this field will be null.
+        createdContract(block: Long): Account
+        # Logs is a list of log entries emitted by this transaction. If the
+        # transaction has not yet been mined, this field will be null.
+        logs: [Log!]
+        r: BigInt!
+        s: BigInt!
+        v: BigInt!
+    }
+
+    # BlockFilterCriteria encapsulates log filter criteria for a filter applied
+    # to a single block.
+    input BlockFilterCriteria {
+        # Addresses is list of addresses that are of interest. If this list is
+        # empty, results will not be filtered by address.
+        addresses: [Address!]
+        # Topics list restricts matches to particular event topics. Each event has a list
+        # of topics. Topics matches a prefix of that list. An empty element array matches any
+        # topic. Non-empty elements represent an alternative that matches any of the
+        # contained topics.
+        #
+        # Examples:
+        #  - [] or nil          matches any topic list
+        #  - [[A]]              matches topic A in first position
+        #  - [[], [B]]          matches any topic in first position, B in second position
+        #  - [[A], [B]]         matches topic A in first position, B in second position
+        #  - [[A, B]], [C, D]]  matches topic (A OR B) in first position, (C OR D) in second position
+        topics: [[Bytes32!]!]
+    }
+
+    # Block is an Ethereum block.
+    type Block {
+        # Number is the number of this block, starting at 0 for the genesis block.
+        number: Long!
+        # Hash is the block hash of this block.
+        hash: Bytes32!
+        # Parent is the parent block of this block.
+        parent: Block
+        # Nonce is the block nonce, an 8 byte sequence determined by the miner.
+        nonce: Bytes!
+        # TransactionsRoot is the keccak256 hash of the root of the trie of transactions in this block.
+        transactionsRoot: Bytes32!
+        # TransactionCount is the number of transactions in this block. if
+        # transactions are not available for this block, this field will be null.
+        transactionCount: Int
+        # StateRoot is the keccak256 hash of the state trie after this block was processed.
+        stateRoot: Bytes32!
+        # ReceiptsRoot is the keccak256 hash of the trie of transaction receipts in this block.
+        receiptsRoot: Bytes32!
+        # Miner is the account that mined this block.
+        miner(block: Long): Account!
+        # ExtraData is an arbitrary data field supplied by the miner.
+        extraData: Bytes!
+        # GasLimit is the maximum amount of gas that was available to transactions in this block.
+        gasLimit: Long!
+        # GasUsed is the amount of gas that was used executing transactions in this block.
+        gasUsed: Long!
+        # Timestamp is the unix timestamp at which this block was mined.
+        timestamp: Long!
+        # LogsBloom is a bloom filter that can be used to check if a block may
+        # contain log entries matching a filter.
+        logsBloom: Bytes!
+        # MixHash is the hash that was used as an input to the PoW process.
+        mixHash: Bytes32!
+        # Difficulty is a measure of the difficulty of mining this block.
+        difficulty: BigInt!
+        # TotalDifficulty is the sum of all difficulty values up to and including
+        # this block.
+        totalDifficulty: BigInt!
+        # OmmerCount is the number of ommers (AKA uncles) associated with this
+        # block. If ommers are unavailable, this field will be null.
+        ommerCount: Int
+        # Ommers is a list of ommer (AKA uncle) blocks associated with this block.
+        # If ommers are unavailable, this field will be null. Depending on your
+        # node, the transactions, transactionAt, transactionCount, ommers,
+        # ommerCount and ommerAt fields may not be available on any ommer blocks.
+        ommers: [Block]
+        # OmmerAt returns the ommer (AKA uncle) at the specified index. If ommers
+        # are unavailable, or the index is out of bounds, this field will be null.
+        ommerAt(index: Int!): Block
+        # OmmerHash is the keccak256 hash of all the ommers (AKA uncles)
+        # associated with this block.
+        ommerHash: Bytes32!
+        # Transactions is a list of transactions associated with this block. If
+        # transactions are unavailable for this block, this field will be null.
+        transactions: [Transaction!]
+        # TransactionAt returns the transaction at the specified index. If
+        # transactions are unavailable for this block, or if the index is out of
+        # bounds, this field will be null.
+        transactionAt(index: Int!): Transaction
+        # Logs returns a filtered set of logs from this block.
+        logs(filter: BlockFilterCriteria!): [Log!]!
+        # Account fetches an Ethereum account at the current block's state.
+        account(address: Address!): Account!
+        # Call executes a local call operation at the current block's state.
+        call(data: CallData!): CallResult
+    }
+
+    # CallData represents the data associated with a local contract call.
+    # All fields are optional.
+    input CallData {
+        # From is the address making the call.
+        from: Address
+        # To is the address the call is sent to.
+        to: Address
+        # Gas is the amount of gas sent with the call.
+        gas: Long
+        # GasPrice is the price, in wei, offered for each unit of gas.
+        gasPrice: BigInt
+        # Value is the value, in wei, sent along with the call.
+        value: BigInt
+        # Data is the data sent to the callee.
+        data: Bytes
+    }
+
+    # CallResult is the result of a local call operation.
+    type CallResult {
+        # Data is the return data of the called contract.
+        data: Bytes!
+        # GasUsed is the amount of gas used by the call, after any refunds.
+        gasUsed: Long!
+        # Status is the result of the call - 1 for success or 0 for failure.
+        status: Long!
+    }
+
+    # FilterCriteria encapsulates log filter criteria for searching log entries.
+    input FilterCriteria {
+        # FromBlock is the block at which to start searching, inclusive. Defaults
+        # to the latest block if not supplied.
+        fromBlock: Long
+        # ToBlock is the block at which to stop searching, inclusive. Defaults
+        # to the latest block if not supplied.
+        toBlock: Long
+        # Addresses is a list of addresses that are of interest. If this list is
+        # empty, results will not be filtered by address.
+        addresses: [Address!]
+        # Topics list restricts matches to particular event topics. Each event has a list
+        # of topics. Topics matches a prefix of that list. An empty element array matches any
+        # topic. Non-empty elements represent an alternative that matches any of the
+        # contained topics.
+        #
+        # Examples:
+        #  - [] or nil          matches any topic list
+        #  - [[A]]              matches topic A in first position
+        #  - [[], [B]]          matches any topic in first position, B in second position
+        #  - [[A], [B]]         matches topic A in first position, B in second position
+        #  - [[A, B]], [C, D]]  matches topic (A OR B) in first position, (C OR D) in second position
+        topics: [[Bytes32!]!]
+    }
+
+    # Storage trie value with IPLD data.
+    type StorageResult {
+        value: Bytes32!
+
+        # CID for the storage trie IPLD block.
+        cid: String!
+
+        # Storage trie IPLD block.
+        ipldBlock: Bytes!
+    }
+
+    input EthHeaderCidCondition {
+        blockNumber: BigInt
+        blockHash: String
+    }
+
+    type EthTransactionCid {
+        cid: String!
+        txHash: String!
+        index: Int!
+        src: String!
+        dst: String!
+        blockByCid: IPFSBlock!
+    }
+
+    type EthTransactionCidsConnection {
+        nodes: [EthTransactionCid]!
+    }
+
+    type IPFSBlock {
+        key: String!
+        data: String!
+    }
+
+    type EthHeaderCid {
+        cid: String!
+        blockNumber: BigInt!
+        blockHash: String!
+        parentHash: String!
+        timestamp: BigInt!
+        stateRoot: String!
+        td: BigInt!
+        txRoot: String!
+        receiptRoot: String!
+        uncleRoot: String!
+        bloom: String!
+        ethTransactionCidsByHeaderId: EthTransactionCidsConnection!
+        blockByCid: IPFSBlock!
+    }
+
+    type EthHeaderCidsConnection {
+        nodes: [EthHeaderCid]!
+    }
+
+    type Query {
+        # Block fetches an Ethereum block by number or by hash. If neither is
+        # supplied, the most recent known block is returned.
+        block(number: Long, hash: Bytes32): Block
+
+        # Blocks returns all the blocks between two numbers, inclusive. If
+        # to is not supplied, it defaults to the most recent known block.
+        blocks(from: Long!, to: Long): [Block!]!
+
+        # Transaction returns a transaction specified by its hash.
+        transaction(hash: Bytes32!): Transaction
+
+        # Logs returns log entries matching the provided filter.
+        logs(filter: FilterCriteria!): [Log!]!
+
+        # Get storage slot by block hash and contract address.
+        getStorageAt(blockHash: Bytes32!, contract: Address!, slot: Bytes32!): StorageResult
+
+        # Get contract logs by block hash and contract address.
+        getLogs(blockHash: Bytes32!, blockNumber: BigInt, addresses: [Address!]): [Log!]
+
+        # PostGraphile alternative to get headers with transactions using block number or block hash.
+        allEthHeaderCids(condition: EthHeaderCidCondition): EthHeaderCidsConnection
+
+        # PostGraphile alternative to get transactions using transaction hash.
+        ethTransactionCidByTxHash(txHash: String!, blockNumber: BigInt): EthTransactionCid
+    }
+`

pkg/graphql/service.go

@@ -0,0 +1,111 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/graph-gophers/graphql-go"
+	"github.com/graph-gophers/graphql-go/relay"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+)
+
+// Service encapsulates a GraphQL service.
+type Service struct {
+	endpoint string           // The host:port endpoint for this service.
+	cors     []string         // Allowed CORS domains
+	vhosts   []string         // Recognised vhosts
+	timeouts rpc.HTTPTimeouts // Timeout settings for HTTP requests.
+	backend  *eth.Backend     // The backend that queries will operate onn.
+	handler  http.Handler     // The `http.Handler` used to answer queries.
+	listener net.Listener     // The listening socket.
+}
+
+// New constructs a new GraphQL service instance.
+func New(backend *eth.Backend, endpoint string, cors, vhosts []string, timeouts rpc.HTTPTimeouts) (*Service, error) {
+	return &Service{
+		endpoint: endpoint,
+		cors:     cors,
+		vhosts:   vhosts,
+		timeouts: timeouts,
+		backend:  backend,
+	}, nil
+}
+
+// Protocols returns the list of protocols exported by this service.
+func (s *Service) Protocols() []p2p.Protocol { return nil }
+
+// APIs returns the list of APIs exported by this service.
+func (s *Service) APIs() []rpc.API { return nil }
+
+// Start is called after all services have been constructed and the networking
+// layer was also initialized to spawn any goroutines required by the service.
+func (s *Service) Start(server *p2p.Server) error {
+	var err error
+	s.handler, err = NewHandler(s.backend)
+	if err != nil {
+		return err
+	}
+
+	handler := node.NewHTTPHandlerStack(s.handler, s.cors, s.vhosts, nil)
+
+	// start http server
+	_, addr, err := node.StartHTTPEndpoint(s.endpoint, rpc.DefaultHTTPTimeouts, handler)
+	if err != nil {
+		utils.Fatalf("Could not start RPC api: %v", err)
+	}
+	extapiURL := fmt.Sprintf("http://%v", addr)
+	log.Infof("GraphQL endpoint opened at %s", extapiURL)
+	return nil
+}
+
+// NewHandler returns a new `http.Handler` that will answer GraphQL queries.
+// It additionally exports an interactive query browser on the / endpoint.
+func NewHandler(backend *eth.Backend) (http.Handler, error) {
+	q := Resolver{backend}
+
+	s, err := graphql.ParseSchema(schema, &q)
+	if err != nil {
+		return nil, err
+	}
+	h := &relay.Handler{Schema: s}
+
+	mux := http.NewServeMux()
+	mux.Handle("/", GraphiQL{})
+	mux.Handle("/graphql", h)
+	mux.Handle("/graphql/", h)
+	return mux, nil
+}
+
+// Stop terminates all goroutines belonging to the service, blocking until they
+// are all terminated.
+func (s *Service) Stop() error {
+	if s.listener != nil {
+		s.listener.Close()
+		s.listener = nil
+		log.Debugf("graphQL endpoint closed for url %s", fmt.Sprintf("http://%s", s.endpoint))
+	}
+	return nil
+}

pkg/graphql/types.go

@@ -0,0 +1,122 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+import (
+	"encoding/hex"
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+// Bytes marshals as a JSON string with \x prefix.
+// The empty slice marshals as "\x".
+type Bytes []byte
+
+// MarshalText implements encoding.TextMarshaler
+func (b Bytes) MarshalText() ([]byte, error) {
+	result := make([]byte, len(b)*2+2)
+	copy(result, `\x`)
+	hex.Encode(result[2:], b)
+	return result, nil
+}
+
+// String returns the hex encoding of b.
+func (b Bytes) String() string {
+	return b.encode()
+}
+
+// Encode encodes b as a hex string with "\x" prefix.
+// This is to make the output to be the same as given by postgraphile.
+// graphql-go prepends another "\" to the output resulting in prefix "\\x".
+func (b Bytes) encode() string {
+	result := make([]byte, len(b)*2+2)
+	copy(result, `\x`)
+	hex.Encode(result[2:], b)
+	return string(result)
+}
+
+type BigInt big.Int
+
+// ToInt converts b to a big.Int.
+func (b *BigInt) ToInt() *big.Int {
+	return (*big.Int)(b)
+}
+
+// String returns value of b as a decimal string.
+func (b *BigInt) String() string {
+	return b.ToInt().String()
+}
+
+// SetUint64 sets b to x and returns x.
+func (b *BigInt) SetUint64(x uint64) *BigInt {
+	var val big.Int
+	val.SetUint64(x)
+	*b = (BigInt)(val)
+	return b
+}
+
+// MarshalText implements encoding.TextMarshaler
+func (b BigInt) MarshalText() ([]byte, error) {
+	return []byte(b.String()), nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler
+func (b *BigInt) UnmarshalText(input []byte) error {
+	raw, err := checkNumberText(input)
+	if err != nil {
+		return err
+	}
+	if len(raw) > 64 {
+		return hexutil.ErrBig256Range
+	}
+
+	var val big.Int
+	val.SetString(string(input[:]), 10)
+	*b = (BigInt)(val)
+	return nil
+}
+
+// ImplementsGraphQLType returns true if BigInt implements the provided GraphQL type.
+func (b BigInt) ImplementsGraphQLType(name string) bool { return name == "BigInt" }
+
+// UnmarshalGraphQL unmarshals the provided GraphQL query data.
+func (b *BigInt) UnmarshalGraphQL(input interface{}) error {
+	var err error
+	switch input := input.(type) {
+	case string:
+		return b.UnmarshalText([]byte(input))
+	case int32:
+		var num big.Int
+		num.SetInt64(int64(input))
+		*b = BigInt(num)
+	default:
+		err = fmt.Errorf("unexpected type %T for BigInt", input)
+	}
+	return err
+}
+
+func checkNumberText(input []byte) (raw []byte, err error) {
+	if len(input) == 0 {
+		return nil, nil // empty strings are allowed
+	}
+	if len(input) > 1 && input[0] == '0' {
+		return nil, hexutil.ErrLeadingZero
+	}
+	return input, nil
+}

pkg/log/log.go

@@ -0,0 +1,244 @@
+// Copyright © 2023 Cerc
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package log
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+)
+
+const (
+	CtxKeyApiMethod   = "api_method"
+	CtxKeyApiParams   = "api_params"
+	CtxKeyApiReqId    = "api_reqid"
+	CtxKeyBlockHash   = "block_hash"
+	CtxKeyBlockNumber = "block_num"
+	CtxKeyConn        = "conn"
+	CtxKeyDuration    = "duration"
+	CtxKeyError       = "err"
+	CtxKeyUniqId      = "uuid"
+	CtxKeyUserId      = "user_id"
+)
+
+// TODO: Allow registering arbitrary keys.
+var registeredKeys = []string{
+	CtxKeyApiMethod,
+	CtxKeyApiParams,
+	CtxKeyApiReqId,
+	CtxKeyBlockHash,
+	CtxKeyBlockNumber,
+	CtxKeyConn,
+	CtxKeyDuration,
+	CtxKeyError,
+	CtxKeyUniqId,
+	CtxKeyUserId,
+}
+
+const FatalLevel = logrus.FatalLevel
+const ErrorLevel = logrus.ErrorLevel
+const InfoLevel = logrus.InfoLevel
+const DebugLevel = logrus.DebugLevel
+const TraceLevel = logrus.TraceLevel
+
+type Entry = logrus.Entry
+type Level = logrus.Level
+
+func WithFieldsFromContext(ctx context.Context) *Entry {
+	entry := logrus.WithContext(ctx)
+
+	for _, key := range registeredKeys {
+		if value := ctx.Value(key); value != nil {
+			entry = entry.WithField(key, value)
+		}
+	}
+	return entry
+}
+
+func Fatalx(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Fatal(args...)
+}
+
+func Errorx(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Error(args...)
+}
+
+func Warnx(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Warn(args...)
+}
+
+func Infox(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Info(args...)
+}
+
+func Debugx(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Debug(args...)
+}
+
+func Tracex(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Trace(args...)
+}
+
+func Errorxf(ctx context.Context, format string, args ...interface{}) {
+	WithFieldsFromContext(ctx).Errorf(format, args...)
+}
+
+func Warnxf(ctx context.Context, format string, args ...interface{}) {
+	WithFieldsFromContext(ctx).Warnf(format, args...)
+}
+
+func Infoxf(ctx context.Context, format string, args ...interface{}) {
+	WithFieldsFromContext(ctx).Infof(format, args...)
+}
+
+func Debugxf(ctx context.Context, format string, args ...interface{}) {
+	WithFieldsFromContext(ctx).Debugf(format, args...)
+}
+
+func Tracexf(ctx context.Context, format string, args ...interface{}) {
+	WithFieldsFromContext(ctx).Tracef(format, args...)
+}
+
+func Fatal(args ...interface{}) {
+	logrus.Fatal(args...)
+}
+
+func Error(args ...interface{}) {
+	logrus.Error(args...)
+}
+
+func Warn(args ...interface{}) {
+	logrus.Warn(args...)
+}
+
+func Info(args ...interface{}) {
+	logrus.Info(args...)
+}
+
+func Debug(args ...interface{}) {
+	logrus.Debug(args...)
+}
+
+func Trace(args ...interface{}) {
+	logrus.Trace(args...)
+}
+
+func Fatalf(format string, args ...interface{}) {
+	logrus.Fatalf(format, args...)
+}
+
+func Errorf(format string, args ...interface{}) {
+	logrus.Errorf(format, args...)
+}
+
+func Warnf(format string, args ...interface{}) {
+	logrus.Warnf(format, args...)
+}
+
+func Infof(format string, args ...interface{}) {
+	logrus.Infof(format, args...)
+}
+
+func Debugf(format string, args ...interface{}) {
+	logrus.Debugf(format, args...)
+}
+
+func Tracef(format string, args ...interface{}) {
+	logrus.Tracef(format, args...)
+}
+
+func SetOutput(out io.Writer) {
+	logrus.SetOutput(out)
+}
+
+func SetLevel(lvl Level) {
+	logrus.SetLevel(lvl)
+}
+
+func IsLevelEnabled(lvl Level) bool {
+	return logrus.IsLevelEnabled(lvl)
+}
+
+func WithError(err error) *Entry {
+	return logrus.WithError(err)
+}
+
+func WithField(field string, value interface{}) *Entry {
+	return logrus.WithField(field, value)
+}
+
+func Init() error {
+	// Set the output.
+	logFile := viper.GetString("log.file")
+	if logFile != "" {
+		file, err := os.OpenFile(logFile,
+			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0640)
+		if err == nil {
+			Infof("Directing output to %s", logFile)
+			SetOutput(file)
+		} else {
+			SetOutput(os.Stdout)
+			Info("Failed to logrus.to file, using default stdout")
+		}
+	} else {
+		SetOutput(os.Stdout)
+	}
+
+	// Set the level.
+	lvl, err := logrus.ParseLevel(viper.GetString("log.level"))
+	if err != nil {
+		return err
+	}
+	SetLevel(lvl)
+
+	formatter := &logrus.TextFormatter{
+		FullTimestamp: true,
+	}
+	// Show file/line number only at Trace level.
+	if lvl >= TraceLevel {
+		logrus.SetReportCaller(true)
+
+		// We need to exclude this wrapper code, logrus.us itself, and the runtime from the stack to show anything useful.
+		// cf. https://github.com/sirupsen/logrus.us/pull/973
+		formatter.CallerPrettyfier = func(frame *runtime.Frame) (function string, file string) {
+			pcs := make([]uintptr, 50)
+			_ = runtime.Callers(0, pcs)
+			frames := runtime.CallersFrames(pcs)
+
+			// Filter logrus.wrapper / logrus.us / runtime frames.
+			for next, again := frames.Next(); again; next, again = frames.Next() {
+				if !strings.Contains(next.File, "sirupsen/logrus.us") &&
+					!strings.HasPrefix(next.Function, "runtime.") &&
+					!strings.Contains(next.File, "ipld-eth-server/pkg/log") {
+					return next.Function, fmt.Sprintf("%s:%d", next.File, next.Line)
+				}
+			}
+
+			// Fallback to the raw info.
+			return frame.Function, fmt.Sprintf("%s:%d", frame.File, frame.Line)
+		}
+	}
+
+	logrus.SetFormatter(formatter)
+	Info("Log level set to ", lvl.String())
+	return nil
+}

pkg/net/api.go

@@ -0,0 +1,90 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// APIName is the namespace for the watcher's eth api
+const APIName = "net"
+
+// APIVersion is the version of the watcher's eth api
+const APIVersion = "0.0.1"
+
+// PublicNetAPI is the net nampespace API
+type PublicNetAPI struct {
+	// Proxy node for forwarding cache misses
+	networkVersion uint64
+	rpc            *rpc.Client
+	ethClient      *ethclient.Client
+}
+
+// NewPublicNetAPI creates a new PublicNetAPI with the provided underlying Backend
+func NewPublicNetAPI(networkID uint64, client *rpc.Client) *PublicNetAPI {
+	var ethClient *ethclient.Client
+	if client != nil {
+		ethClient = ethclient.NewClient(client)
+	}
+	return &PublicNetAPI{
+		networkVersion: networkID,
+		rpc:            client,
+		ethClient:      ethClient,
+	}
+}
+
+// Listening returns an indication if the node is listening for network connections.
+func (pna *PublicNetAPI) Listening() bool {
+	// in this case it is actually whether or not the proxied node is listening
+	if pna.rpc != nil {
+		var listening bool
+		if err := pna.rpc.Call(&listening, "net_listening"); err == nil {
+			return listening
+		}
+	}
+	return false
+}
+
+// PeerCount returns the number of connected peers
+func (pna *PublicNetAPI) PeerCount() hexutil.Uint {
+	// in this case it is actually the peer count of the proxied node
+	if pna.rpc != nil {
+		var num hexutil.Uint
+		if err := pna.rpc.Call(&num, "net_peerCount"); err == nil {
+			return num
+		}
+	}
+	return hexutil.Uint(0)
+}
+
+// Version returns the current ethereum protocol version.
+func (pna *PublicNetAPI) Version() string {
+	if pna.networkVersion != 0 {
+		return fmt.Sprintf("%d", pna.networkVersion)
+	}
+	if pna.rpc != nil {
+		var version string
+		if err := pna.rpc.Call(&version, "net_version"); err == nil {
+			return version
+		}
+	}
+	return ""
+}

pkg/net/api_test.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/net"
+)
+
+var _ = Describe("API", func() {
+	var (
+		api *net.PublicNetAPI
+	)
+	BeforeEach(func() {
+		api = net.NewPublicNetAPI(1, nil)
+	})
+	Describe("net_listening", func() {
+		It("Retrieves whether or not the node is listening to the p2p network", func() {
+			listening := api.Listening()
+			Expect(listening).To(BeFalse())
+		})
+	})
+
+	Describe("net_version", func() {
+		It("Retrieves the network id", func() {
+			version := api.Version()
+			Expect(version).To(Equal("1"))
+		})
+	})
+	// TODO: test PeerCount with mock proxy node
+})

pkg/net/net_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestNetSuite(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "eth ipld server net suite test")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

pkg/prom/db_stats_collector.go

@@ -0,0 +1,159 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"database/sql"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const subsystem = "connections"
+
+// DBStatsGetter is an interface that gets sql.DBStats.
+type DBStatsGetter interface {
+	Stats() sql.DBStats
+}
+
+// DBStatsCollector implements the prometheus.Collector interface.
+type DBStatsCollector struct {
+	sg DBStatsGetter
+
+	// descriptions of exported metrics
+	maxOpenDesc           *prometheus.Desc
+	openDesc              *prometheus.Desc
+	inUseDesc             *prometheus.Desc
+	idleDesc              *prometheus.Desc
+	waitedForDesc         *prometheus.Desc
+	blockedSecondsDesc    *prometheus.Desc
+	closedMaxIdleDesc     *prometheus.Desc
+	closedMaxLifetimeDesc *prometheus.Desc
+}
+
+// NewDBStatsCollector creates a new DBStatsCollector.
+func NewDBStatsCollector(dbName string, sg DBStatsGetter) *DBStatsCollector {
+	labels := prometheus.Labels{"db_name": dbName}
+	return &DBStatsCollector{
+		sg: sg,
+		maxOpenDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "max_open"),
+			"Maximum number of open connections to the database.",
+			nil,
+			labels,
+		),
+		openDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "open"),
+			"The number of established connections both in use and idle.",
+			nil,
+			labels,
+		),
+		inUseDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "in_use"),
+			"The number of connections currently in use.",
+			nil,
+			labels,
+		),
+		idleDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "idle"),
+			"The number of idle connections.",
+			nil,
+			labels,
+		),
+		waitedForDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "waited_for"),
+			"The total number of connections waited for.",
+			nil,
+			labels,
+		),
+		blockedSecondsDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "blocked_seconds"),
+			"The total time blocked waiting for a new connection.",
+			nil,
+			labels,
+		),
+		closedMaxIdleDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "closed_max_idle"),
+			"The total number of connections closed due to SetMaxIdleConns.",
+			nil,
+			labels,
+		),
+		closedMaxLifetimeDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "closed_max_lifetime"),
+			"The total number of connections closed due to SetConnMaxLifetime.",
+			nil,
+			labels,
+		),
+	}
+}
+
+// Describe implements the prometheus.Collector interface.
+func (c DBStatsCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.maxOpenDesc
+	ch <- c.openDesc
+	ch <- c.inUseDesc
+	ch <- c.idleDesc
+	ch <- c.waitedForDesc
+	ch <- c.blockedSecondsDesc
+	ch <- c.closedMaxIdleDesc
+	ch <- c.closedMaxLifetimeDesc
+}
+
+// Collect implements the prometheus.Collector interface.
+func (c DBStatsCollector) Collect(ch chan<- prometheus.Metric) {
+	stats := c.sg.Stats()
+
+	ch <- prometheus.MustNewConstMetric(
+		c.maxOpenDesc,
+		prometheus.GaugeValue,
+		float64(stats.MaxOpenConnections),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.openDesc,
+		prometheus.GaugeValue,
+		float64(stats.OpenConnections),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.inUseDesc,
+		prometheus.GaugeValue,
+		float64(stats.InUse),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.idleDesc,
+		prometheus.GaugeValue,
+		float64(stats.Idle),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.waitedForDesc,
+		prometheus.CounterValue,
+		float64(stats.WaitCount),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.blockedSecondsDesc,
+		prometheus.CounterValue,
+		stats.WaitDuration.Seconds(),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.closedMaxIdleDesc,
+		prometheus.CounterValue,
+		float64(stats.MaxIdleClosed),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.closedMaxLifetimeDesc,
+		prometheus.CounterValue,
+		float64(stats.MaxLifetimeClosed),
+	)
+}

pkg/prom/middleware.go

@@ -0,0 +1,146 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/google/uuid"
+
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+const (
+	jsonMethod               = "method"
+	jsonParams               = "params"
+	jsonReqId                = "id"
+	headerUserId             = "X-User-Id"
+	headerOriginalRemoteAddr = "X-Original-Remote-Addr"
+)
+
+// Peek at the request and update the Context accordingly (eg, API method, user ID, etc.)
+func preprocessRequest(r *http.Request) (*http.Request, error) {
+	// Generate a unique ID for this request.
+	uniqId, err := uuid.NewUUID()
+	if nil != err {
+		return nil, err
+	}
+
+	// Read the body so that we can peek inside.
+	body, err := io.ReadAll(r.Body)
+	if nil != err {
+		return nil, err
+	}
+
+	// Replace it with a re-readable copy.
+	r.Body = io.NopCloser(bytes.NewBuffer(body))
+
+	// All API requests should be JSON.
+	var result map[string]interface{}
+	if len(body) > 0 {
+		err = json.Unmarshal(body, &result)
+		if nil != err {
+			return nil, err
+		}
+	}
+
+	// Pull out the method name, request ID, user ID, and address info.
+	reqId := fmt.Sprintf("%g", result[jsonReqId])
+	reqMethod := fmt.Sprintf("%v", result[jsonMethod])
+	reqParams := fmt.Sprintf("%v", result[jsonParams])
+	// Truncate parameters unless trace logging is enabled.
+	if !log.IsLevelEnabled(log.TraceLevel) {
+		if len(reqParams) > 250 {
+			reqParams = reqParams[:250] + "..."
+		}
+	}
+	userId := r.Header.Get(headerUserId)
+	conn := r.Header.Get(headerOriginalRemoteAddr)
+	if len(conn) == 0 {
+		conn = r.RemoteAddr
+	}
+
+	// Add it all to the request context.
+	ctx := r.Context()
+	ctx = context.WithValue(ctx, log.CtxKeyUniqId, uniqId.String())
+	ctx = context.WithValue(ctx, log.CtxKeyApiMethod, reqMethod)
+	ctx = context.WithValue(ctx, log.CtxKeyApiParams, string(reqParams))
+	ctx = context.WithValue(ctx, log.CtxKeyApiReqId, reqId)
+	ctx = context.WithValue(ctx, log.CtxKeyUserId, userId)
+	ctx = context.WithValue(ctx, log.CtxKeyConn, conn)
+
+	return r.WithContext(ctx), nil
+}
+
+// HTTPMiddleware http connection metric reader
+func HTTPMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		r, err := preprocessRequest(r)
+		if nil != err {
+			log.WithError(err).Error("Error preprocessing request")
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		ctx := r.Context()
+		apiMethod := fmt.Sprintf("%s", ctx.Value(log.CtxKeyApiMethod))
+
+		if metrics {
+			httpCount.WithLabelValues(apiMethod).Inc()
+		}
+
+		log.Debugx(ctx, "START")
+		next.ServeHTTP(w, r)
+		duration := time.Now().Sub(start)
+		log.Debugxf(context.WithValue(ctx, log.CtxKeyDuration, duration.Milliseconds()), "END")
+
+		if metrics {
+			httpDuration.WithLabelValues(apiMethod).Observe(duration.Seconds())
+		}
+	})
+}
+
+// WSMiddleware websocket connection counter
+func WSMiddleware(next http.Handler) http.Handler {
+	if !metrics {
+		return next
+	}
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		wsCount.Inc()
+		next.ServeHTTP(w, r)
+		wsCount.Dec()
+	})
+}
+
+// IPCMiddleware unix-socket connection counter
+func IPCMiddleware(server *rpc.Server, client rpc.Conn) {
+	if metrics {
+		ipcCount.Inc()
+	}
+	server.ServeCodec(rpc.NewCodec(client), 0)
+	if metrics {
+		ipcCount.Dec()
+	}
+}

pkg/prom/prom.go

@@ -0,0 +1,80 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"github.com/jmoiron/sqlx"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const (
+	namespace = "ipld_eth_server"
+
+	subsystemHTTP = "http"
+	subsystemWS   = "ws"
+	subsystemIPC  = "ipc"
+)
+
+var (
+	metrics bool
+
+	httpCount    *prometheus.CounterVec
+	httpDuration *prometheus.HistogramVec
+	wsCount      prometheus.Gauge
+	ipcCount     prometheus.Gauge
+)
+
+// Init module initialization
+func Init() {
+	metrics = true
+
+	httpCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystemHTTP,
+		Name:      "count",
+		Help:      "http request count",
+	}, []string{"method"})
+
+	httpDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Subsystem: subsystemHTTP,
+		Name:      "duration",
+		Help:      "http request duration",
+	}, []string{"method"})
+
+	wsCount = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemWS,
+		Name:      "count",
+		Help:      "websocket connection count",
+	})
+
+	ipcCount = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemIPC,
+		Name:      "count",
+		Help:      "unix socket connection count",
+	})
+}
+
+// RegisterDBCollector create metric colletor for given connection
+func RegisterDBCollector(name string, db *sqlx.DB) {
+	if metrics {
+		prometheus.Register(NewDBStatsCollector(name, db))
+	}
+}

pkg/prom/serve.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var errPromHTTP = errors.New("can't start http server for prometheus")
+
+// Serve start listening http
+func Serve(addr string) *http.Server {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	srv := http.Server{
+		Addr:    addr,
+		Handler: mux,
+	}
+	go func() {
+		if err := srv.ListenAndServe(); err != nil {
+			log.
+				WithError(err).
+				WithField("module", "prom").
+				WithField("addr", addr).
+				Fatal(errPromHTTP)
+		}
+	}()
+	return &srv
+}

pkg/rpc/check.go

@@ -0,0 +1,37 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import "github.com/ethereum/go-ethereum/rpc"
+
+// checkModuleAvailability check that all names given in modules are actually
+// available API services.
+func checkModuleAvailability(modules []string, apis []rpc.API) (bad, available []string) {
+	availableSet := make(map[string]struct{})
+	for _, api := range apis {
+		if _, ok := availableSet[api.Namespace]; !ok {
+			availableSet[api.Namespace] = struct{}{}
+			available = append(available, api.Namespace)
+		}
+	}
+	for _, name := range modules {
+		if _, ok := availableSet[name]; !ok {
+			bad = append(bad, name)
+		}
+	}
+	return bad, available
+}

pkg/rpc/http.go

@@ -0,0 +1,49 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import (
+	"fmt"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/prom"
+)
+
+// StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules.
+func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (*rpc.Server, error) {
+
+	srv := rpc.NewServer()
+	err := node.RegisterApis(apis, modules, srv)
+	if err != nil {
+		utils.Fatalf("Could not register HTTP API: %w", err)
+	}
+	handler := prom.HTTPMiddleware(node.NewHTTPHandlerStack(srv, cors, vhosts, nil))
+
+	// start http server
+	_, addr, err := node.StartHTTPEndpoint(endpoint, rpc.DefaultHTTPTimeouts, handler)
+	if err != nil {
+		utils.Fatalf("Could not start RPC api: %v", err)
+	}
+	extapiURL := fmt.Sprintf("http://%s", addr)
+	log.Infof("HTTP endpoint opened at %s", extapiURL)
+
+	return srv, err
+}

pkg/rpc/ipc.go

@@ -0,0 +1,92 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/ethereum/go-ethereum/p2p/netutil"
+	"github.com/ethereum/go-ethereum/rpc"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/prom"
+)
+
+var (
+	//  On Linux, sun_path is 108 bytes in size
+	// see http://man7.org/linux/man-pages/man7/unix.7.html
+	maxPathSize = 108
+)
+
+// ipcListen will create a Unix socket on the given endpoint.
+func ipcListen(endpoint string) (net.Listener, error) {
+	if len(endpoint) > int(maxPathSize) {
+		log.Warn(fmt.Sprintf("The ipc endpoint is longer than %d characters. ", maxPathSize),
+			"endpoint", endpoint)
+	}
+
+	// Ensure the IPC path exists and remove any previous leftover
+	if err := os.MkdirAll(filepath.Dir(endpoint), 0751); err != nil {
+		return nil, err
+	}
+	os.Remove(endpoint)
+	l, err := net.Listen("unix", endpoint)
+	if err != nil {
+		return nil, err
+	}
+	os.Chmod(endpoint, 0600)
+	return l, nil
+}
+
+func ipcServe(srv *rpc.Server, listener net.Listener) {
+	for {
+		conn, err := listener.Accept()
+		if netutil.IsTemporaryError(err) {
+			log.WithError(err).Warn("rpc accept error")
+			continue
+		}
+		if err != nil {
+			log.WithError(err).Warn("unknown error")
+			continue
+		}
+		log.WithField("addr", conn.RemoteAddr()).Trace("accepted ipc connection")
+		go prom.IPCMiddleware(srv, conn)
+	}
+}
+
+// StartIPCEndpoint starts an IPC endpoint.
+func StartIPCEndpoint(ipcEndpoint string, apis []rpc.API) (net.Listener, *rpc.Server, error) {
+	// Register all the APIs exposed by the services.
+	handler := rpc.NewServer()
+	for _, api := range apis {
+		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
+			return nil, nil, err
+		}
+		log.Debug("IPC registered", "namespace", api.Namespace)
+	}
+	// All APIs registered, start the IPC listener.
+	listener, err := ipcListen(ipcEndpoint)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	go ipcServe(handler, listener)
+	return listener, handler, nil
+}

pkg/rpc/ws.go

@@ -0,0 +1,64 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import (
+	"net"
+	"net/http"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/prom"
+)
+
+// StartWSEndpoint starts a websocket endpoint.
+func StartWSEndpoint(endpoint string, apis []rpc.API, modules []string, wsOrigins []string) (net.Listener, *rpc.Server, error) {
+	// All APIs registered, start the HTTP listener
+	var (
+		listener net.Listener
+		err      error
+	)
+
+	// Register all the APIs exposed by the services
+	handler := rpc.NewServer()
+	err = node.RegisterApis(apis, modules, handler)
+	if err != nil {
+		utils.Fatalf("Could not register WS API: %w", err)
+	}
+
+	if listener, err = net.Listen("tcp", endpoint); err != nil {
+		return nil, nil, err
+	}
+
+	wsServer := NewWSServer(wsOrigins, handler)
+	wsServer.Handler = prom.WSMiddleware(wsServer.Handler)
+	go wsServer.Serve(listener)
+	log.Infof("WS endpoint opened at ws://%s", endpoint)
+
+	return listener, handler, err
+
+}
+
+// NewWSServer creates a new websocket RPC server around an API provider.
+//
+// Deprecated: use prc.Server.WebsocketHandler
+func NewWSServer(allowedOrigins []string, srv *rpc.Server) *http.Server {
+	return &http.Server{Handler: srv.WebsocketHandler(allowedOrigins)}
+}

pkg/serve/api.go

@@ -17,15 +17,8 @@
 package serve
 
 import (
-	"context"
-
-	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/cerc-io/plugeth-statediff/types"
 	"github.com/ethereum/go-ethereum/rpc"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-	v "github.com/vulcanize/ipld-eth-server/version"
 )
 
 // APIName is the namespace used for the state diffing service API
@@ -36,85 +29,25 @@ const APIVersion = "0.0.1"
 
 // PublicServerAPI is the public api for the watcher
 type PublicServerAPI struct {
-	w Server
+	w   Server
+	rpc *rpc.Client
 }
 
 // NewPublicServerAPI creates a new PublicServerAPI with the provided underlying Server process
-func NewPublicServerAPI(w Server) *PublicServerAPI {
+func NewPublicServerAPI(w Server, client *rpc.Client) *PublicServerAPI {
 	return &PublicServerAPI{
-		w: w,
+		w:   w,
+		rpc: client,
 	}
 }
 
-// Stream is the public method to setup a subscription that fires off IPLD payloads as they are processed
-func (api *PublicServerAPI) Stream(ctx context.Context, params eth.SubscriptionSettings) (*rpc.Subscription, error) {
-	// ensure that the RPC connection supports subscriptions
-	notifier, supported := rpc.NotifierFromContext(ctx)
-	if !supported {
-		return nil, rpc.ErrNotificationsUnsupported
+// WatchAddress makes a geth WatchAddress API call with the given operation and args
+func (api *PublicServerAPI) WatchAddress(operation types.OperationType, args []types.WatchAddressArg) error {
+	err := api.rpc.Call(nil, "statediff_watchAddress", operation, args)
+	if err != nil {
+		// Return the error directly to match the native Geth API
+		return err
 	}
 
-	// create subscription and start waiting for stream events
-	rpcSub := notifier.CreateSubscription()
-
-	go func() {
-		// subscribe to events from the SyncPublishScreenAndServe service
-		payloadChannel := make(chan SubscriptionPayload, PayloadChanBufferSize)
-		quitChan := make(chan bool, 1)
-		go api.w.Subscribe(rpcSub.ID, payloadChannel, quitChan, params)
-
-		// loop and await payloads and relay them to the subscriber using notifier
-		for {
-			select {
-			case packet := <-payloadChannel:
-				if err := notifier.Notify(rpcSub.ID, packet); err != nil {
-					log.Error("Failed to send watcher data packet", "err", err)
-					api.w.Unsubscribe(rpcSub.ID)
-					return
-				}
-			case <-rpcSub.Err():
-				api.w.Unsubscribe(rpcSub.ID)
-				return
-			case <-quitChan:
-				// don't need to unsubscribe from the watcher, the service does so before sending the quit signal this way
-				return
-			}
-		}
-	}()
-
-	return rpcSub, nil
-}
-
-// Chain returns the chain type that this watcher instance supports
-func (api *PublicServerAPI) Chain() shared.ChainType {
-	return api.w.Chain()
-}
-
-// Struct for holding watcher meta data
-type InfoAPI struct{}
-
-// NewInfoAPI creates a new InfoAPI
-func NewInfoAPI() *InfoAPI {
-	return &InfoAPI{}
-}
-
-// Modules returns modules supported by this api
-func (iapi *InfoAPI) Modules() map[string]string {
-	return map[string]string{
-		"vdb": "Stream",
-	}
-}
-
-// NodeInfo gathers and returns a collection of metadata for the watcher
-func (iapi *InfoAPI) NodeInfo() *p2p.NodeInfo {
-	return &p2p.NodeInfo{
-		// TODO: formalize this
-		ID:   "vulcanizeDB",
-		Name: "ipld-eth-server",
-	}
-}
-
-// Version returns the version of the watcher
-func (iapi *InfoAPI) Version() string {
-	return v.VersionWithMeta
+	return nil
 }

pkg/serve/config.go

@@ -17,36 +17,105 @@
 package serve
 
 import (
+	"errors"
+	"fmt"
+	"math/big"
 	"os"
 	"path/filepath"
+	"time"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/node"
-
+	"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
+	"github.com/cerc-io/plugeth-statediff/indexer/node"
+	"github.com/cerc-io/plugeth-statediff/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/jmoiron/sqlx"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-indexer/utils"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/prom"
+	ethServerShared "github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
 )
 
 // Env variables
 const (
-	SERVER_WS_PATH   = "SERVER_WS_PATH"
-	SERVER_IPC_PATH  = "SERVER_IPC_PATH"
-	SERVER_HTTP_PATH = "SERVER_HTTP_PATH"
+	SERVER_WS_PATH      = "SERVER_WS_PATH"
+	SERVER_IPC_PATH     = "SERVER_IPC_PATH"
+	SERVER_HTTP_PATH    = "SERVER_HTTP_PATH"
+	SERVER_GRAPHQL_PATH = "SERVER_GRAPHQL_PATH"
 
 	SERVER_MAX_IDLE_CONNECTIONS = "SERVER_MAX_IDLE_CONNECTIONS"
 	SERVER_MAX_OPEN_CONNECTIONS = "SERVER_MAX_OPEN_CONNECTIONS"
 	SERVER_MAX_CONN_LIFETIME    = "SERVER_MAX_CONN_LIFETIME"
+
+	ETH_DEFAULT_SENDER_ADDR    = "ETH_DEFAULT_SENDER_ADDR"
+	ETH_RPC_GAS_CAP            = "ETH_RPC_GAS_CAP"
+	ETH_CHAIN_CONFIG           = "ETH_CHAIN_CONFIG"
+	ETH_SUPPORTS_STATEDIFF     = "ETH_SUPPORTS_STATEDIFF"
+	ETH_STATEDIFF_TIMEOUT      = "ETH_STATEDIFF_TIMEOUT"
+	ETH_FORWARD_ETH_CALLS      = "ETH_FORWARD_ETH_CALLS"
+	ETH_FORWARD_GET_STORAGE_AT = "ETH_FORWARD_GET_STORAGE_AT"
+	ETH_PROXY_ON_ERROR         = "ETH_PROXY_ON_ERROR"
+	ETH_GETLOGS_BLOCK_LIMIT    = "ETH_GETLOGS_BLOCK_LIMIT"
+
+	VALIDATOR_ENABLED         = "VALIDATOR_ENABLED"
+	VALIDATOR_EVERY_NTH_BLOCK = "VALIDATOR_EVERY_NTH_BLOCK"
+
+	HTTP_TIMEOUT = "HTTP_TIMEOUT"
+
+	ETH_WS_PATH       = "ETH_WS_PATH"
+	ETH_HTTP_PATH     = "ETH_HTTP_PATH"
+	ETH_NODE_ID       = "ETH_NODE_ID"
+	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
+	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
+	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
+	ETH_CHAIN_ID      = "ETH_CHAIN_ID"
+
+	DATABASE_NAME                 = "DATABASE_NAME"
+	DATABASE_HOSTNAME             = "DATABASE_HOSTNAME"
+	DATABASE_PORT                 = "DATABASE_PORT"
+	DATABASE_USER                 = "DATABASE_USER"
+	DATABASE_PASSWORD             = "DATABASE_PASSWORD"
+	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
+	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
+	DATABASE_MAX_CONN_LIFETIME    = "DATABASE_MAX_CONN_LIFETIME"
 )
 
 // Config struct
 type Config struct {
-	DB           *postgres.DB
-	DBConfig     postgres.Config
-	WSEndpoint   string
+	DB       *sqlx.DB
+	DBConfig postgres.Config
+
+	WSEnabled  bool
+	WSEndpoint string
+
+	HTTPEnabled  bool
 	HTTPEndpoint string
-	IPCEndpoint  string
+
+	IPCEnabled  bool
+	IPCEndpoint string
+
+	EthGraphqlEnabled  bool
+	EthGraphqlEndpoint string
+
+	ChainConfig         *params.ChainConfig
+	DefaultSender       *common.Address
+	RPCGasCap           *big.Int
+	EthHttpEndpoint     string
+	Client              *rpc.Client
+	SupportStateDiff    bool
+	StateDiffTimeout    time.Duration
+	ForwardEthCalls     bool
+	ForwardGetStorageAt bool
+	ProxyOnError        bool
+	GetLogsBlockLimit   int64
+	NodeNetworkID       string
+
+	// Cache configuration.
+	GroupCache *ethServerShared.GroupCacheConfig
+
+	StateValidationEnabled       bool
+	StateValidationEveryNthBlock uint64
 }
 
 // NewConfig is used to initialize a watcher config from a .toml file
@@ -54,36 +123,148 @@ type Config struct {
 func NewConfig() (*Config, error) {
 	c := new(Config)
 
+	viper.BindEnv("server.httpPath", SERVER_HTTP_PATH)
 	viper.BindEnv("server.wsPath", SERVER_WS_PATH)
 	viper.BindEnv("server.ipcPath", SERVER_IPC_PATH)
-	viper.BindEnv("server.httpPath", SERVER_HTTP_PATH)
+	viper.BindEnv("server.graphqlPath", SERVER_GRAPHQL_PATH)
 
-	c.DBConfig.Init()
+	viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
+	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
+	viper.BindEnv("ethereum.httpPath", ETH_HTTP_PATH)
+	viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP)
+	viper.BindEnv("ethereum.supportsStateDiff", ETH_SUPPORTS_STATEDIFF)
+	viper.BindEnv("ethereum.stateDiffTimeout", ETH_STATEDIFF_TIMEOUT)
+	viper.BindEnv("ethereum.forwardEthCalls", ETH_FORWARD_ETH_CALLS)
+	viper.BindEnv("ethereum.forwardGetStorageAt", ETH_FORWARD_GET_STORAGE_AT)
+	viper.BindEnv("ethereum.proxyOnError", ETH_PROXY_ON_ERROR)
+	viper.BindEnv("ethereum.getLogsBlockLimit", ETH_GETLOGS_BLOCK_LIMIT)
+	viper.BindEnv("log.file", "LOG_FILE")
+	viper.BindEnv("log.level", "LOG_LEVEL")
 
-	wsPath := viper.GetString("server.wsPath")
-	if wsPath == "" {
-		wsPath = "127.0.0.1:8080"
-	}
-	c.WSEndpoint = wsPath
-	ipcPath := viper.GetString("server.ipcPath")
-	if ipcPath == "" {
-		home, err := os.UserHomeDir()
-		if err != nil {
+	c.dbInit()
+	ethHTTP := viper.GetString("ethereum.httpPath")
+	ethHTTPEndpoint := fmt.Sprintf("http://%s", ethHTTP)
+
+	// At least one of chain ID and chain config must be passed.
+	// If both are passed, the chain ID must match the config.
+	chainID := viper.GetUint64("ethereum.chainID")
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+	var err error
+	if chainConfigPath != "" {
+		if c.ChainConfig, err = utils.LoadConfig(chainConfigPath); err != nil {
+			return nil, err
+		}
+		// Only validate the chain ID if it was actually passed
+		if viper.GetString("ethereum.chainID") != "" && c.ChainConfig.ChainID.Uint64() != chainID {
+			return nil, fmt.Errorf("passed chain ID %d does not match chain config chain ID %d",
+				chainID, c.ChainConfig.ChainID.Uint64())
+		}
+	} else {
+		if c.ChainConfig, err = utils.ChainConfig(chainID); err != nil {
 			return nil, err
 		}
-		ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
 	}
-	c.IPCEndpoint = ipcPath
-	httpPath := viper.GetString("server.httpPath")
-	if httpPath == "" {
-		httpPath = "127.0.0.1:8081"
-	}
-	c.HTTPEndpoint = httpPath
-	overrideDBConnConfig(&c.DBConfig)
-	serveDB := utils.LoadPostgres(c.DBConfig, node.Info{})
-	c.DB = &serveDB
 
-	return c, nil
+	nodeInfo, cli, err := getEthNodeAndClient(ethHTTPEndpoint, chainID)
+	if err != nil {
+		return nil, err
+	}
+	c.NodeNetworkID = nodeInfo.NetworkID
+	c.Client = cli
+	c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff")
+	c.ForwardEthCalls = viper.GetBool("ethereum.forwardEthCalls")
+	c.ForwardGetStorageAt = viper.GetBool("ethereum.forwardGetStorageAt")
+	c.ProxyOnError = viper.GetBool("ethereum.proxyOnError")
+	c.EthHttpEndpoint = ethHTTPEndpoint
+
+	if viper.IsSet("ethereum.getLogsBlockLimit") {
+		c.GetLogsBlockLimit = viper.GetInt64("ethereum.getLogsBlockLimit")
+	} else {
+		c.GetLogsBlockLimit = 500
+	}
+
+	// websocket server
+	wsEnabled := viper.GetBool("server.ws")
+	if wsEnabled {
+		wsPath := viper.GetString("server.wsPath")
+		if wsPath == "" {
+			wsPath = "127.0.0.1:8080"
+		}
+		c.WSEndpoint = wsPath
+	}
+	c.WSEnabled = wsEnabled
+
+	// ipc server
+	ipcEnabled := viper.GetBool("server.ipc")
+	if ipcEnabled {
+		ipcPath := viper.GetString("server.ipcPath")
+		if ipcPath == "" {
+			home, err := os.UserHomeDir()
+			if err != nil {
+				return nil, err
+			}
+			ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
+		}
+		c.IPCEndpoint = ipcPath
+	}
+	c.IPCEnabled = ipcEnabled
+
+	// http server
+	httpEnabled := viper.GetBool("server.http")
+	if httpEnabled {
+		httpPath := viper.GetString("server.httpPath")
+		if httpPath == "" {
+			httpPath = "127.0.0.1:8081"
+		}
+		c.HTTPEndpoint = httpPath
+	}
+	c.HTTPEnabled = httpEnabled
+
+	// eth graphql endpoint
+	ethGraphqlEnabled := viper.GetBool("server.graphql")
+	if ethGraphqlEnabled {
+		ethGraphqlPath := viper.GetString("server.graphqlPath")
+		if ethGraphqlPath == "" {
+			ethGraphqlPath = "127.0.0.1:8082"
+		}
+		c.EthGraphqlEndpoint = ethGraphqlPath
+	}
+	c.EthGraphqlEnabled = ethGraphqlEnabled
+
+	overrideDBConnConfig(&c.DBConfig)
+	serveDB, err := ethServerShared.NewDB(c.DBConfig.DbConnectionString(), c.DBConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	prom.RegisterDBCollector(c.DBConfig.DatabaseName, serveDB)
+	c.DB = serveDB
+
+	rpcGasCapStr := viper.GetString("ethereum.rpcGasCap")
+	if rpcGasCapStr != "" {
+		if rpcGasCap, ok := new(big.Int).SetString(rpcGasCapStr, 10); ok {
+			c.RPCGasCap = rpcGasCap
+		}
+	} else {
+		c.RPCGasCap = big.NewInt(0)
+	}
+	if sdTimeout := viper.GetString("ethereum.stateDiffTimeout"); sdTimeout != "" {
+		var err error
+		if c.StateDiffTimeout, err = time.ParseDuration(sdTimeout); err != nil {
+			return nil, err
+		}
+	} else {
+		c.StateDiffTimeout = ethServerShared.DefaultStateDiffTimeout
+	}
+	if c.StateDiffTimeout < 0 {
+		return nil, errors.New("ethereum.stateDiffTimeout < 0")
+	}
+
+	c.loadGroupCacheConfig()
+
+	c.loadValidatorConfig()
+
+	return c, err
 }
 
 func overrideDBConnConfig(con *postgres.Config) {
@@ -91,6 +272,77 @@ func overrideDBConnConfig(con *postgres.Config) {
 	viper.BindEnv("database.server.maxOpen", SERVER_MAX_OPEN_CONNECTIONS)
 	viper.BindEnv("database.server.maxLifetime", SERVER_MAX_CONN_LIFETIME)
 	con.MaxIdle = viper.GetInt("database.server.maxIdle")
-	con.MaxOpen = viper.GetInt("database.server.maxOpen")
-	con.MaxLifetime = viper.GetInt("database.server.maxLifetime")
+	con.MaxConns = viper.GetInt("database.server.maxOpen")
+	con.MaxConnLifetime = time.Duration(viper.GetInt("database.server.maxLifetime"))
+}
+
+func (c *Config) dbInit() {
+	viper.BindEnv("database.name", DATABASE_NAME)
+	viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
+	viper.BindEnv("database.port", DATABASE_PORT)
+	viper.BindEnv("database.user", DATABASE_USER)
+	viper.BindEnv("database.password", DATABASE_PASSWORD)
+	viper.BindEnv("database.maxIdle", DATABASE_MAX_IDLE_CONNECTIONS)
+	viper.BindEnv("database.maxOpen", DATABASE_MAX_OPEN_CONNECTIONS)
+	viper.BindEnv("database.maxLifetime", DATABASE_MAX_CONN_LIFETIME)
+
+	c.DBConfig.DatabaseName = viper.GetString("database.name")
+	c.DBConfig.Hostname = viper.GetString("database.hostname")
+	c.DBConfig.Port = viper.GetInt("database.port")
+	c.DBConfig.Username = viper.GetString("database.user")
+	c.DBConfig.Password = viper.GetString("database.password")
+	c.DBConfig.MaxIdle = viper.GetInt("database.maxIdle")
+	c.DBConfig.MaxConns = viper.GetInt("database.maxOpen")
+	c.DBConfig.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime"))
+}
+
+func (c *Config) loadGroupCacheConfig() {
+	viper.BindEnv("groupcache.pool.enabled", ethServerShared.GcachePoolEnabled)
+	viper.BindEnv("groupcache.pool.httpEndpoint", ethServerShared.GcachePoolHttpPath)
+	viper.BindEnv("groupcache.pool.peerHttpEndpoints", ethServerShared.GcachePoolHttpPeers)
+	viper.BindEnv("groupcache.statedb.cacheSizeInMB", ethServerShared.GcacheStatedbCacheSize)
+	viper.BindEnv("groupcache.statedb.cacheExpiryInMins", ethServerShared.GcacheStatedbCacheExpiry)
+	viper.BindEnv("groupcache.statedb.logStatsIntervalInSecs", ethServerShared.GcacheStatedbLogStatsInterval)
+
+	gcc := ethServerShared.GroupCacheConfig{}
+	gcc.Pool.Enabled = viper.GetBool("groupcache.pool.enabled")
+	if gcc.Pool.Enabled {
+		gcc.Pool.HttpEndpoint = viper.GetString("groupcache.pool.httpEndpoint")
+		gcc.Pool.PeerHttpEndpoints = viper.GetStringSlice("groupcache.pool.peerHttpEndpoints")
+	}
+
+	// Irrespective of whether the pool is enabled, we always use the hot/local cache.
+	gcc.StateDB.CacheSizeInMB = viper.GetInt("groupcache.statedb.cacheSizeInMB")
+	gcc.StateDB.CacheExpiryInMins = viper.GetInt("groupcache.statedb.cacheExpiryInMins")
+	gcc.StateDB.LogStatsIntervalInSecs = viper.GetInt("groupcache.statedb.logStatsIntervalInSecs")
+
+	c.GroupCache = &gcc
+}
+
+func (c *Config) loadValidatorConfig() {
+	viper.BindEnv("validator.enabled", VALIDATOR_ENABLED)
+	viper.BindEnv("validator.everyNthBlock", VALIDATOR_EVERY_NTH_BLOCK)
+
+	c.StateValidationEnabled = viper.GetBool("validator.enabled")
+	c.StateValidationEveryNthBlock = viper.GetUint64("validator.everyNthBlock")
+}
+
+// GetEthNodeAndClient returns eth node info and client from path url
+func getEthNodeAndClient(path string, chainid uint64) (node.Info, *rpc.Client, error) {
+	viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
+	viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
+	viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
+	viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
+
+	rpcClient, err := rpc.Dial(path)
+	if err != nil {
+		return node.Info{}, nil, err
+	}
+	return node.Info{
+		ID:           viper.GetString("ethereum.nodeID"),
+		ClientName:   viper.GetString("ethereum.clientName"),
+		GenesisBlock: viper.GetString("ethereum.genesisBlock"),
+		NetworkID:    viper.GetString("ethereum.networkID"),
+		ChainID:      chainid,
+	}, rpcClient, nil
 }

pkg/serve/service.go

@@ -17,22 +17,21 @@
 package serve
 
 import (
-	"fmt"
+	"strconv"
 	"sync"
+	"time"
 
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/eth/tracers"
 	ethnode "github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	log "github.com/sirupsen/logrus"
+	"github.com/jmoiron/sqlx"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/debug"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/net"
 )
 
 const (
@@ -43,51 +42,65 @@ const (
 // and indexing all chain data; screening this data; and serving it up to subscribed clients
 // This service is compatible with the Ethereum service interface (node.Service)
 type Server interface {
-	// APIs(), Protocols(), Start() and Stop()
-	ethnode.Service
+	// Start() and Stop()
+	ethnode.Lifecycle
+	APIs() []rpc.API
+	Protocols() []p2p.Protocol
 	// Pub-Sub handling event loop
-	Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth2.ConvertedPayload)
-	// Method to subscribe to the service
-	Subscribe(id rpc.ID, sub chan<- SubscriptionPayload, quitChan chan<- bool, params eth.SubscriptionSettings)
-	// Method to unsubscribe from the service
-	Unsubscribe(id rpc.ID)
-	// Method to access chain type
-	Chain() shared.ChainType
+	Serve(wg *sync.WaitGroup)
+	// Backend exposes the server's backend
+	Backend() *eth.Backend
 }
 
 // Service is the underlying struct for the watcher
 type Service struct {
 	// Used to sync access to the Subscriptions
 	sync.Mutex
-	// Interface for filtering and serving data according to subscribed clients according to their specification
-	Filterer eth.Filterer
-	// Interface for fetching IPLD objects from IPFS
-	IPLDFetcher eth.Fetcher
-	// Interface for searching and retrieving CIDs from Postgres index
-	Retriever eth.Retriever
 	// Used to signal shutdown of the service
 	QuitChan chan bool
-	// A mapping of rpc.IDs to their subscription channels, mapped to their subscription type (hash of the StreamFilters)
-	Subscriptions map[common.Hash]map[rpc.ID]Subscription
-	// A mapping of subscription params hash to the corresponding subscription params
-	SubscriptionTypes map[common.Hash]eth.SubscriptionSettings
-	// Underlying db
-	db *postgres.DB
-	// wg for syncing serve processes
-	serveWg *sync.WaitGroup
+	// Underlying db connection pool
+	db *sqlx.DB
+	// rpc client for forwarding cache misses
+	client *rpc.Client
+	// whether the proxied client supports state diffing
+	supportsStateDiffing bool
+	// timeout for statediff RPC calls
+	stateDiffTimeout time.Duration
+	// backend for the server
+	backend *eth.Backend
+	// whether to forward eth_calls directly to proxy node
+	forwardEthCalls bool
+	// whether to forward eth_getStorageAt directly to proxy node
+	forwardGetStorageAt bool
+	// the maximum size of the block range to use in GetLogs
+	getLogsBlockLimit int64
+	// whether to forward all calls to proxy node if they throw an error locally
+	proxyOnError bool
+	// eth node network id
+	nodeNetworkId string
 }
 
 // NewServer creates a new Server using an underlying Service struct
 func NewServer(settings *Config) (Server, error) {
-	sn := new(Service)
-	sn.Retriever = eth.NewCIDRetriever(settings.DB)
-	sn.IPLDFetcher = eth.NewIPLDFetcher(settings.DB)
-	sn.Filterer = eth.NewResponseFilterer()
-	sn.db = settings.DB
-	sn.QuitChan = make(chan bool)
-	sn.Subscriptions = make(map[common.Hash]map[rpc.ID]Subscription)
-	sn.SubscriptionTypes = make(map[common.Hash]eth.SubscriptionSettings)
-	return sn, nil
+	sap := new(Service)
+	sap.db = settings.DB
+	sap.QuitChan = make(chan bool)
+	sap.client = settings.Client
+	sap.supportsStateDiffing = settings.SupportStateDiff
+	sap.stateDiffTimeout = settings.StateDiffTimeout
+	sap.forwardEthCalls = settings.ForwardEthCalls
+	sap.forwardGetStorageAt = settings.ForwardGetStorageAt
+	sap.getLogsBlockLimit = settings.GetLogsBlockLimit
+	sap.proxyOnError = settings.ProxyOnError
+	sap.nodeNetworkId = settings.NodeNetworkID
+	var err error
+	sap.backend, err = eth.NewEthBackend(sap.db, &eth.Config{
+		ChainConfig:      settings.ChainConfig,
+		VMConfig:         vm.Config{NoBaseFee: true},
+		RPCGasCap:        settings.RPCGasCap,
+		GroupCacheConfig: settings.GroupCache,
+	})
+	return sap, err
 }
 
 // Protocols exports the services p2p protocols, this service has none
@@ -97,284 +110,81 @@ func (sap *Service) Protocols() []p2p.Protocol {
 
 // APIs returns the RPC descriptors the watcher service offers
 func (sap *Service) APIs() []rpc.API {
-	infoAPI := NewInfoAPI()
+	networkID, _ := strconv.ParseUint(sap.nodeNetworkId, 10, 64)
 	apis := []rpc.API{
 		{
 			Namespace: APIName,
 			Version:   APIVersion,
-			Service:   NewPublicServerAPI(sap),
+			Service:   NewPublicServerAPI(sap, sap.client),
 			Public:    true,
 		},
 		{
-			Namespace: "rpc",
-			Version:   APIVersion,
-			Service:   infoAPI,
-			Public:    true,
-		},
-		{
-			Namespace: "net",
-			Version:   APIVersion,
-			Service:   infoAPI,
-			Public:    true,
-		},
-		{
-			Namespace: "admin",
-			Version:   APIVersion,
-			Service:   infoAPI,
+			Namespace: net.APIName,
+			Version:   net.APIVersion,
+			Service:   net.NewPublicNetAPI(networkID, sap.client),
 			Public:    true,
 		},
 	}
-	backend, err := eth.NewEthBackend(sap.db)
+	conf := eth.APIConfig{
+		SupportsStateDiff:   sap.supportsStateDiffing,
+		ForwardEthCalls:     sap.forwardEthCalls,
+		ForwardGetStorageAt: sap.forwardGetStorageAt,
+		ProxyOnError:        sap.proxyOnError,
+		StateDiffTimeout:    sap.stateDiffTimeout,
+		GetLogsBlockLimit:   sap.getLogsBlockLimit,
+	}
+	ethAPI, err := eth.NewPublicEthAPI(sap.backend, sap.client, conf)
 	if err != nil {
-		log.Error(err)
-		return nil
+		log.Fatalf("unable to create public eth api: %v", err)
 	}
-	return append(apis, rpc.API{
-		Namespace: eth.APIName,
-		Version:   eth.APIVersion,
-		Service:   eth.NewPublicEthAPI(backend),
-		Public:    true,
-	})
+
+	debugTracerAPI := tracers.APIs(&debug.Backend{Backend: *sap.backend})[0]
+
+	return append(apis,
+		rpc.API{
+			Namespace: eth.APIName,
+			Version:   eth.APIVersion,
+			Service:   ethAPI,
+			Public:    true,
+		},
+		debugTracerAPI,
+	)
 }
 
 // Serve listens for incoming converter data off the screenAndServePayload from the Sync process
 // It filters and sends this data to any subscribers to the service
 // This process can also be stood up alone, without an screenAndServePayload attached to a Sync process
 // and it will hang on the WaitGroup indefinitely, allowing the Service to serve historical data requests only
-func (sap *Service) Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth2.ConvertedPayload) {
-	sap.serveWg = wg
+func (sap *Service) Serve(wg *sync.WaitGroup) {
 	go func() {
 		wg.Add(1)
 		defer wg.Done()
-		for {
-			select {
-			case payload := <-screenAndServePayload:
-				sap.filterAndServe(payload)
-			case <-sap.QuitChan:
-				log.Info("quiting eth ipld server process")
-				return
-			}
-		}
+		<-sap.QuitChan
+		log.Info("quiting eth ipld server process")
 	}()
-	log.Info("eth ipld server process successfully spun up")
-}
-
-// filterAndServe filters the payload according to each subscription type and sends to the subscriptions
-func (sap *Service) filterAndServe(payload eth2.ConvertedPayload) {
-	log.Debug("sending eth ipld payload to subscriptions")
-	sap.Lock()
-	sap.serveWg.Add(1)
-	defer sap.Unlock()
-	defer sap.serveWg.Done()
-	for ty, subs := range sap.Subscriptions {
-		// Retrieve the subscription parameters for this subscription type
-		subConfig, ok := sap.SubscriptionTypes[ty]
-		if !ok {
-			log.Errorf("eth ipld server subscription configuration for subscription type %s not available", ty.Hex())
-			sap.closeType(ty)
-			continue
-		}
-		if subConfig.End.Int64() > 0 && subConfig.End.Int64() < payload.Block.Number().Int64() {
-			// We are not out of range for this subscription type
-			// close it, and continue to the next
-			sap.closeType(ty)
-			continue
-		}
-		response, err := sap.Filterer.Filter(subConfig, payload)
-		if err != nil {
-			log.Errorf("eth ipld server filtering error: %v", err)
-			sap.closeType(ty)
-			continue
-		}
-		responseRLP, err := rlp.EncodeToBytes(response)
-		if err != nil {
-			log.Errorf("eth ipld server rlp encoding error: %v", err)
-			continue
-		}
-		for id, sub := range subs {
-			select {
-			case sub.PayloadChan <- SubscriptionPayload{Data: responseRLP, Err: "", Flag: EmptyFlag, Height: response.BlockNumber.Int64()}:
-				log.Debugf("sending eth ipld server payload to subscription %s", id)
-			default:
-				log.Infof("unable to send eth ipld payload to subscription %s; channel has no receiver", id)
-			}
-		}
-	}
-}
-
-// Subscribe is used by the API to remotely subscribe to the service loop
-// The params must be rlp serializable and satisfy the SubscriptionSettings() interface
-func (sap *Service) Subscribe(id rpc.ID, sub chan<- SubscriptionPayload, quitChan chan<- bool, params eth.SubscriptionSettings) {
-	sap.serveWg.Add(1)
-	defer sap.serveWg.Done()
-	log.Infof("new eth ipld subscription %s", id)
-	subscription := Subscription{
-		ID:          id,
-		PayloadChan: sub,
-		QuitChan:    quitChan,
-	}
-	// Subscription type is defined as the hash of the rlp-serialized subscription settings
-	by, err := rlp.EncodeToBytes(params)
-	if err != nil {
-		sendNonBlockingErr(subscription, err)
-		sendNonBlockingQuit(subscription)
-		return
-	}
-	subscriptionType := crypto.Keccak256Hash(by)
-	if !params.BackFillOnly {
-		// Add subscriber
-		sap.Lock()
-		if sap.Subscriptions[subscriptionType] == nil {
-			sap.Subscriptions[subscriptionType] = make(map[rpc.ID]Subscription)
-		}
-		sap.Subscriptions[subscriptionType][id] = subscription
-		sap.SubscriptionTypes[subscriptionType] = params
-		sap.Unlock()
-	}
-	// If the subscription requests a backfill, use the Postgres index to lookup and retrieve historical data
-	// Otherwise we only filter new data as it is streamed in from the state diffing geth node
-	if params.BackFill || params.BackFillOnly {
-		if err := sap.sendHistoricalData(subscription, id, params); err != nil {
-			sendNonBlockingErr(subscription, fmt.Errorf("eth ipld server subscription backfill error: %v", err))
-			sendNonBlockingQuit(subscription)
-			return
-		}
-	}
-}
-
-// sendHistoricalData sends historical data to the requesting subscription
-func (sap *Service) sendHistoricalData(sub Subscription, id rpc.ID, params eth.SubscriptionSettings) error {
-	log.Infof("sending eth ipld historical data to subscription %s", id)
-	// Retrieve cached CIDs relevant to this subscriber
-	var endingBlock int64
-	var startingBlock int64
-	var err error
-	startingBlock, err = sap.Retriever.RetrieveFirstBlockNumber()
-	if err != nil {
-		return err
-	}
-	if startingBlock < params.Start.Int64() {
-		startingBlock = params.Start.Int64()
-	}
-	endingBlock, err = sap.Retriever.RetrieveLastBlockNumber()
-	if err != nil {
-		return err
-	}
-	if endingBlock > params.End.Int64() && params.End.Int64() > 0 && params.End.Int64() > startingBlock {
-		endingBlock = params.End.Int64()
-	}
-	log.Debugf("eth ipld historical data starting block: %d", params.Start.Int64())
-	log.Debugf("eth ipld historical data ending block: %d", endingBlock)
-	go func() {
-		sap.serveWg.Add(1)
-		defer sap.serveWg.Done()
-		for i := startingBlock; i <= endingBlock; i++ {
-			select {
-			case <-sap.QuitChan:
-				log.Infof("ethereum historical data feed to subscription %s closed", id)
-				return
-			default:
-			}
-			cidWrappers, empty, err := sap.Retriever.Retrieve(params, i)
-			if err != nil {
-				sendNonBlockingErr(sub, fmt.Errorf("eth ipld server cid retrieval error at block %d\r%s", i, err.Error()))
-				continue
-			}
-			if empty {
-				continue
-			}
-			for _, cids := range cidWrappers {
-				response, err := sap.IPLDFetcher.Fetch(cids)
-				if err != nil {
-					sendNonBlockingErr(sub, fmt.Errorf("eth ipld server ipld fetching error at block %d\r%s", i, err.Error()))
-					continue
-				}
-				responseRLP, err := rlp.EncodeToBytes(response)
-				if err != nil {
-					log.Error(err)
-					continue
-				}
-				select {
-				case sub.PayloadChan <- SubscriptionPayload{Data: responseRLP, Err: "", Flag: EmptyFlag, Height: response.BlockNumber.Int64()}:
-					log.Debugf("eth ipld server sending historical data payload to subscription %s", id)
-				default:
-					log.Infof("eth ipld server unable to send backFill payload to subscription %s; channel has no receiver", id)
-				}
-			}
-		}
-		// when we are done backfilling send an empty payload signifying so in the msg
-		select {
-		case sub.PayloadChan <- SubscriptionPayload{Data: nil, Err: "", Flag: BackFillCompleteFlag}:
-			log.Debugf("eth ipld server sending backFill completion notice to subscription %s", id)
-		default:
-			log.Infof("eth ipld server unable to send backFill completion notice to subscription %s", id)
-		}
-	}()
-	return nil
-}
-
-// Unsubscribe is used by the API to remotely unsubscribe to the StateDiffingService loop
-func (sap *Service) Unsubscribe(id rpc.ID) {
-	log.Infof("unsubscribing %s from the eth ipld server", id)
-	sap.Lock()
-	for ty := range sap.Subscriptions {
-		delete(sap.Subscriptions[ty], id)
-		if len(sap.Subscriptions[ty]) == 0 {
-			// If we removed the last subscription of this type, remove the subscription type outright
-			delete(sap.Subscriptions, ty)
-			delete(sap.SubscriptionTypes, ty)
-		}
-	}
-	sap.Unlock()
+	log.Debug("eth ipld server process successfully spun up")
 }
 
 // Start is used to begin the service
 // This is mostly just to satisfy the node.Service interface
-func (sap *Service) Start(*p2p.Server) error {
+func (sap *Service) Start() error {
 	log.Info("starting eth ipld server")
 	wg := new(sync.WaitGroup)
-	payloadChan := make(chan eth2.ConvertedPayload, PayloadChanBufferSize)
-	sap.Serve(wg, payloadChan)
+	sap.Serve(wg)
 	return nil
 }
 
 // Stop is used to close down the service
 // This is mostly just to satisfy the node.Service interface
 func (sap *Service) Stop() error {
-	log.Infof("stopping eth ipld server")
+	log.Info("stopping eth ipld server")
 	sap.Lock()
 	close(sap.QuitChan)
-	sap.close()
 	sap.Unlock()
 	return nil
 }
 
-// Chain returns the chain type for this service
-func (sap *Service) Chain() shared.ChainType {
-	return shared.Ethereum
-}
-
-// close is used to close all listening subscriptions
-// close needs to be called with subscription access locked
-func (sap *Service) close() {
-	log.Infof("closing all eth ipld server subscriptions")
-	for subType, subs := range sap.Subscriptions {
-		for _, sub := range subs {
-			sendNonBlockingQuit(sub)
-		}
-		delete(sap.Subscriptions, subType)
-		delete(sap.SubscriptionTypes, subType)
-	}
-}
-
-// closeType is used to close all subscriptions of given type
-// closeType needs to be called with subscription access locked
-func (sap *Service) closeType(subType common.Hash) {
-	log.Infof("closing all eth ipld server subscriptions of type %s", subType.String())
-	subs := sap.Subscriptions[subType]
-	for _, sub := range subs {
-		sendNonBlockingQuit(sub)
-	}
-	delete(sap.Subscriptions, subType)
-	delete(sap.SubscriptionTypes, subType)
+// Backend exposes the server's backend
+func (sap *Service) Backend() *eth.Backend {
+	return sap.backend
 }