Create issues-notion-sync.yml

Implement postgraphile graphql queries in V3

.github/workflows/issues-notion-sync.yml

@@ -0,0 +1,29 @@
+name: Notion Sync
+
+on:
+  workflow_dispatch:
+  issues:
+    types:
+      [
+        opened,
+        edited,
+        labeled,
+        unlabeled,
+        assigned,
+        unassigned,
+        milestoned,
+        demilestoned,
+        reopened,
+        closed,
+      ]
+
+jobs:
+  notion_job:
+    runs-on: ubuntu-latest
+    name: Add GitHub Issues to Notion
+    steps:
+      - name: Add GitHub Issues to Notion
+        uses: vulcanize/notion-github-action@v1.2.4-issueid
+        with:
+          notion-token: ${{ secrets.NOTION_TOKEN }}
+          notion-db: ${{ secrets.NOTION_DATABASE }}

.github/workflows/on-master.yaml

@@ -1,25 +0,0 @@
-name: Docker Compose Build
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag vulcanize/ipld-eth-server docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Push
-        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
-

.github/workflows/on-pr.yaml

@@ -1,12 +1,10 @@
 name: Docker Build
 
 on: [pull_request]
-
 jobs:
-  build:
+  run-tests:
-    name: Run docker build
+    uses: ./.github/workflows/tests.yaml
-    runs-on: ubuntu-latest
+    secrets:
-    steps:
+      BUILD_HOSTNAME: ${{ secrets.BUILD_HOSTNAME }}
-      - uses: actions/checkout@v2
+      BUILD_USERNAME: ${{ secrets.BUILD_USERNAME }}
-      - name: Run docker build
+      BUILD_KEY: ${{ secrets.BUILD_KEY }}
-        run: make docker-build

.github/workflows/publish.yaml

@@ -3,9 +3,34 @@ on:
   release:
     types: [published]
 jobs:
+  run-tests:
+    uses: ./.github/workflows/tests.yaml
+    secrets:
+      BUILD_HOSTNAME: ${{ secrets.BUILD_HOSTNAME }}
+      BUILD_USERNAME: ${{ secrets.BUILD_USERNAME }}
+      BUILD_KEY: ${{ secrets.BUILD_KEY }}
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    needs: run-tests
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the version
+        id: vars
+        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+      - name: Run docker build
+        run: make docker-build
+      - name: Tag docker image
+        run: docker tag vulcanize/ipld-eth-server docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
+      - name: Docker Login
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+      - name: Docker Push
+        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
+
   push_to_registries:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
+    needs: build
     steps:
       - name: Get the version
         id: vars
@@ -22,4 +47,3 @@ jobs:
         run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}}
       - name: Docker Push to Docker Hub
         run: docker push vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}}
-

.github/workflows/run_unit_test.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e
+
+# Set up repo
+start_dir=$(pwd)
+temp_dir=$(mktemp -d)
+cd $temp_dir
+git clone -b $(cat /tmp/git_head_ref) "https://github.com/$(cat /tmp/git_repository).git"
+cd ipld-eth-server
+
+## Remove the branch and github related info. This way future runs wont be confused.
+rm -f /tmp/git_head_ref /tmp/git_repository
+
+# Spin up DB
+docker-compose -f docker-compose.yml up -d ipld-eth-db
+trap "docker-compose down --remove-orphans; cd $start_dir ; rm -r $temp_dir" SIGINT SIGTERM ERR
+sleep 10
+
+# Remove old logs so there's no confusion, then run test
+rm -f /tmp/test.log /tmp/return_test.txt
+PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test >  /tmp/test.log
+echo $? > /tmp/return_test.txt
+
+# Clean up
+docker-compose -f docker-compose.yml down -v --remove-orphans
+cd $start_dir
+rm -fr $temp_dir
+

.github/workflows/tests.yaml

@@ -0,0 +1,189 @@
+name: Test the stack.
+on:
+  workflow_call:
+    secrets:
+      BUILD_HOSTNAME:
+        required: true
+      BUILD_USERNAME:
+        required: true
+      BUILD_KEY:
+        required: true
+
+jobs:
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run docker build
+        run: make docker-build
+  test:
+    name: Run unit tests
+    env:
+      GOPATH: /tmp/go
+      # To run the unit tests you need to add secrets to your repository.
+      BUILD_HOSTNAME: ${{ secrets.BUILD_HOSTNAME }}
+      BUILD_USERNAME: ${{ secrets.BUILD_USERNAME }}
+      BUILD_KEY: ${{ secrets.BUILD_KEY }}
+    #strategy:
+    #  matrix:
+    #    go-version: [1.16.x, 1.17.x]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      # Passed experience with GHA has taught me to store variables in files instead of passing them as variables.
+      - name: Output variables to files
+        run: |
+          echo $GITHUB_REPOSITORY > /tmp/git_repository
+          [ -z "$GITHUB_HEAD_REF" ] && echo $GITHUB_REF_NAME > /tmp/git_head_ref || echo $GITHUB_HEAD_REF > /tmp/git_head_ref
+          echo "-----BEGIN OPENSSH PRIVATE KEY-----" >> /tmp/key
+          echo ${{ env.BUILD_KEY }} >> /tmp/key
+          echo "-----END OPENSSH PRIVATE KEY-----" >> /tmp/key
+          chmod 400 /tmp/key
+          cat  /tmp/git_repository
+          cat  /tmp/git_head_ref
+          echo
+
+      - name: Raw SCP
+        run: |
+          scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/git_repository ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/git_repository
+          scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key /tmp/git_head_ref ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/git_head_ref
+          scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q -i /tmp/key .github/workflows/run_unit_test.sh ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/run_unit_test.sh
+
+      - name: Trigger Unit Test
+        run: |
+          ssh -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q  -i /tmp/key ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }} go install github.com/onsi/ginkgo/ginkgo@latest
+          ssh -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q  -i /tmp/key ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }} /tmp/run_unit_test.sh
+
+      - name: Get the logs and cat them
+        run: |
+          scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q  -i /tmp/key ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/test.log .
+          cat ./test.log
+
+      - name: Check Error Code
+        run: |
+          scp -o 'StrictHostKeyChecking no' -o UserKnownHostsFile=/dev/null -q  -i /tmp/key ${{ env.BUILD_USERNAME }}@${{ env.BUILD_HOSTNAME }}:/tmp/return_test.txt .
+          [ $(cat ./return_test.txt) -eq 0 ]
+
+  integrationtest:
+    name: Run integration tests
+    env:
+      STACK_ORCHESTRATOR_REF: fcbc74451c5494664fe21f765e89c9c6565c07cb
+      GO_ETHEREUM_REF: 498101102c891c4f8c3cab5649158c642ee1fd6b
+      GOPATH: /tmp/go
+      DB_WRITE: true
+      ETH_FORWARD_ETH_CALLS: false
+      ETH_PROXY_ON_ERROR: false
+      ETH_HTTP_PATH: "go-ethereum:8545"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-server"
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.STACK_ORCHESTRATOR_REF }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.GO_ETHEREUM_REF }}
+          repository: vulcanize/go-ethereum
+          path: "./go-ethereum/"
+      - name: Create config file
+        run: |
+          echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ > ./config.sh
+          echo vulcanize_ipld_eth_server=$GITHUB_WORKSPACE/ipld-eth-server/ >> ./config.sh
+          echo db_write=$DB_WRITE >> ./config.sh
+          echo eth_forward_eth_calls=$ETH_FORWARD_ETH_CALLS >> ./config.sh
+          echo eth_proxy_on_error=$ETH_PROXY_ON_ERROR >> ./config.sh
+          echo eth_http_path=$ETH_HTTP_PATH >> ./config.sh
+          cat ./config.sh
+      - name: Build geth
+        run: |
+          cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts
+          ./compile-geth.sh \
+          -p "$GITHUB_WORKSPACE/config.sh" \
+          -e docker
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-db.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-server.yml" \
+          --env-file "$GITHUB_WORKSPACE/config.sh" \
+          up -d --build
+      - name: Test
+        run: |
+          cd $GITHUB_WORKSPACE/ipld-eth-server
+          while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8081)" != "200" ]; do echo "waiting for ipld-eth-server..." && sleep 5; done && \
+          while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
+          make integrationtest
+
+  integrationtest_forwardethcalls:
+    name: Run integration tests for direct proxy fall-through of eth_calls
+    env:
+      STACK_ORCHESTRATOR_REF: fcbc74451c5494664fe21f765e89c9c6565c07cb
+      GO_ETHEREUM_REF: 498101102c891c4f8c3cab5649158c642ee1fd6b
+      GOPATH: /tmp/go
+      DB_WRITE: false
+      ETH_FORWARD_ETH_CALLS: true
+      ETH_PROXY_ON_ERROR: false
+      ETH_HTTP_PATH: "go-ethereum:8545"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ">=1.18.0"
+          check-latest: true
+      - uses: actions/checkout@v2
+        with:
+          path: "./ipld-eth-server"
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.STACK_ORCHESTRATOR_REF }}
+          path: "./stack-orchestrator/"
+          repository: vulcanize/stack-orchestrator
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.GO_ETHEREUM_REF }}
+          repository: vulcanize/go-ethereum
+          path: "./go-ethereum/"
+      - name: Create config file
+        run: |
+          echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ > ./config.sh
+          echo vulcanize_ipld_eth_server=$GITHUB_WORKSPACE/ipld-eth-server/ >> ./config.sh
+          echo db_write=$DB_WRITE >> ./config.sh
+          echo eth_forward_eth_calls=$ETH_FORWARD_ETH_CALLS >> ./config.sh
+          echo eth_proxy_on_error=$ETH_PROXY_ON_ERROR >> ./config.sh
+          echo eth_http_path=$ETH_HTTP_PATH >> ./config.sh
+          cat ./config.sh
+      - name: Build geth
+        run: |
+          cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts
+          ./compile-geth.sh \
+          -p "$GITHUB_WORKSPACE/config.sh" \
+          -e docker
+      - name: Run docker compose
+        run: |
+          docker-compose  \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/latest/docker-compose-db.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
+          -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-ipld-eth-server.yml" \
+          --env-file "$GITHUB_WORKSPACE/config.sh" \
+          up -d --build
+      - name: Test
+        run: |
+          cd $GITHUB_WORKSPACE/ipld-eth-server
+          while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8081)" != "200" ]; do echo "waiting for ipld-eth-server..." && sleep 5; done && \
+          while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
+          make integrationtest

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13-alpine as builder
+FROM golang:1.18-alpine as builder
 
 RUN apk --update --no-cache add make git g++ linux-headers
 # DEBUG
@@ -6,8 +6,17 @@ RUN apk add busybox-extras
 
 # Build ipld-eth-server
 WORKDIR /go/src/github.com/vulcanize/ipld-eth-server
-ADD . .
+
-RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-eth-server .
+# Cache the modules
+ENV GO111MODULE=on
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+COPY . .
+
+# Build the binary
+RUN GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-eth-server .
 
 # Copy migration tool
 WORKDIR /
@@ -35,7 +44,6 @@ COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-serv
 # keep binaries immutable
 COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/ipld-eth-server ipld-eth-server
 COPY --from=builder /goose goose
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb
 COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/environments environments
 
 ENTRYPOINT ["/app/entrypoint.sh"]

Makefile

@@ -3,16 +3,11 @@ BASE = $(GOPATH)/src/$(PACKAGE)
 PKGS = go list ./... | grep -v "^vendor/"
 
 # Tools
-## Testing library
-GINKGO = $(BIN)/ginkgo
-$(BIN)/ginkgo:
-	go get -u github.com/onsi/ginkgo/ginkgo
 
 ## Migration tool
 GOOSE = $(BIN)/goose
 $(BIN)/goose:
-	go get -u -d github.com/pressly/goose/cmd/goose
+	go get -u github.com/pressly/goose/cmd/goose
-	go build -tags='no_mysql no_sqlite' -o $(BIN)/goose github.com/pressly/goose/cmd/goose
 
 ## Source linter
 LINT = $(BIN)/golint
@@ -27,8 +22,9 @@ $(BIN)/gometalinter.v2:
 
 
 .PHONY: installtools
-installtools: | $(LINT) $(GOOSE) $(GINKGO)
+installtools: | $(LINT) $(GOOSE)
 	echo "Installing tools"
+	go mod download
 
 .PHONY: metalint
 metalint: | $(METALINT)
@@ -46,33 +42,31 @@ HOST_NAME = localhost
 PORT = 5432
 NAME =
 USER = postgres
-CONNECT_STRING=postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
+PASSWORD = password
+CONNECT_STRING=postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
 
 #Test
 TEST_DB = vulcanize_testing
-TEST_CONNECT_STRING = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
+TEST_CONNECT_STRING = postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(TEST_DB)?sslmode=disable
+TEST_CONNECT_STRING_LOCAL = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
 
 .PHONY: test
-test: | $(GINKGO) $(LINT)
+test: |  $(GOOSE)
 	go vet ./...
 	go fmt ./...
-	dropdb --if-exists $(TEST_DB)
+	go run github.com/onsi/ginkgo/ginkgo  -r --skipPackage=test
-	createdb $(TEST_DB)
-	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
-	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" reset
-	make migrate NAME=$(TEST_DB)
-	$(GINKGO) -r --skipPackage=integration_tests,integration
 
 .PHONY: integrationtest
-integrationtest: | $(GINKGO) $(LINT)
+integrationtest: | $(GOOSE)
 	go vet ./...
 	go fmt ./...
-	dropdb --if-exists $(TEST_DB)
+	go run github.com/onsi/ginkgo/ginkgo -r test/ -v
-	createdb $(TEST_DB)
+
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" up
+.PHONY: test_local
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" reset
+test_local: | $(GOOSE)
-	make migrate NAME=$(TEST_DB)
+	go vet ./...
-	$(GINKGO) -r integration_test/
+	go fmt ./...
+	./scripts/run_unit_test.sh
 
 build:
 	go fmt ./...

README.md

@@ -14,7 +14,7 @@
 ## Background
 NOTE: WIP
 
-ipld-eth-server is used to service queries against the indexed Ethereum IPLD objects indexed by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer).
+ipld-eth-server is used to service queries against the Ethereum IPLD objects indexed by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer).
 
 It exposes standard Ethereum JSON RPC endpoints on top of the database, in some cases these endpoints can leverage the unique indexes to improve query performance.
 Additional, unique endpoints are exposed which utilize the new indexes and state diff data objects.
@@ -33,9 +33,9 @@ External dependency
 ## Install
 Start by downloading ipld-eth-server and moving into the repo:
 
-`GO111MODULE=off go get -d github.com/vulcanize/ipld-eth-server`
+`GO111MODULE=off go get -d github.com/vulcanize/ipld-eth-server/v3`
 
-`cd $GOPATH/src/github.com/vulcanize/ipld-eth-server`
+`cd $GOPATH/src/github.com/vulcanize/ipld-eth-server/v3@v3.x.x`
 
 Then, build the binary:
 
@@ -66,15 +66,23 @@ The corresponding CLI flags can be found with the `./ipld-eth-server serve --hel
     ipcPath = "~/.vulcanize/vulcanize.ipc" # $SERVER_IPC_PATH
     wsPath = "127.0.0.1:8081" # $SERVER_WS_PATH
     httpPath = "127.0.0.1:8082" # $SERVER_HTTP_PATH
+    graphql = true # $SERVER_GRAPHQL
+    graphqlEndpoint = "" # $SERVER_GRAPHQL_ENDPOINT
 
 [ethereum]
     chainID = "1" # $ETH_CHAIN_ID
     defaultSender = "" # $ETH_DEFAULT_SENDER_ADDR
+    rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
+    httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    nodeID = "arch1" # $ETH_NODE_ID
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
+    networkID = "1" # $ETH_NETWORK_ID
 ```
 
-The `database` fields are for connecting to a Postgres database that has been/is being populated by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer).
+The `database` fields are for connecting to a Postgres database that has been/is being populated by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer)  
 The `server` fields set the paths for exposing the ipld-eth-server endpoints  
-The `ethereum` fields set the chainID and default sender address to use for EVM simulation
+The `ethereum` fields set the chainID and default sender address to use for EVM simulation, and can optionally be used to configure a remote eth node to forward cache misses to  
 
 
 ### Endpoints
@@ -85,18 +93,63 @@ TODO: Port the IPLD RPC subscription endpoints after the decoupling
 ipld-eth-server currently recapitulates portions of the Ethereum JSON-RPC api standard.
 
 The currently supported standard endpoints are:  
+`eth_call`  
+`eth_getBalance`  
+`eth_getStorageAt`  
+`eth_getCode`  
+`eth_getProof`  
 `eth_blockNumber`  
-`eth_getLogs`  
 `eth_getHeaderByNumber`  
+`eth_getHeaderByHash`  
 `eth_getBlockByNumber`  
 `eth_getBlockByHash`  
+`eth_getTransactionCount`  
+`eth_getBlockTransactionCountByHash`  
+`eth_getBlockTransactionCountByNumber`  
 `eth_getTransactionByHash`  
+`eth_getRawTransactionByHash`  
+`eth_getTransactionByBlockHashAndIndex`  
+`eth_getTransactionByBlockNumberAndIndex`  
+`eth_getRawTransactionByBlockHashAndIndex`  
+`eth_getRawTransactionByBlockNumberAndIndex`  
+`eth_getTransactionReceipt`  
+`eth_getLogs`  
+`eth_getUncleCountByBlockHash`  
+`eth_getUncleCountByBlockNumber`  
+`eth_getUncleByBlockHashAndIndex`  
+`eth_getUncleByBlockNumberAndIndex`  
+
+TODO: Add the rest of the standard endpoints and unique endpoints (e.g. getSlice)
+
+
+### CLI Options and Environment variables
+
+
+| CLI Option                    | Environment Variable          | Default Value    | Comment                             |
+| ----------------------------- | ----------------------------- | ---------------- | ----------------------------------- |
+| `database-hostname`           | `DATABASE_HOSTNAME`           | localhost        | IPLD database host                  |
+| `database-port`               | `DATABASE_PORT`               | 5432             | IPLD database port                  |
+| `database-name`               | `DATABASE_NAME`               | vulcanize_public | IPLD database name                  |
+| `database-user`               | `DATABASE_USER`               |                  | IPLD database user                  |
+| `database-password`           | `DATABASE_PASSWORD`           |                  | IPLD database password              |
+| `eth-server-graphql`          | `ETH_SERVER_GRAPHQL`          | false            | If `true` enable Eth GraphQL Server |
+| `eth-server-graphql-path` | `ETH_SERVER_GRAPHQLPATH` |                  | If `eth-server-graphql` set to true, endpoint url for graphql server (host:port)                               |
+| `eth-server-http`          | `ETH_SERVER_HTTP`          | true            | If `true` enable Eth HTTP JSON-RPC Server |
+| `eth-server-http-path`          | `ETH_SERVER_HTTPPATH`          |             | If `eth-server-http` set to `true`, endpoint url for Eth HTTP JSON-RPC server (host:port)  |
+| `eth-server-ws`          | `ETH_SERVER_WS`          | false            | If `true` enable Eth WS JSON-RPC Server |
+| `eth-server-ws-path`          | `ETH_SERVER_WSPATH`          |             | If `eth-server-ws` set to `true`, endpoint url for Eth WS JSON-RPC server (host:port)  |
+| `eth-server-ipc`          | `ETH_SERVER_IPC`          | false            | If `true` enable Eth IPC JSON-RPC Server |
+| `eth-server-ipc-path`          | `ETH_SERVER_IPC_PATH`          |             | If `eth-server-ws` set to `true`, path for Eth IPC JSON-RPC server |
+| `ipld-server-graphql`          | `IPLD_SERVER_GRAPHQL`          | false            | If `true` enable IPLD GraphQL Server |
+| `ipld-server-graphql-path` | `IPLD_SERVER_GRAPHQLPATH` |                  | If `ipld-server-graphql` set to true, endpoint url for graphql server (host:port)                               |
+| `ipld-postgraphile-path` | `IPLD_POSTGRAPHILEPATH` |                  | If `ipld-server-graphql` set to true, http url for postgraphile server on top of IPLD db                        |
+| `tracing-http-path` | `TRACING_HTTPPATH` |                  | If `ipld-server-graphql` set to true, http url for tracing server                        |
+| `tracing-postgraphile-path` | `TRACING.POSTGRAPHILEPATH` |                  | If `ipld-server-graphql` set to true, http url for postgraphile server on top of tracing db                        |
 
-TODO: Add the rest of the standard endpoints add unique endpoints (e.g. getSlice)
 
 ### Testing
-`make test` will run the unit tests  
+
-`make test` setups a clean `vulcanize_testing` db
+Follow steps in [test/README.md](./test/README.md)
 
 ## Monitoring
 

chain.json

@@ -0,0 +1,16 @@
+{
+  "chainId": 4,
+  "homesteadBlock": 1,
+  "eip150Block": 2,
+  "eip150Hash": "0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9",
+  "eip155Block": 3,
+  "eip158Block": 3,
+  "byzantiumBlock": 3,
+  "constantinopleBlock": 3,
+  "petersburgBlock": 3,
+  "istanbulBlock": 3,
+  "clique": {
+    "period": 15,
+    "epoch": 30000
+  }
+}

cmd/common.go

@@ -0,0 +1,37 @@
+// Copyright © 2021 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func addDatabaseFlags(command *cobra.Command) {
+	// database flags
+	command.PersistentFlags().String("database-name", "vulcanize_public", "database name")
+	command.PersistentFlags().Int("database-port", 5432, "database port")
+	command.PersistentFlags().String("database-hostname", "localhost", "database hostname")
+	command.PersistentFlags().String("database-user", "", "database user")
+	command.PersistentFlags().String("database-password", "", "database password")
+
+	// database flag bindings
+	viper.BindPFlag("database.name", command.PersistentFlags().Lookup("database-name"))
+	viper.BindPFlag("database.port", command.PersistentFlags().Lookup("database-port"))
+	viper.BindPFlag("database.hostname", command.PersistentFlags().Lookup("database-hostname"))
+	viper.BindPFlag("database.user", command.PersistentFlags().Lookup("database-user"))
+	viper.BindPFlag("database.password", command.PersistentFlags().Lookup("database-password"))
+}

cmd/root.go

@@ -19,16 +19,20 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
+	"github.com/joho/godotenv"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/ipld-eth-server/pkg/prom"
+
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/prom"
 )
 
 var (
 	cfgFile        string
+	envFile        string
 	subCommand     string
 	logWithCommand log.Entry
 )
@@ -46,7 +50,8 @@ func Execute() {
 }
 
 func initFuncs(cmd *cobra.Command, args []string) {
-	logfile := viper.GetString("logfile")
+	viper.BindEnv("log.file", "LOGRUS_FILE")
+	logfile := viper.GetString("log.file")
 	if logfile != "" {
 		file, err := os.OpenFile(logfile,
 			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
@@ -68,11 +73,11 @@ func initFuncs(cmd *cobra.Command, args []string) {
 		prom.Init()
 	}
 
-	if viper.GetBool("http") {
+	if viper.GetBool("prom.http") {
 		addr := fmt.Sprintf(
 			"%s:%s",
-			viper.GetString("http.addr"),
+			viper.GetString("prom.http.addr"),
-			viper.GetString("http.port"),
+			viper.GetString("prom.http.port"),
 		)
 		prom.Serve(addr)
 	}
@@ -98,45 +103,55 @@ func init() {
 	viper.AutomaticEnv()
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
-	rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
+	rootCmd.PersistentFlags().StringVar(&envFile, "env", "", "environment file location")
-	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
+
-	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
-	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
-	rootCmd.PersistentFlags().String("database-user", "", "database user")
-	rootCmd.PersistentFlags().String("database-password", "", "database password")
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
-	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
+	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
+	rootCmd.PersistentFlags().String("log-file", "", "file path for logging")
 
 	rootCmd.PersistentFlags().Bool("metrics", false, "enable metrics")
 
-	rootCmd.PersistentFlags().Bool("http", false, "enable http service for prometheus")
+	rootCmd.PersistentFlags().Bool("prom-http", false, "enable http service for prometheus")
-	rootCmd.PersistentFlags().String("http-addr", "127.0.0.1", "http host for prometheus")
+	rootCmd.PersistentFlags().String("prom-http-addr", "127.0.0.1", "http host for prometheus")
-	rootCmd.PersistentFlags().String("http-port", "8090", "http port for prometheus")
+	rootCmd.PersistentFlags().String("prom-http-port", "8090", "http port for prometheus")
 
-	viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
-	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
-	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
-	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
-	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
-	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
+	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
 
 	viper.BindPFlag("metrics", rootCmd.PersistentFlags().Lookup("metrics"))
 
-	viper.BindPFlag("http", rootCmd.PersistentFlags().Lookup("http"))
+	viper.BindPFlag("prom.http", rootCmd.PersistentFlags().Lookup("prom-http"))
-	viper.BindPFlag("http.addr", rootCmd.PersistentFlags().Lookup("http-addr"))
+	viper.BindPFlag("prom.http.addr", rootCmd.PersistentFlags().Lookup("prom-http-addr"))
-	viper.BindPFlag("http.port", rootCmd.PersistentFlags().Lookup("http-port"))
+	viper.BindPFlag("prom.http.port", rootCmd.PersistentFlags().Lookup("prom-http-port"))
 }
 
 func initConfig() {
+	if cfgFile == "" && envFile == "" {
+		log.Fatal("No configuration file specified, use --config , --env flag to provide configuration")
+	}
+
 	if cfgFile != "" {
+		if filepath.Ext(cfgFile) != ".toml" {
+			log.Fatal("Provide .toml file for --config flag")
+		}
+
 		viper.SetConfigFile(cfgFile)
-		if err := viper.ReadInConfig(); err == nil {
+		if err := viper.ReadInConfig(); err != nil {
-			log.Printf("Using config file: %s", viper.ConfigFileUsed())
+			log.Fatalf("Couldn't read config file: %s", err.Error())
-		} else {
-			log.Fatal(fmt.Sprintf("Couldn't read config file: %s", err.Error()))
 		}
-	} else {
+
-		log.Warn("No config file passed with --config flag")
+		log.Infof("Using config file: %s", viper.ConfigFileUsed())
+	}
+
+	if envFile != "" {
+		if filepath.Ext(envFile) != ".env" {
+			log.Fatal("Provide .env file for --env flag")
+		}
+
+		if err := godotenv.Load(envFile); err != nil {
+			log.Fatalf("Failed to set environment variable from env file: %s", err.Error())
+		}
+
+		log.Infof("Using env file: %s", envFile)
 	}
 }

cmd/serve.go

@@ -16,23 +16,31 @@
 package cmd
 
 import (
+	"errors"
+	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
+	"strings"
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/mailgun/groupcache/v2"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"github.com/vulcanize/gap-filler/pkg/mux"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/graphql"
-	srpc "github.com/vulcanize/ipld-eth-server/pkg/rpc"
+	srpc "github.com/vulcanize/ipld-eth-server/v3/pkg/rpc"
-
+	s "github.com/vulcanize/ipld-eth-server/v3/pkg/serve"
-	s "github.com/vulcanize/ipld-eth-server/pkg/serve"
+	v "github.com/vulcanize/ipld-eth-server/v3/version"
-	v "github.com/vulcanize/ipld-eth-server/version"
 )
 
+var ErrNoRpcEndpoints = errors.New("no rpc endpoints is available")
+
 // serveCmd represents the serve command
 var serveCmd = &cobra.Command{
 	Use:   "serve",
@@ -70,49 +78,334 @@ func serve() {
 	if err := startServers(server, serverConfig); err != nil {
 		logWithCommand.Fatal(err)
 	}
+	graphQL, err := startEthGraphQL(server, serverConfig)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
 
-	shutdown := make(chan os.Signal)
+	err = startIpldGraphQL(serverConfig)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	err = startGroupCacheService(serverConfig)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	if serverConfig.StateValidationEnabled {
+		go startStateTrieValidator(serverConfig, server)
+		logWithCommand.Info("state validator enabled")
+	} else {
+		logWithCommand.Info("state validator disabled")
+	}
+
+	shutdown := make(chan os.Signal, 1)
 	signal.Notify(shutdown, os.Interrupt)
 	<-shutdown
+	if graphQL != nil {
+		graphQL.Stop()
+	}
 	server.Stop()
 	wg.Wait()
 }
 
 func startServers(server s.Server, settings *s.Config) error {
+	if settings.IPCEnabled {
 		logWithCommand.Info("starting up IPC server")
 		_, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
 		if err != nil {
 			return err
 		}
+	} else {
+		logWithCommand.Info("IPC server is disabled")
+	}
+
+	if settings.WSEnabled {
 		logWithCommand.Info("starting up WS server")
-	_, _, err = srpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb"}, nil, true)
+		_, _, err := srpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb", "net"}, nil, true)
 		if err != nil {
 			return err
 		}
-	logWithCommand.Info("starting up HTTP server")
+	} else {
-	_, _, err = srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"eth"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
+		logWithCommand.Info("WS server is disabled")
+	}
 
+	if settings.HTTPEnabled {
+		logWithCommand.Info("starting up HTTP server")
+		_, err := srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"vdb", "eth", "net"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
+		if err != nil {
 			return err
 		}
+	} else {
+		logWithCommand.Info("HTTP server is disabled")
+	}
+
+	return nil
+}
+
+func startEthGraphQL(server s.Server, settings *s.Config) (graphQLServer *graphql.Service, err error) {
+	if settings.EthGraphqlEnabled {
+		logWithCommand.Info("starting up ETH GraphQL server")
+		endPoint := settings.EthGraphqlEndpoint
+		if endPoint != "" {
+			graphQLServer, err = graphql.New(server.Backend(), endPoint, nil, []string{"*"}, rpc.HTTPTimeouts{})
+			if err != nil {
+				return
+			}
+			err = graphQLServer.Start(nil)
+		}
+	} else {
+		logWithCommand.Info("ETH GraphQL server is disabled")
+	}
+
+	return
+}
+
+func startIpldGraphQL(settings *s.Config) error {
+	if settings.IpldGraphqlEnabled {
+		logWithCommand.Info("starting up IPLD GraphQL server")
+
+		gqlIpldAddr, err := url.Parse(settings.IpldPostgraphileEndpoint)
+		if err != nil {
+			return err
+		}
+
+		gqlTracingAPIAddr, err := url.Parse(settings.TracingPostgraphileEndpoint)
+		if err != nil {
+			return err
+		}
+
+		ethClients, err := parseRpcAddresses(settings.EthHttpEndpoint)
+		if err != nil {
+			return err
+		}
+
+		var tracingClients []*rpc.Client
+		tracingEndpoint := viper.GetString("tracing.httpPath")
+		if tracingEndpoint != "" {
+			tracingClients, err = parseRpcAddresses(tracingEndpoint)
+			if err != nil {
+				return err
+			}
+		}
+
+		router, err := mux.NewServeMux(&mux.Options{
+			BasePath:       "/",
+			EnableGraphiQL: true,
+			Postgraphile: mux.PostgraphileOptions{
+				Default:    gqlIpldAddr,
+				TracingAPI: gqlTracingAPIAddr,
+			},
+			RPC: mux.RPCOptions{
+				DefaultClients: ethClients,
+				TracingClients: tracingClients,
+			},
+		})
+		if err != nil {
+			return err
+		}
+
+		go http.ListenAndServe(settings.IpldGraphqlEndpoint, router)
+	} else {
+		logWithCommand.Info("IPLD GraphQL server is disabled")
+	}
+
+	return nil
+}
+
+func startGroupCacheService(settings *s.Config) error {
+	gcc := settings.GroupCache
+
+	if gcc.Pool.Enabled {
+		logWithCommand.Info("starting up groupcache pool HTTTP server")
+
+		pool := groupcache.NewHTTPPoolOpts(gcc.Pool.HttpEndpoint, &groupcache.HTTPPoolOptions{})
+		pool.Set(gcc.Pool.PeerHttpEndpoints...)
+
+		httpURL, err := url.Parse(gcc.Pool.HttpEndpoint)
+		if err != nil {
+			return err
+		}
+
+		server := http.Server{
+			Addr:    httpURL.Host,
+			Handler: pool,
+		}
+
+		// Start a HTTP server to listen for peer requests from the groupcache
+		go server.ListenAndServe()
+
+		logWithCommand.Infof("groupcache pool endpoint opened for url %s", httpURL)
+	} else {
+		logWithCommand.Info("Groupcache pool is disabled")
+	}
+
+	return nil
+}
+
+func startStateTrieValidator(config *s.Config, server s.Server) {
+	validateEveryNthBlock := config.StateValidationEveryNthBlock
+
+	var lastBlockNumber uint64
+	backend := server.Backend()
+
+	for {
+		time.Sleep(5 * time.Second)
+
+		block, err := backend.CurrentBlock()
+		if err != nil {
+			log.Errorln("Error fetching current block for state trie validator")
+			continue
+		}
+
+		stateRoot := block.Root()
+		blockNumber := block.NumberU64()
+		blockHash := block.Hash()
+
+		if validateEveryNthBlock <= 0 || // Used for static replicas where block number doesn't progress.
+			(blockNumber > lastBlockNumber) && (blockNumber%validateEveryNthBlock == 0) {
+
+			// The validate trie call will take a long time on mainnet, e.g. a few hours.
+			if err = backend.ValidateTrie(stateRoot); err != nil {
+				log.Fatalf("Error validating trie for block number %d hash %s state root %s",
+					blockNumber,
+					blockHash,
+					stateRoot,
+				)
+			}
+
+			log.Infof("Successfully validated trie for block number %d hash %s state root %s",
+				blockNumber,
+				blockHash,
+				stateRoot,
+			)
+
+			if validateEveryNthBlock <= 0 {
+				// Static replica, sleep a long-ish time (1/2 of cache expiry time) since we only need to keep the cache warm.
+				time.Sleep((time.Minute * time.Duration(config.GroupCache.StateDB.CacheExpiryInMins)) / 2)
+			}
+
+			lastBlockNumber = blockNumber
+		}
+	}
+}
+
+func parseRpcAddresses(value string) ([]*rpc.Client, error) {
+	rpcAddresses := strings.Split(value, ",")
+	rpcClients := make([]*rpc.Client, 0, len(rpcAddresses))
+	for _, address := range rpcAddresses {
+		rpcClient, err := rpc.Dial(address)
+		if err != nil {
+			logWithCommand.Errorf("couldn't connect to %s. Error: %s", address, err)
+			continue
+		}
+
+		rpcClients = append(rpcClients, rpcClient)
+	}
+
+	if len(rpcClients) == 0 {
+		logWithCommand.Error(ErrNoRpcEndpoints)
+		return nil, ErrNoRpcEndpoints
+	}
+
+	return rpcClients, nil
+}
 
 func init() {
 	rootCmd.AddCommand(serveCmd)
 
-	// flags for all config variables
+	addDatabaseFlags(serveCmd)
-	serveCmd.PersistentFlags().String("server-ws-path", "", "vdb server ws path")
-	serveCmd.PersistentFlags().String("server-http-path", "", "vdb server http path")
-	serveCmd.PersistentFlags().String("server-ipc-path", "", "vdb server ipc path")
 
+	// flags for all config variables
+	// eth graphql and json-rpc parameters
+	serveCmd.PersistentFlags().Bool("eth-server-graphql", false, "turn on the eth graphql server")
+	serveCmd.PersistentFlags().String("eth-server-graphql-path", "", "endpoint url for eth graphql server (host:port)")
+	serveCmd.PersistentFlags().Bool("eth-server-http", true, "turn on the eth http json-rpc server")
+	serveCmd.PersistentFlags().String("eth-server-http-path", "", "endpoint url for eth http json-rpc server (host:port)")
+	serveCmd.PersistentFlags().Bool("eth-server-ws", false, "turn on the eth websocket json-rpc server")
+	serveCmd.PersistentFlags().String("eth-server-ws-path", "", "endpoint url for eth websocket json-rpc server (host:port)")
+	serveCmd.PersistentFlags().Bool("eth-server-ipc", false, "turn on the eth ipc json-rpc server")
+	serveCmd.PersistentFlags().String("eth-server-ipc-path", "", "path for eth ipc json-rpc server")
+
+	// ipld and tracing graphql parameters
+	serveCmd.PersistentFlags().Bool("ipld-server-graphql", false, "turn on the ipld graphql server")
+	serveCmd.PersistentFlags().String("ipld-server-graphql-path", "", "endpoint url for ipld graphql server (host:port)")
+	serveCmd.PersistentFlags().String("ipld-postgraphile-path", "", "http url to postgraphile on top of ipld database")
+	serveCmd.PersistentFlags().String("tracing-http-path", "", "http url to tracing service")
+	serveCmd.PersistentFlags().String("tracing-postgraphile-path", "", "http url to postgraphile on top of tracing db")
+
+	serveCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
+	serveCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
+	serveCmd.PersistentFlags().String("eth-client-name", "Geth", "eth client name")
+	serveCmd.PersistentFlags().String("eth-genesis-block", "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", "eth genesis block hash")
+	serveCmd.PersistentFlags().String("eth-network-id", "1", "eth network id")
 	serveCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
 	serveCmd.PersistentFlags().String("eth-default-sender", "", "default sender address")
 	serveCmd.PersistentFlags().String("eth-rpc-gas-cap", "", "rpc gas cap (for eth_Call execution)")
+	serveCmd.PersistentFlags().String("eth-chain-config", "", "json chain config file location")
+	serveCmd.PersistentFlags().Bool("eth-supports-state-diff", false, "whether the proxy ethereum client supports statediffing endpoints")
+	serveCmd.PersistentFlags().Bool("eth-forward-eth-calls", false, "whether to immediately forward eth_calls to proxy client")
+	serveCmd.PersistentFlags().Bool("eth-proxy-on-error", true, "whether to forward all failed calls to proxy client")
+
+	// groupcache flags
+	serveCmd.PersistentFlags().Bool("gcache-pool-enabled", false, "turn on the groupcache pool")
+	serveCmd.PersistentFlags().String("gcache-pool-http-path", "", "http url for groupcache node")
+	serveCmd.PersistentFlags().StringArray("gcache-pool-http-peers", []string{}, "http urls for groupcache peers")
+	serveCmd.PersistentFlags().Int("gcache-statedb-cache-size", 16, "state DB cache size in MB")
+	serveCmd.PersistentFlags().Int("gcache-statedb-cache-expiry", 60, "state DB cache expiry time in mins")
+	serveCmd.PersistentFlags().Int("gcache-statedb-log-stats-interval", 60, "state DB cache stats log interval in secs")
+
+	// state validator flags
+	serveCmd.PersistentFlags().Bool("validator-enabled", false, "turn on the state validator")
+	serveCmd.PersistentFlags().Uint("validator-every-nth-block", 1500, "only validate every Nth block")
 
 	// and their bindings
-	viper.BindPFlag("server.wsPath", serveCmd.PersistentFlags().Lookup("server-ws-path"))
+	// eth graphql server
-	viper.BindPFlag("server.httpPath", serveCmd.PersistentFlags().Lookup("server-http-path"))
+	viper.BindPFlag("eth.server.graphql", serveCmd.PersistentFlags().Lookup("eth-server-graphql"))
-	viper.BindPFlag("server.ipcPath", serveCmd.PersistentFlags().Lookup("server-ipc-path"))
+	viper.BindPFlag("eth.server.graphqlPath", serveCmd.PersistentFlags().Lookup("eth-server-graphql-path"))
 
+	// eth http json-rpc server
+	viper.BindPFlag("eth.server.http", serveCmd.PersistentFlags().Lookup("eth-server-http"))
+	viper.BindPFlag("eth.server.httpPath", serveCmd.PersistentFlags().Lookup("eth-server-http-path"))
+
+	// eth websocket json-rpc server
+	viper.BindPFlag("eth.server.ws", serveCmd.PersistentFlags().Lookup("eth-server-ws"))
+	viper.BindPFlag("eth.server.wsPath", serveCmd.PersistentFlags().Lookup("eth-server-ws-path"))
+
+	// eth ipc json-rpc server
+	viper.BindPFlag("eth.server.ipc", serveCmd.PersistentFlags().Lookup("eth-server-ipc"))
+	viper.BindPFlag("eth.server.ipcPath", serveCmd.PersistentFlags().Lookup("eth-server-ipc-path"))
+
+	// ipld and tracing graphql parameters
+	viper.BindPFlag("ipld.server.graphql", serveCmd.PersistentFlags().Lookup("ipld-server-graphql"))
+	viper.BindPFlag("ipld.server.graphqlPath", serveCmd.PersistentFlags().Lookup("ipld-server-graphql-path"))
+	viper.BindPFlag("ipld.postgraphilePath", serveCmd.PersistentFlags().Lookup("ipld-postgraphile-path"))
+	viper.BindPFlag("tracing.httpPath", serveCmd.PersistentFlags().Lookup("tracing-http-path"))
+	viper.BindPFlag("tracing.postgraphilePath", serveCmd.PersistentFlags().Lookup("tracing-postgraphile-path"))
+
+	viper.BindPFlag("ethereum.httpPath", serveCmd.PersistentFlags().Lookup("eth-http-path"))
+	viper.BindPFlag("ethereum.nodeID", serveCmd.PersistentFlags().Lookup("eth-node-id"))
+	viper.BindPFlag("ethereum.clientName", serveCmd.PersistentFlags().Lookup("eth-client-name"))
+	viper.BindPFlag("ethereum.genesisBlock", serveCmd.PersistentFlags().Lookup("eth-genesis-block"))
+	viper.BindPFlag("ethereum.networkID", serveCmd.PersistentFlags().Lookup("eth-network-id"))
 	viper.BindPFlag("ethereum.chainID", serveCmd.PersistentFlags().Lookup("eth-chain-id"))
 	viper.BindPFlag("ethereum.defaultSender", serveCmd.PersistentFlags().Lookup("eth-default-sender"))
 	viper.BindPFlag("ethereum.rpcGasCap", serveCmd.PersistentFlags().Lookup("eth-rpc-gas-cap"))
+	viper.BindPFlag("ethereum.chainConfig", serveCmd.PersistentFlags().Lookup("eth-chain-config"))
+	viper.BindPFlag("ethereum.supportsStateDiff", serveCmd.PersistentFlags().Lookup("eth-supports-state-diff"))
+	viper.BindPFlag("ethereum.forwardEthCalls", serveCmd.PersistentFlags().Lookup("eth-forward-eth-calls"))
+	viper.BindPFlag("ethereum.proxyOnError", serveCmd.PersistentFlags().Lookup("eth-proxy-on-error"))
+
+	// groupcache flags
+	viper.BindPFlag("groupcache.pool.enabled", serveCmd.PersistentFlags().Lookup("gcache-pool-enabled"))
+	viper.BindPFlag("groupcache.pool.httpEndpoint", serveCmd.PersistentFlags().Lookup("gcache-pool-http-path"))
+	viper.BindPFlag("groupcache.pool.peerHttpEndpoints", serveCmd.PersistentFlags().Lookup("gcache-pool-http-peers"))
+	viper.BindPFlag("groupcache.statedb.cacheSizeInMB", serveCmd.PersistentFlags().Lookup("gcache-statedb-cache-size"))
+	viper.BindPFlag("groupcache.statedb.cacheExpiryInMins", serveCmd.PersistentFlags().Lookup("gcache-statedb-cache-expiry"))
+	viper.BindPFlag("groupcache.statedb.logStatsIntervalInSecs", serveCmd.PersistentFlags().Lookup("gcache-statedb-log-stats-interval"))
+
+	// state validator flags
+	viper.BindPFlag("validator.enabled", serveCmd.PersistentFlags().Lookup("validator-enabled"))
+	viper.BindPFlag("validator.everyNthBlock", serveCmd.PersistentFlags().Lookup("validator-every-nth-block"))
 }

cmd/subscribe.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -28,9 +27,9 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/ipld-eth-server/pkg/client"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/client"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-	w "github.com/vulcanize/ipld-eth-server/pkg/serve"
+	w "github.com/vulcanize/ipld-eth-server/v3/pkg/serve"
 )
 
 // subscribeCmd represents the subscribe command
@@ -129,7 +128,7 @@ func subscribe() {
 			}
 			// This assumes leafs only
 			for _, stateNode := range ethData.StateNodes {
-				var acct state.Account
+				var acct types.StateAccount
 				err = rlp.DecodeBytes(stateNode.IPLD.Data, &acct)
 				if err != nil {
 					logWithCommand.Error(err)

cmd/validate.go

@@ -0,0 +1,87 @@
+// Copyright © 2021 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	validator "github.com/vulcanize/eth-ipfs-state-validator/v3/pkg"
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb/v3/postgres"
+
+	s "github.com/vulcanize/ipld-eth-server/v3/pkg/serve"
+)
+
+const GroupName = "statedb-validate"
+const CacheExpiryInMins = 8 * 60 // 8 hours
+const CacheSizeInMB = 16         // 16 MB
+
+var validateCmd = &cobra.Command{
+	Use:   "validate",
+	Short: "valdiate state",
+	Long:  `This command validates the trie for the given state root`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *log.WithField("SubCommand", subCommand)
+		validate()
+	},
+}
+
+func validate() {
+	config, err := s.NewConfig()
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	stateRootStr := viper.GetString("stateRoot")
+	if stateRootStr == "" {
+		logWithCommand.Fatal("must provide a state root for state validation")
+	}
+
+	stateRoot := common.HexToHash(stateRootStr)
+	cacheSize := viper.GetInt("cacheSize")
+
+	ethDB := ipfsethdb.NewDatabase(config.DB, ipfsethdb.CacheConfig{
+		Name:           GroupName,
+		Size:           cacheSize * 1024 * 1024,
+		ExpiryDuration: time.Minute * time.Duration(CacheExpiryInMins),
+	})
+
+	val := validator.NewValidator(nil, ethDB)
+	if err = val.ValidateTrie(stateRoot); err != nil {
+		log.Fatalln("Error validating state root")
+	}
+
+	stats := ethDB.(*ipfsethdb.Database).GetCacheStats()
+	log.Debugf("groupcache stats %+v", stats)
+
+	log.Infoln("Successfully validated state root")
+}
+
+func init() {
+	rootCmd.AddCommand(validateCmd)
+
+	addDatabaseFlags(validateCmd)
+
+	validateCmd.PersistentFlags().String("state-root", "", "root of the state trie we wish to validate")
+	viper.BindPFlag("stateRoot", validateCmd.PersistentFlags().Lookup("state-root"))
+
+	validateCmd.PersistentFlags().Int("cache-size", CacheSizeInMB, "cache size in MB")
+	viper.BindPFlag("cacheSize", validateCmd.PersistentFlags().Lookup("cache-size"))
+}

cmd/version.go

@@ -19,7 +19,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	v "github.com/vulcanize/ipld-eth-server/version"
+	v "github.com/vulcanize/ipld-eth-server/v3/version"
 )
 
 // versionCmd represents the version command

db/migrations/00001_create_ipfs_blocks_table.sql

@@ -1,8 +0,0 @@
--- +goose Up
-CREATE TABLE IF NOT EXISTS public.blocks (
-  key TEXT UNIQUE NOT NULL,
-  data BYTEA NOT NULL
-);
-
--- +goose Down
-DROP TABLE public.blocks;

db/migrations/00002_create_nodes_table.sql

@@ -1,12 +0,0 @@
--- +goose Up
-CREATE TABLE nodes (
-  id            SERIAL PRIMARY KEY,
-  client_name   VARCHAR,
-  genesis_block VARCHAR(66),
-  network_id    VARCHAR,
-  node_id       VARCHAR(128),
-  CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id)
-);
-
--- +goose Down
-DROP TABLE nodes;

db/migrations/00003_create_eth_schema.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE SCHEMA eth;
-
--- +goose Down
-DROP SCHEMA eth;

db/migrations/00004_create_eth_header_cids_table.sql

@@ -1,23 +0,0 @@
--- +goose Up
-CREATE TABLE eth.header_cids (
-  id                    SERIAL PRIMARY KEY,
-  block_number          BIGINT NOT NULL,
-  block_hash            VARCHAR(66) NOT NULL,
-  parent_hash           VARCHAR(66) NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  td                    NUMERIC NOT NULL,
-  node_id               INTEGER NOT NULL REFERENCES nodes (id) ON DELETE CASCADE,
-  reward                NUMERIC NOT NULL,
-  state_root            VARCHAR(66) NOT NULL,
-  tx_root               VARCHAR(66) NOT NULL,
-  receipt_root          VARCHAR(66) NOT NULL,
-  uncle_root            VARCHAR(66) NOT NULL,
-  bloom                 BYTEA NOT NULL,
-  timestamp             NUMERIC NOT NULL,
-  times_validated       INTEGER NOT NULL DEFAULT 1,
-  UNIQUE (block_number, block_hash)
-);
-
--- +goose Down
-DROP TABLE eth.header_cids;

db/migrations/00005_create_eth_uncle_cids_table.sql

@@ -1,14 +0,0 @@
--- +goose Up
-CREATE TABLE eth.uncle_cids (
-  id                    SERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  block_hash            VARCHAR(66) NOT NULL,
-  parent_hash           VARCHAR(66) NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  reward                NUMERIC NOT NULL,
-  UNIQUE (header_id, block_hash)
-);
-
--- +goose Down
-DROP TABLE eth.uncle_cids;

db/migrations/00006_create_eth_transaction_cids_table.sql

@@ -1,17 +0,0 @@
--- +goose Up
-CREATE TABLE eth.transaction_cids (
-  id                    SERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  tx_hash               VARCHAR(66) NOT NULL,
-  index                 INTEGER NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  dst                   VARCHAR(66) NOT NULL,
-  src                   VARCHAR(66) NOT NULL,
-  deployment            BOOL NOT NULL,
-  tx_data               BYTEA,
-  UNIQUE (header_id, tx_hash)
-);
-
--- +goose Down
-DROP TABLE eth.transaction_cids;

db/migrations/00007_create_eth_receipt_cids_table.sql

@@ -1,18 +0,0 @@
--- +goose Up
-CREATE TABLE eth.receipt_cids (
-  id                    SERIAL PRIMARY KEY,
-  tx_id                 INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  contract              VARCHAR(66),
-  contract_hash         VARCHAR(66),
-  topic0s               VARCHAR(66)[],
-  topic1s               VARCHAR(66)[],
-  topic2s               VARCHAR(66)[],
-  topic3s               VARCHAR(66)[],
-  log_contracts         VARCHAR(66)[],
-  UNIQUE (tx_id)
-);
-
--- +goose Down
-DROP TABLE eth.receipt_cids;

db/migrations/00008_create_eth_state_cids_table.sql

@@ -1,15 +0,0 @@
--- +goose Up
-CREATE TABLE eth.state_cids (
-  id                    SERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  state_leaf_key        VARCHAR(66),
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  state_path            BYTEA,
-  node_type             INTEGER NOT NULL,
-  diff                  BOOLEAN NOT NULL DEFAULT FALSE,
-  UNIQUE (header_id, state_path)
-);
-
--- +goose Down
-DROP TABLE eth.state_cids;

db/migrations/00009_create_eth_storage_cids_table.sql

@@ -1,15 +0,0 @@
--- +goose Up
-CREATE TABLE eth.storage_cids (
-  id                    SERIAL PRIMARY KEY,
-  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  storage_leaf_key      VARCHAR(66),
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  storage_path          BYTEA,
-  node_type             INTEGER NOT NULL,
-  diff                  BOOLEAN NOT NULL DEFAULT FALSE,
-  UNIQUE (state_id, storage_path)
-);
-
--- +goose Down
-DROP TABLE eth.storage_cids;

db/migrations/00010_create_eth_state_accouts_table.sql

@@ -1,13 +0,0 @@
--- +goose Up
-CREATE TABLE eth.state_accounts (
-  id                    SERIAL PRIMARY KEY,
-  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  balance               NUMERIC NOT NULL,
-  nonce                 INTEGER NOT NULL,
-  code_hash             BYTEA NOT NULL,
-  storage_root          VARCHAR(66) NOT NULL,
-  UNIQUE (state_id)
-);
-
--- +goose Down
-DROP TABLE eth.state_accounts;

db/migrations/00011_create_postgraphile_comments.sql

@@ -1,6 +0,0 @@
--- +goose Up
-COMMENT ON TABLE public.nodes IS E'@name NodeInfo';
-COMMENT ON TABLE eth.transaction_cids IS E'@name EthTransactionCids';
-COMMENT ON TABLE eth.header_cids IS E'@name EthHeaderCids';
-COMMENT ON COLUMN public.nodes.node_id IS E'@name ChainNodeID';
-COMMENT ON COLUMN eth.header_cids.node_id IS E'@name EthNodeID';

db/migrations/00012_add_chain_id_to_nodes.sql

@@ -1,21 +0,0 @@
--- +goose Up
-ALTER TABLE public.nodes
-ADD COLUMN chain_id INTEGER DEFAULT 1;
-
-ALTER TABLE public.nodes
-DROP CONSTRAINT node_uc;
-
-ALTER TABLE public.nodes
-ADD CONSTRAINT node_uc
-UNIQUE (genesis_block, network_id, node_id, chain_id);
-
--- +goose Down
-ALTER TABLE public.nodes
-DROP CONSTRAINT node_uc;
-
-ALTER TABLE public.nodes
-ADD CONSTRAINT node_uc
-UNIQUE (genesis_block, network_id, node_id);
-
-ALTER TABLE public.nodes
-DROP COLUMN chain_id;

db/migrations/00013_potgraphile_triggers.sql

@@ -1,69 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-CREATE FUNCTION eth.graphql_subscription() returns TRIGGER as $$
-declare
-    table_name text = TG_ARGV[0];
-    attribute text = TG_ARGV[1];
-    id text;
-begin
-    execute 'select $1.' || quote_ident(attribute)
-        using new
-        into id;
-    perform pg_notify('postgraphile:' || table_name,
-                      json_build_object(
-                              '__node__', json_build_array(
-                              table_name,
-                              id
-                          )
-                          )::text
-        );
-    return new;
-end;
-$$ language plpgsql;
--- +goose StatementEnd
-
-CREATE TRIGGER header_cids_ai
-    after INSERT ON eth.header_cids
-    for each row
-    execute procedure eth.graphql_subscription('header_cids', 'id');
-
-CREATE TRIGGER receipt_cids_ai
-    after INSERT ON eth.receipt_cids
-    for each row
-    execute procedure eth.graphql_subscription('receipt_cids', 'id');
-
-CREATE TRIGGER state_accounts_ai
-    after INSERT ON eth.state_accounts
-    for each row
-    execute procedure eth.graphql_subscription('state_accounts', 'id');
-
-CREATE TRIGGER state_cids_ai
-    after INSERT ON eth.state_cids
-    for each row
-    execute procedure eth.graphql_subscription('state_cids', 'id');
-
-CREATE TRIGGER storage_cids_ai
-    after INSERT ON eth.storage_cids
-    for each row
-    execute procedure eth.graphql_subscription('storage_cids', 'id');
-
-CREATE TRIGGER transaction_cids_ai
-    after INSERT ON eth.transaction_cids
-    for each row
-    execute procedure eth.graphql_subscription('transaction_cids', 'id');
-
-CREATE TRIGGER uncle_cids_ai
-    after INSERT ON eth.uncle_cids
-    for each row
-    execute procedure eth.graphql_subscription('uncle_cids', 'id');
-
--- +goose Down
-DROP TRIGGER uncle_cids_ai ON eth.uncle_cids;
-DROP TRIGGER transaction_cids_ai ON eth.transaction_cids;
-DROP TRIGGER storage_cids_ai ON eth.storage_cids;
-DROP TRIGGER state_cids_ai ON eth.state_cids;
-DROP TRIGGER state_accounts_ai ON eth.state_accounts;
-DROP TRIGGER receipt_cids_ai ON eth.receipt_cids;
-DROP TRIGGER header_cids_ai ON eth.header_cids;
-
-DROP FUNCTION eth.graphql_subscription();

db/migrations/00014_create_cid_indexes.sql

@@ -1,121 +0,0 @@
--- +goose Up
--- header indexes
-CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
-
-CREATE INDEX block_hash_index ON eth.header_cids USING btree (block_hash);
-
-CREATE INDEX header_cid_index ON eth.header_cids USING btree (cid);
-
-CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
-
-CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
-
-CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp);
-
--- transaction indexes
-CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
-
-CREATE INDEX tx_hash_index ON eth.transaction_cids USING btree (tx_hash);
-
-CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
-
-CREATE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
-
-CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
-
-CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
-
--- receipt indexes
-CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id);
-
-CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
-
-CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
-
-CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
-
-CREATE INDEX rct_contract_hash_index ON eth.receipt_cids USING btree (contract_hash);
-
-CREATE INDEX rct_topic0_index ON eth.receipt_cids USING gin (topic0s);
-
-CREATE INDEX rct_topic1_index ON eth.receipt_cids USING gin (topic1s);
-
-CREATE INDEX rct_topic2_index ON eth.receipt_cids USING gin (topic2s);
-
-CREATE INDEX rct_topic3_index ON eth.receipt_cids USING gin (topic3s);
-
-CREATE INDEX rct_log_contract_index ON eth.receipt_cids USING gin (log_contracts);
-
--- state node indexes
-CREATE INDEX state_header_id_index ON eth.state_cids USING btree (header_id);
-
-CREATE INDEX state_leaf_key_index ON eth.state_cids USING btree (state_leaf_key);
-
-CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid);
-
-CREATE INDEX state_mh_index ON eth.state_cids USING btree (mh_key);
-
-CREATE INDEX state_path_index ON eth.state_cids USING btree (state_path);
-
--- storage node indexes
-CREATE INDEX storage_state_id_index ON eth.storage_cids USING btree (state_id);
-
-CREATE INDEX storage_leaf_key_index ON eth.storage_cids USING btree (storage_leaf_key);
-
-CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid);
-
-CREATE INDEX storage_mh_index ON eth.storage_cids USING btree (mh_key);
-
-CREATE INDEX storage_path_index ON eth.storage_cids USING btree (storage_path);
-
--- state accounts indexes
-CREATE INDEX account_state_id_index ON eth.state_accounts USING btree (state_id);
-
-CREATE INDEX storage_root_index ON eth.state_accounts USING btree (storage_root);
-
--- +goose Down
--- state account indexes
-DROP INDEX eth.storage_root_index;
-DROP INDEX eth.account_state_id_index;
-
--- storage node indexes
-DROP INDEX eth.storage_path_index;
-DROP INDEX eth.storage_mh_index;
-DROP INDEX eth.storage_cid_index;
-DROP INDEX eth.storage_leaf_key_index;
-DROP INDEX eth.storage_state_id_index;
-
--- state node indexes
-DROP INDEX eth.state_path_index;
-DROP INDEX eth.state_mh_index;
-DROP INDEX eth.state_cid_index;
-DROP INDEX eth.state_leaf_key_index;
-DROP INDEX eth.state_header_id_index;
-
--- receipt indexes
-DROP INDEX eth.rct_log_contract_index;
-DROP INDEX eth.rct_topic3_index;
-DROP INDEX eth.rct_topic2_index;
-DROP INDEX eth.rct_topic1_index;
-DROP INDEX eth.rct_topic0_index;
-DROP INDEX eth.rct_contract_hash_index;
-DROP INDEX eth.rct_contract_index;
-DROP INDEX eth.rct_mh_index;
-DROP INDEX eth.rct_cid_index;
-DROP INDEX eth.rct_tx_id_index;
-
--- transaction indexes
-DROP INDEX eth.tx_src_index;
-DROP INDEX eth.tx_dst_index;
-DROP INDEX eth.tx_mh_index;
-DROP INDEX eth.tx_cid_index;
-DROP INDEX eth.tx_hash_index;
-DROP INDEX eth.tx_header_id_index;
-
--- header indexes
-DROP INDEX eth.timestamp_index;
-DROP INDEX eth.state_root_index;
-DROP INDEX eth.header_mh_index;
-DROP INDEX eth.header_cid_index;
-DROP INDEX eth.block_hash_index;
-DROP INDEX eth.block_number_index;

db/migrations/00015_create_canonical_hash_finder_functions.sql

@@ -1,48 +0,0 @@
--- +goose Up
--- +goose StatementBegin
--- returns the number of child headers that reference backwards to the header with the provided hash
-CREATE OR REPLACE FUNCTION header_weight(hash VARCHAR(66)) RETURNS BIGINT
-AS $$
-  WITH RECURSIVE validator AS (
-          SELECT block_hash, parent_hash, block_number
-          FROM eth.header_cids
-          WHERE block_hash = hash
-      UNION
-          SELECT eth.header_cids.block_hash, eth.header_cids.parent_hash, eth.header_cids.block_number
-          FROM eth.header_cids
-          INNER JOIN validator
-            ON eth.header_cids.parent_hash = validator.block_hash
-            AND eth.header_cids.block_number = validator.block_number + 1
-  )
-  SELECT COUNT(*) FROM validator;
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose StatementBegin
--- returns the id for the header at the provided height which is heaviest
-CREATE OR REPLACE FUNCTION canonical_header(height BIGINT) RETURNS INT AS
-$BODY$
-DECLARE
-  current_weight INT;
-  heaviest_weight INT DEFAULT 0;
-  heaviest_id INT;
-  r eth.header_cids%ROWTYPE;
-BEGIN
-  FOR r IN SELECT * FROM eth.header_cids
-  WHERE block_number = height
-  LOOP
-    SELECT INTO current_weight * FROM header_weight(r.block_hash);
-    IF current_weight > heaviest_weight THEN
-        heaviest_weight := current_weight;
-        heaviest_id := r.id;
-    END IF;
-  END LOOP;
-  RETURN heaviest_id;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose Down
-DROP FUNCTION header_weight;
-DROP FUNCTION canonical_header;

db/migrations/00016_remove_deployment_from_tx_cids.sql

@@ -1,7 +0,0 @@
--- +goose Up
-ALTER TABLE eth.transaction_cids
-DROP COLUMN deployment;
-
--- +goose Down
-ALTER TABLE eth.transaction_cids
-ADD COLUMN deployment BOOL NOT NULL DEFAULT FALSE;

db/migrations/00017_state_and_storage_ids_use_big_serial.sql

@@ -1,25 +0,0 @@
--- +goose Up
-ALTER TABLE eth.storage_cids
-ALTER COLUMN state_id TYPE BIGINT;
-
-ALTER TABLE eth.state_accounts
-ALTER COLUMN state_id TYPE BIGINT;
-
-ALTER TABLE eth.state_cids
-ALTER COLUMN id TYPE BIGINT;
-
-ALTER TABLE eth.storage_cids
-ALTER COLUMN id TYPE BIGINT;
-
--- +goose Down
-ALTER TABLE eth.storage_cids
-ALTER COLUMN id TYPE INTEGER;
-
-ALTER TABLE eth.state_cids
-ALTER COLUMN id TYPE INTEGER;
-
-ALTER TABLE eth.state_accounts
-ALTER COLUMN state_id TYPE INTEGER;
-
-ALTER TABLE eth.storage_cids
-ALTER COLUMN state_id TYPE INTEGER;

docker-compose.test.yml

@@ -0,0 +1,12 @@
+version: '3.2'
+
+services:
+  contract:
+    build:
+      context: ./test/contract
+      args:
+        ETH_ADDR: "http://go-ethereum:8545"
+    environment:
+      ETH_ADDR: "http://go-ethereum:8545"
+    ports:
+      - "127.0.0.1:3000:3000"

docker-compose.yml

@@ -1,68 +1,54 @@
 version: '3.2'
 
 services:
-  dapptools:
+  ipld-eth-db:
-    restart: unless-stopped
-    image: vulcanize/dapptools:v0.29.0-statediff-0.0.2
-    ports:
-      - "127.0.0.1:8545:8545"
-      - "127.0.0.1:8546:8546"
-
-  db:
     restart: always
-    image: postgres:10.12-alpine
+    image: vulcanize/ipld-eth-db:v3.2.0
     environment:
       POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_public"
+      POSTGRES_DB: "vulcanize_testing"
       POSTGRES_PASSWORD: "password"
     volumes:
      - vdb_db_eth_server:/var/lib/postgresql/data
     ports:
      - "127.0.0.1:8077:5432"
-
+    command: ["postgres", "-c", "log_statement=all"]
-  eth-indexer:
-    restart: unless-stopped
-    depends_on:
-      - db
-      - dapptools
-    image: vulcanize/ipld-eth-indexer:v0.3.0-alpha
-    environment:
-      DATABASE_NAME: vulcanize_public
-      DATABASE_HOSTNAME: db
-      DATABASE_PORT: 5432
-      DATABASE_USER: vdbm
-      DATABASE_PASSWORD: password
-      ETH_WS_PATH: "dapptools:8546"
-      ETH_HTTP_PATH: "dapptools:8545"
-      ETH_CHAIN_ID: 4
-      ETH_NETWORK_ID: 4
-      VDB_COMMAND: sync
 
   eth-server:
+    restart: unless-stopped
     depends_on:
-      - db
+      - ipld-eth-db
     build:
       context: ./
       cache_from:
        - alpine:latest
        - golang:1.13-alpine
     environment:
+      IPLD_SERVER_GRAPHQL: "true"
+      IPLD_POSTGRAPHILEPATH: http://graphql:5000
+      ETH_SERVER_HTTPPATH: 0.0.0.0:8081
       VDB_COMMAND: "serve"
-      DATABASE_NAME: "vulcanize_public"
+      ETH_CHAIN_CONFIG: "/tmp/chain.json"
-      DATABASE_HOSTNAME: "db"
+      DATABASE_NAME: "vulcanize_testing"
+      DATABASE_HOSTNAME: "ipld-eth-db"
       DATABASE_PORT: 5432
       DATABASE_USER: "vdbm"
       DATABASE_PASSWORD: "password"
-      SERVER_WS_PATH: "0.0.0.0:8081"
+      ETH_CHAIN_ID: 4
-      SERVER_HTTP_PATH: "0.0.0.0:8082"
+      ETH_FORWARD_ETH_CALLS: $ETH_FORWARD_ETH_CALLS
+      ETH_PROXY_ON_ERROR: $ETH_PROXY_ON_ERROR
+      ETH_HTTP_PATH: $ETH_HTTP_PATH
+    volumes:
+    - type: bind
+      source: ./chain.json
+      target: /tmp/chain.json
     ports:
-     - "127.0.0.1:8080:8080"
      - "127.0.0.1:8081:8081"
 
   graphql:
     restart: unless-stopped
     depends_on:
-      - db
+      - ipld-eth-db
     image: vulcanize/postgraphile:v1.0.1
     environment:
       - PG_HOST=db

documentation/apis.md

@@ -40,9 +40,9 @@ An example of how to subscribe to a real-time Ethereum data feed from ipld-eth-s
     	"github.com/ethereum/go-ethereum/rpc"
     	"github.com/spf13/viper"
 
-        "github.com/vulcanize/ipld-eth-server/pkg/client"
+        "github.com/vulcanize/ipld-eth-server/v3/pkg/client"
-        "github.com/vulcanize/ipld-eth-server/pkg/eth"
+        "github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-        "github.com/vulcanize/ipld-eth-server/pkg/watch"
+        "github.com/vulcanize/ipld-eth-server/v3/pkg/watch"
     )
 
     config, _ := eth.NewEthSubscriptionConfig()
@@ -160,9 +160,9 @@ An example of how to subscribe to a real-time Bitcoin data feed from ipld-eth-se
     	"github.com/ethereum/go-ethereum/rpc"
     	"github.com/spf13/viper"
 
-    	"github.com/vulcanize/ipld-eth-server/pkg/btc"
+    	"github.com/vulcanize/ipld-eth-server/v3/pkg/btc"
-    	"github.com/vulcanize/ipld-eth-server/pkg/client"
+    	"github.com/vulcanize/ipld-eth-server/v3/pkg/client"
-    	"github.com/vulcanize/ipld-eth-server/pkg/watch"
+    	"github.com/vulcanize/ipld-eth-server/v3/pkg/watch"
     )
 
     config, _ := btc.NewBtcSubscriptionConfig()

entrypoint.sh

@@ -1,20 +1,4 @@
 #!/bin/sh
-# Runs the db migrations and starts the watcher services
-
-# Construct the connection string for postgres
-VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
-
-# Run the DB migrations
-echo "Connecting with: $VDB_PG_CONNECT"
-echo "Running database migrations"
-./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
-rv=$?
-
-if [ $rv != 0 ]; then
-  echo "Could not run migrations. Are the database details correct?"
-  exit 1
-fi
-
 
 echo "Beginning the ipld-eth-server process"
 

environments/example.toml

@@ -12,8 +12,19 @@
     ipcPath = "~/.vulcanize/vulcanize.ipc" # $SERVER_IPC_PATH
     wsPath = "127.0.0.1:8081" # $SERVER_WS_PATH
     httpPath = "127.0.0.1:8082" # $SERVER_HTTP_PATH
+    graphql = true # $SERVER_GRAPHQL
+    graphqlEndpoint = "127.0.0.1:8083" # $SERVER_GRAPHQL_ENDPOINT
 
 [ethereum]
+    chainConfig = "./chain.json" # ETH_CHAIN_CONFIG
     chainID = "1" # $ETH_CHAIN_ID
     defaultSender = "" # $ETH_DEFAULT_SENDER_ADDR
     rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
+    httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF
+    forwardEthCalls = false # $ETH_FORWARD_ETH_CALLS
+    proxyOnError = true # $ETH_PROXY_ON_ERROR
+    nodeID = "arch1" # $ETH_NODE_ID
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
+    networkID = "1" # $ETH_NETWORK_ID

go.mod

@@ -1,25 +1,291 @@
-module github.com/vulcanize/ipld-eth-server
+module github.com/vulcanize/ipld-eth-server/v3
 
-go 1.13
+go 1.18
 
 require (
-	github.com/ethereum/go-ethereum v1.9.11
+	github.com/ethereum/go-ethereum v1.10.18
-	github.com/ipfs/go-block-format v0.0.2
+	github.com/graph-gophers/graphql-go v1.3.0
-	github.com/ipfs/go-cid v0.0.5
+	github.com/ipfs/go-block-format v0.0.3
-	github.com/ipfs/go-ipfs-blockstore v1.0.0
+	github.com/ipfs/go-cid v0.0.7
+	github.com/ipfs/go-ipfs-blockstore v1.0.1
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
 	github.com/ipfs/go-ipld-format v0.2.0
-	github.com/jmoiron/sqlx v1.2.0
+	github.com/jinzhu/now v1.1.5 // indirect
-	github.com/lib/pq v1.5.2
+	github.com/jmoiron/sqlx v1.3.5
-	github.com/multiformats/go-multihash v0.0.13
+	github.com/joho/godotenv v1.4.0
-	github.com/onsi/ginkgo v1.12.1
+	github.com/lib/pq v1.10.5
-	github.com/onsi/gomega v1.10.1
+	github.com/machinebox/graphql v0.2.2
-	github.com/prometheus/client_golang v1.5.1
+	github.com/mailgun/groupcache/v2 v2.3.0
-	github.com/sirupsen/logrus v1.6.0
+	github.com/multiformats/go-multihash v0.1.0
-	github.com/spf13/cobra v1.0.0
+	github.com/onsi/ginkgo v1.16.5
-	github.com/spf13/viper v1.7.0
+	github.com/onsi/gomega v1.19.0
-	github.com/vulcanize/ipld-eth-indexer v0.6.0-alpha
+	github.com/prometheus/client_golang v1.11.0
-	github.com/vulcanize/pg-ipfs-ethdb v0.0.1-alpha
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cobra v1.4.0
+	github.com/spf13/viper v1.11.0
+	github.com/thoas/go-funk v0.9.2 // indirect
+	github.com/vulcanize/eth-ipfs-state-validator/v3 v3.0.2
+	github.com/vulcanize/gap-filler v0.3.1
+	github.com/vulcanize/ipfs-ethdb/v3 v3.0.3
+	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	gorm.io/driver/postgres v1.3.7
+	gorm.io/gorm v1.23.5
 )
 
-replace github.com/ethereum/go-ethereum v1.9.11 => github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.8
+require (
+	bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect
+	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
+	github.com/Stebalien/go-bitfield v0.0.1 // indirect
+	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
+	github.com/benbjohnson/clock v1.1.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/btcsuite/btcd v0.22.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
+	github.com/cskr/pubsub v1.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
+	github.com/deckarep/golang-set v1.8.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/deepmap/oapi-codegen v1.8.2 // indirect
+	github.com/edsrzf/mmap-go v1.0.0 // indirect
+	github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
+	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
+	github.com/flynn/noise v1.0.0 // indirect
+	github.com/francoispqt/gojay v1.2.13 // indirect
+	github.com/friendsofgo/graphiql v0.2.2 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
+	github.com/georgysavva/scany v0.2.9 // indirect
+	github.com/go-ole/go-ole v1.2.1 // indirect
+	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/graphql-go/graphql v0.7.9 // indirect
+	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-bexpr v0.1.10 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
+	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/huin/goupnp v1.0.3 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/influxdata/influxdb v1.8.3 // indirect
+	github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
+	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-bitswap v0.4.0 // indirect
+	github.com/ipfs/go-blockservice v0.1.7 // indirect
+	github.com/ipfs/go-cidutil v0.0.2 // indirect
+	github.com/ipfs/go-datastore v0.4.6 // indirect
+	github.com/ipfs/go-ds-measure v0.1.0 // indirect
+	github.com/ipfs/go-fetcher v1.5.0 // indirect
+	github.com/ipfs/go-filestore v1.0.0 // indirect
+	github.com/ipfs/go-fs-lock v0.0.7 // indirect
+	github.com/ipfs/go-graphsync v0.8.0 // indirect
+	github.com/ipfs/go-ipfs v0.10.0 // indirect
+	github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
+	github.com/ipfs/go-ipfs-config v0.16.0 // indirect
+	github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
+	github.com/ipfs/go-ipfs-exchange-interface v0.0.1 // indirect
+	github.com/ipfs/go-ipfs-exchange-offline v0.0.1 // indirect
+	github.com/ipfs/go-ipfs-files v0.0.8 // indirect
+	github.com/ipfs/go-ipfs-keystore v0.0.2 // indirect
+	github.com/ipfs/go-ipfs-pinner v0.1.2 // indirect
+	github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
+	github.com/ipfs/go-ipfs-pq v0.0.2 // indirect
+	github.com/ipfs/go-ipfs-provider v0.6.1 // indirect
+	github.com/ipfs/go-ipfs-routing v0.1.0 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
+	github.com/ipfs/go-ipld-cbor v0.0.5 // indirect
+	github.com/ipfs/go-ipld-legacy v0.1.0 // indirect
+	github.com/ipfs/go-ipns v0.1.2 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.3.0 // indirect
+	github.com/ipfs/go-merkledag v0.4.0 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-mfs v0.1.2 // indirect
+	github.com/ipfs/go-namesys v0.3.1 // indirect
+	github.com/ipfs/go-path v0.1.2 // indirect
+	github.com/ipfs/go-peertaskqueue v0.4.0 // indirect
+	github.com/ipfs/go-unixfs v0.2.5 // indirect
+	github.com/ipfs/go-unixfsnode v1.1.3 // indirect
+	github.com/ipfs/go-verifcid v0.0.1 // indirect
+	github.com/ipfs/interface-go-ipfs-core v0.5.1 // indirect
+	github.com/ipld/go-codec-dagpb v1.3.0 // indirect
+	github.com/ipld/go-ipld-prime v0.12.2 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.12.1 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgtype v1.11.0 // indirect
+	github.com/jackc/pgx/v4 v4.16.1 // indirect
+	github.com/jackc/puddle v1.2.1 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/jinzhu/copier v0.2.4 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/klauspost/compress v1.11.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/koron/go-ssdp v0.0.2 // indirect
+	github.com/libp2p/go-addr-util v0.1.0 // indirect
+	github.com/libp2p/go-buffer-pool v0.0.2 // indirect
+	github.com/libp2p/go-cidranger v1.1.0 // indirect
+	github.com/libp2p/go-conn-security-multistream v0.2.1 // indirect
+	github.com/libp2p/go-doh-resolver v0.3.1 // indirect
+	github.com/libp2p/go-eventbus v0.2.1 // indirect
+	github.com/libp2p/go-flow-metrics v0.0.3 // indirect
+	github.com/libp2p/go-libp2p v0.15.0 // indirect
+	github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052 // indirect
+	github.com/libp2p/go-libp2p-autonat v0.4.2 // indirect
+	github.com/libp2p/go-libp2p-blankhost v0.2.0 // indirect
+	github.com/libp2p/go-libp2p-circuit v0.4.0 // indirect
+	github.com/libp2p/go-libp2p-connmgr v0.2.4 // indirect
+	github.com/libp2p/go-libp2p-core v0.9.0 // indirect
+	github.com/libp2p/go-libp2p-discovery v0.5.1 // indirect
+	github.com/libp2p/go-libp2p-kad-dht v0.13.1 // indirect
+	github.com/libp2p/go-libp2p-kbucket v0.4.7 // indirect
+	github.com/libp2p/go-libp2p-loggables v0.1.0 // indirect
+	github.com/libp2p/go-libp2p-mplex v0.4.1 // indirect
+	github.com/libp2p/go-libp2p-nat v0.0.6 // indirect
+	github.com/libp2p/go-libp2p-noise v0.2.2 // indirect
+	github.com/libp2p/go-libp2p-peerstore v0.2.8 // indirect
+	github.com/libp2p/go-libp2p-pnet v0.2.0 // indirect
+	github.com/libp2p/go-libp2p-pubsub v0.5.4 // indirect
+	github.com/libp2p/go-libp2p-pubsub-router v0.4.0 // indirect
+	github.com/libp2p/go-libp2p-quic-transport v0.12.0 // indirect
+	github.com/libp2p/go-libp2p-record v0.1.3 // indirect
+	github.com/libp2p/go-libp2p-routing-helpers v0.2.3 // indirect
+	github.com/libp2p/go-libp2p-swarm v0.5.3 // indirect
+	github.com/libp2p/go-libp2p-tls v0.2.0 // indirect
+	github.com/libp2p/go-libp2p-transport-upgrader v0.4.6 // indirect
+	github.com/libp2p/go-libp2p-xor v0.0.0-20210714161855-5c005aca55db // indirect
+	github.com/libp2p/go-libp2p-yamux v0.5.4 // indirect
+	github.com/libp2p/go-maddr-filter v0.1.0 // indirect
+	github.com/libp2p/go-mplex v0.3.0 // indirect
+	github.com/libp2p/go-msgio v0.0.6 // indirect
+	github.com/libp2p/go-nat v0.0.5 // indirect
+	github.com/libp2p/go-netroute v0.1.6 // indirect
+	github.com/libp2p/go-openssl v0.0.7 // indirect
+	github.com/libp2p/go-reuseport v0.0.2 // indirect
+	github.com/libp2p/go-reuseport-transport v0.0.5 // indirect
+	github.com/libp2p/go-sockaddr v0.1.1 // indirect
+	github.com/libp2p/go-stream-muxer-multistream v0.3.0 // indirect
+	github.com/libp2p/go-tcp-transport v0.2.8 // indirect
+	github.com/libp2p/go-ws-transport v0.5.0 // indirect
+	github.com/libp2p/go-yamux/v2 v2.2.0 // indirect
+	github.com/libp2p/zeroconf/v2 v2.0.0 // indirect
+	github.com/lucas-clemente/quic-go v0.26.0 // indirect
+	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
+	github.com/marten-seemann/qtls-go1-17 v0.1.1 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.1 // indirect
+	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
+	github.com/matryer/is v1.4.0 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/miekg/dns v1.1.43 // indirect
+	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
+	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
+	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/mitchellh/pointerstructure v1.2.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.0.3 // indirect
+	github.com/multiformats/go-base36 v0.1.0 // indirect
+	github.com/multiformats/go-multiaddr v0.4.0 // indirect
+	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.0.3 // indirect
+	github.com/multiformats/go-multicodec v0.3.0 // indirect
+	github.com/multiformats/go-multistream v0.2.2 // indirect
+	github.com/multiformats/go-varint v0.0.6 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
+	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
+	github.com/pganalyze/pg_query_go/v2 v2.1.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.30.0 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/tsdb v0.7.1 // indirect
+	github.com/rjeczalik/notify v0.9.1 // indirect
+	github.com/rs/cors v1.7.0 // indirect
+	github.com/segmentio/fasthash v1.0.3 // indirect
+	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
+	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/afero v1.8.2 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.7.1 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
+	github.com/tklauser/go-sysconf v0.3.5 // indirect
+	github.com/tklauser/numcpus v0.2.2 // indirect
+	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef // indirect
+	github.com/valyala/fastjson v1.6.3 // indirect
+	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
+	github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
+	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
+	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
+	github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 // indirect
+	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
+	github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect
+	go.opencensus.io v0.23.0 // indirect
+	go.opentelemetry.io/otel v0.20.0 // indirect
+	go.opentelemetry.io/otel/metric v0.20.0 // indirect
+	go.opentelemetry.io/otel/trace v0.20.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/dig v1.10.0 // indirect
+	go.uber.org/fx v1.13.1 // indirect
+	go.uber.org/multierr v1.7.0 // indirect
+	go.uber.org/zap v1.19.0 // indirect
+	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
+	golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
+	golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 // indirect
+	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/ini.v1 v1.66.4 // indirect
+	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/urfave/cli.v1 v1.20.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	lukechampine.com/blake3 v1.1.6 // indirect
+)
+
+replace github.com/ethereum/go-ethereum v1.10.18 => github.com/vulcanize/go-ethereum v1.10.18-statediff-3.2.2

main.go

@@ -18,7 +18,7 @@ package main
 import (
 	"github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/ipld-eth-server/cmd"
+	"github.com/vulcanize/ipld-eth-server/v3/cmd"
 )
 
 func main() {

pkg/client/client.go

@@ -20,11 +20,10 @@ package client
 import (
 	"context"
 
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-
 	"github.com/ethereum/go-ethereum/rpc"
 
-	"github.com/vulcanize/ipld-eth-server/pkg/serve"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/serve"
 )
 
 // Client is used to subscribe to the ipld-eth-server ipld data stream

pkg/eth/backend.go

@@ -18,46 +18,83 @@ package eth
 
 import (
 	"context"
+	"database/sql"
 	"errors"
+	"fmt"
 	"math/big"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/bloombits"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	ipfsethdb "github.com/vulcanize/pg-ipfs-ethdb"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/jmoiron/sqlx"
+	log "github.com/sirupsen/logrus"
+	validator "github.com/vulcanize/eth-ipfs-state-validator/v3/pkg"
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb/v3/postgres"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	ethServerShared "github.com/ethereum/go-ethereum/statediff/indexer/shared"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
+
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
 )
 
 var (
 	errPendingBlockNumber  = errors.New("pending block number not supported")
 	errNegativeBlockNumber = errors.New("negative block number not supported")
+	errHeaderHashNotFound  = errors.New("header for hash not found")
+	errHeaderNotFound      = errors.New("header not found")
+
+	// errMissingSignature is returned if a block's extra-data section doesn't seem
+	// to contain a 65 byte secp256k1 signature.
+	errMissingSignature = errors.New("extra-data 65 byte signature suffix missing")
 )
 
 const (
 	RetrieveCanonicalBlockHashByNumber = `SELECT block_hash FROM eth.header_cids
 									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
-									WHERE id = (SELECT canonical_header($1))`
+									WHERE block_hash = (SELECT canonical_header_hash($1))`
 	RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids
 									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
-									WHERE id = (SELECT canonical_header($1))`
+									WHERE block_hash = (SELECT canonical_header_hash($1))`
+	RetrieveTD = `SELECT CAST(td as Text) FROM eth.header_cids
+			WHERE header_cids.block_hash = $1`
+	RetrieveRPCTransaction = `SELECT blocks.data, block_hash, block_number, index FROM public.blocks, eth.transaction_cids, eth.header_cids
+			WHERE blocks.key = transaction_cids.mh_key
+			AND transaction_cids.header_id = header_cids.block_hash
+			AND transaction_cids.tx_hash = $1`
+	RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids
+											WHERE state_accounts.header_id = state_cids.header_id AND state_accounts.state_path = state_cids.state_path
+											AND state_cids.header_id = header_cids.block_hash
+											AND state_leaf_key = $1
+											AND block_number <= (SELECT block_number
+																FROM eth.header_cids
+																WHERE block_hash = $2)
+											AND header_cids.block_hash = (SELECT canonical_header_hash(block_number))
+											ORDER BY block_number DESC
+											LIMIT 1`
+	RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1`
+)
+
+const (
+	StateDBGroupCacheName = "statedb"
 )
 
 type Backend struct {
 	// underlying postgres db
-	DB *postgres.DB
+	DB *sqlx.DB
 
 	// postgres db interfaces
 	Retriever     *CIDRetriever
@@ -73,14 +110,29 @@ type Backend struct {
 
 type Config struct {
 	ChainConfig      *params.ChainConfig
-	VmConfig      vm.Config
+	VMConfig         vm.Config
 	DefaultSender    *common.Address
 	RPCGasCap        *big.Int
+	GroupCacheConfig *shared.GroupCacheConfig
+}
+
+func NewEthBackend(db *sqlx.DB, c *Config) (*Backend, error) {
+	gcc := c.GroupCacheConfig
+
+	groupName := gcc.StateDB.Name
+	if groupName == "" {
+		groupName = StateDBGroupCacheName
 	}
 
-func NewEthBackend(db *postgres.DB, c *Config) (*Backend, error) {
 	r := NewCIDRetriever(db)
-	ethDB := ipfsethdb.NewDatabase(db.DB)
+	ethDB := ipfsethdb.NewDatabase(db, ipfsethdb.CacheConfig{
+		Name:           groupName,
+		Size:           gcc.StateDB.CacheSizeInMB * 1024 * 1024,
+		ExpiryDuration: time.Minute * time.Duration(gcc.StateDB.CacheExpiryInMins),
+	})
+
+	logStateDBStatsOnTimer(ethDB.(*ipfsethdb.Database), gcc)
+
 	return &Backend{
 		DB:            db,
 		Retriever:     r,
@@ -92,6 +144,11 @@ func NewEthBackend(db *postgres.DB, c *Config) (*Backend, error) {
 	}, nil
 }
 
+// ChainDb returns the backend's underlying chain database
+func (b *Backend) ChainDb() ethdb.Database {
+	return b.EthDB
+}
+
 // HeaderByNumber gets the canonical header for the provided block number
 func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumber) (*types.Header, error) {
 	var err error
@@ -133,12 +190,35 @@ func (b *Backend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.He
 	return header, rlp.DecodeBytes(headerRLP, header)
 }
 
+// HeaderByNumberOrHash gets the header for the provided block hash or number
+func (b *Backend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.HeaderByNumber(ctx, blockNr)
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		header, err := b.HeaderByHash(ctx, hash)
+		if err != nil {
+			return nil, err
+		}
+		if header == nil {
+			return nil, errors.New("header for hash not found")
+		}
+		canonicalHash, err := b.GetCanonicalHash(header.Number.Uint64())
+		if err != nil {
+			return nil, err
+		}
+		if blockNrOrHash.RequireCanonical && canonicalHash != hash {
+			return nil, errors.New("hash is not currently canonical")
+		}
+		return header, nil
+	}
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
+}
+
 // GetTd gets the total difficulty at the given block hash
 func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) {
-	pgStr := `SELECT td FROM eth.header_cids
-			WHERE header_cids.block_hash = $1`
 	var tdStr string
-	err := b.DB.Get(&tdStr, pgStr, blockHash.String())
+	err := b.DB.Get(&tdStr, RetrieveTD, blockHash.String())
 	if err != nil {
 		return nil, err
 	}
@@ -149,21 +229,42 @@ func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) {
 	return td, nil
 }
 
-// GetLogs returns all the logs for the given block hash
+// CurrentBlock returns the current block
-func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
+func (b *Backend) CurrentBlock() (*types.Block, error) {
-	_, receiptBytes, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(hash)
+	block, err := b.BlockByNumber(context.Background(), rpc.LatestBlockNumber)
+	return block, err
+}
+
+// BlockByNumberOrHash returns block by number or hash
+func (b *Backend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.BlockByNumber(ctx, blockNr)
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		header, err := b.HeaderByHash(ctx, hash)
 		if err != nil {
 			return nil, err
 		}
-	logs := make([][]*types.Log, len(receiptBytes))
+		if header == nil {
-	for i, rctBytes := range receiptBytes {
+			return nil, errors.New("header for hash not found")
-		var rct types.Receipt
+		}
-		if err := rlp.DecodeBytes(rctBytes, &rct); err != nil {
+		canonicalHash, err := b.GetCanonicalHash(header.Number.Uint64())
+		if err != nil {
 			return nil, err
 		}
-		logs[i] = rct.Logs
+		if blockNrOrHash.RequireCanonical && canonicalHash != hash {
+			return nil, errors.New("hash is not currently canonical")
 		}
-	return logs, nil
+		block, err := b.BlockByHash(ctx, hash)
+		if err != nil {
+			return nil, err
+		}
+		if block == nil {
+			return nil, errors.New("header found, but block body is missing")
+		}
+		return block, nil
+	}
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
 }
 
 // BlockByNumber returns the requested canonical block.
@@ -189,14 +290,20 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 		return nil, errNegativeBlockNumber
 	}
 	// Get the canonical hash
-	canonicalHash := b.GetCanonicalHash(uint64(number))
+	canonicalHash, err := b.GetCanonicalHash(uint64(number))
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
 		return nil, err
 	}
 	// Retrieve all the CIDs for the block
 	// TODO: optimize this by retrieving iplds directly rather than the cids first (this is remanent from when we fetched iplds through ipfs blockservice interface)
 	headerCID, uncleCIDs, txCIDs, rctCIDs, err := b.Retriever.RetrieveBlockByHash(canonicalHash)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
 		return nil, err
 	}
 
@@ -217,55 +324,71 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 	}()
 
 	// Fetch and decode the header IPLD
-	headerIPLD, err := b.Fetcher.FetchHeader(tx, headerCID)
+	var headerIPLD models.IPLDModel
+	headerIPLD, err = b.Fetcher.FetchHeader(tx, headerCID)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
 		return nil, err
 	}
 	var header types.Header
-	if err := rlp.DecodeBytes(headerIPLD.Data, &header); err != nil {
+	err = rlp.DecodeBytes(headerIPLD.Data, &header)
+	if err != nil {
 		return nil, err
 	}
 	// Fetch and decode the uncle IPLDs
-	uncleIPLDs, err := b.Fetcher.FetchUncles(tx, uncleCIDs)
+	var uncleIPLDs []models.IPLDModel
+	uncleIPLDs, err = b.Fetcher.FetchUncles(tx, uncleCIDs)
 	if err != nil {
 		return nil, err
 	}
 	var uncles []*types.Header
 	for _, uncleIPLD := range uncleIPLDs {
 		var uncle types.Header
-		if err := rlp.DecodeBytes(uncleIPLD.Data, &uncle); err != nil {
+		err = rlp.DecodeBytes(uncleIPLD.Data, &uncle)
+		if err != nil {
 			return nil, err
 		}
 		uncles = append(uncles, &uncle)
 	}
 	// Fetch and decode the transaction IPLDs
-	txIPLDs, err := b.Fetcher.FetchTrxs(tx, txCIDs)
+	var txIPLDs []models.IPLDModel
+	txIPLDs, err = b.Fetcher.FetchTrxs(tx, txCIDs)
 	if err != nil {
 		return nil, err
 	}
 	var transactions []*types.Transaction
 	for _, txIPLD := range txIPLDs {
 		var transaction types.Transaction
-		if err := rlp.DecodeBytes(txIPLD.Data, &transaction); err != nil {
+		err = transaction.UnmarshalBinary(txIPLD.Data)
+		if err != nil {
 			return nil, err
 		}
 		transactions = append(transactions, &transaction)
 	}
 	// Fetch and decode the receipt IPLDs
-	rctIPLDs, err := b.Fetcher.FetchRcts(tx, rctCIDs)
+	var rctIPLDs []models.IPLDModel
+	rctIPLDs, err = b.Fetcher.FetchRcts(tx, rctCIDs)
 	if err != nil {
 		return nil, err
 	}
 	var receipts []*types.Receipt
 	for _, rctIPLD := range rctIPLDs {
 		var receipt types.Receipt
-		if err := rlp.DecodeBytes(rctIPLD.Data, &receipt); err != nil {
+		nodeVal, err := DecodeLeafNode(rctIPLD.Data)
+		if err != nil {
+			return nil, err
+		}
+
+		err = receipt.UnmarshalBinary(nodeVal)
+		if err != nil {
 			return nil, err
 		}
 		receipts = append(receipts, &receipt)
 	}
 	// Compose everything together into a complete block
-	return types.NewBlock(&header, transactions, uncles, receipts), err
+	return types.NewBlock(&header, transactions, uncles, receipts, new(trie.Trie)), err
 }
 
 // BlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full
@@ -274,6 +397,9 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 	// Retrieve all the CIDs for the block
 	headerCID, uncleCIDs, txCIDs, rctCIDs, err := b.Retriever.RetrieveBlockByHash(hash)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
 		return nil, err
 	}
 
@@ -294,55 +420,80 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 	}()
 
 	// Fetch and decode the header IPLD
-	headerIPLD, err := b.Fetcher.FetchHeader(tx, headerCID)
+	var headerIPLD models.IPLDModel
+	headerIPLD, err = b.Fetcher.FetchHeader(tx, headerCID)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
 		return nil, err
 	}
 	var header types.Header
-	if err := rlp.DecodeBytes(headerIPLD.Data, &header); err != nil {
+	err = rlp.DecodeBytes(headerIPLD.Data, &header)
+	if err != nil {
 		return nil, err
 	}
 	// Fetch and decode the uncle IPLDs
-	uncleIPLDs, err := b.Fetcher.FetchUncles(tx, uncleCIDs)
+	var uncleIPLDs []models.IPLDModel
+	uncleIPLDs, err = b.Fetcher.FetchUncles(tx, uncleCIDs)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
 		return nil, err
 	}
 	var uncles []*types.Header
 	for _, uncleIPLD := range uncleIPLDs {
 		var uncle types.Header
-		if err := rlp.DecodeBytes(uncleIPLD.Data, &uncle); err != nil {
+		err = rlp.DecodeBytes(uncleIPLD.Data, &uncle)
+		if err != nil {
 			return nil, err
 		}
 		uncles = append(uncles, &uncle)
 	}
 	// Fetch and decode the transaction IPLDs
-	txIPLDs, err := b.Fetcher.FetchTrxs(tx, txCIDs)
+	var txIPLDs []models.IPLDModel
+	txIPLDs, err = b.Fetcher.FetchTrxs(tx, txCIDs)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
 		return nil, err
 	}
 	var transactions []*types.Transaction
 	for _, txIPLD := range txIPLDs {
 		var transaction types.Transaction
-		if err := rlp.DecodeBytes(txIPLD.Data, &transaction); err != nil {
+		err = transaction.UnmarshalBinary(txIPLD.Data)
+		if err != nil {
 			return nil, err
 		}
 		transactions = append(transactions, &transaction)
 	}
 	// Fetch and decode the receipt IPLDs
-	rctIPLDs, err := b.Fetcher.FetchRcts(tx, rctCIDs)
+	var rctIPLDs []models.IPLDModel
+	rctIPLDs, err = b.Fetcher.FetchRcts(tx, rctCIDs)
 	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
 		return nil, err
 	}
 	var receipts []*types.Receipt
 	for _, rctIPLD := range rctIPLDs {
 		var receipt types.Receipt
-		if err := rlp.DecodeBytes(rctIPLD.Data, &receipt); err != nil {
+		nodeVal, err := DecodeLeafNode(rctIPLD.Data)
+		if err != nil {
+			return nil, err
+		}
+
+		err = receipt.UnmarshalBinary(nodeVal)
+		if err != nil {
 			return nil, err
 		}
 		receipts = append(receipts, &receipt)
 	}
 	// Compose everything together into a complete block
-	return types.NewBlock(&header, transactions, uncles, receipts), err
+	return types.NewBlock(&header, transactions, uncles, receipts, new(trie.Trie)), err
 }
 
 // GetTransaction retrieves a tx by hash
@@ -354,223 +505,56 @@ func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*type
 		BlockNumber uint64 `db:"block_number"`
 		Index       uint64 `db:"index"`
 	}
-	pgStr := `SELECT blocks.data, block_hash, block_number, index FROM public.blocks, eth.transaction_cids, eth.header_cids
+	if err := b.DB.Get(&tempTxStruct, RetrieveRPCTransaction, txHash.String()); err != nil {
-			WHERE blocks.key = transaction_cids.mh_key
-			AND transaction_cids.header_id = header_cids.id
-			AND transaction_cids.tx_hash = $1`
-	if err := b.DB.Get(&tempTxStruct, pgStr, txHash.String()); err != nil {
 		return nil, common.Hash{}, 0, 0, err
 	}
 	var transaction types.Transaction
-	if err := rlp.DecodeBytes(tempTxStruct.Data, &transaction); err != nil {
+	if err := transaction.UnmarshalBinary(tempTxStruct.Data); err != nil {
 		return nil, common.Hash{}, 0, 0, err
 	}
 	return &transaction, common.HexToHash(tempTxStruct.BlockHash), tempTxStruct.BlockNumber, tempTxStruct.Index, nil
 }
 
-// extractLogsOfInterest returns logs from the receipt IPLD
+// GetReceipts retrieves receipts for provided block hash
-func extractLogsOfInterest(rctIPLDs []ipfs.BlockModel, wantedTopics [][]string) ([]*types.Log, error) {
+func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {
-	var logs []*types.Log
+	_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(hash)
-	for _, rctIPLD := range rctIPLDs {
+	if err != nil {
-		rctRLP := rctIPLD
-		var rct types.Receipt
-		if err := rlp.DecodeBytes(rctRLP.Data, &rct); err != nil {
 		return nil, err
 	}
+	rcts := make(types.Receipts, len(receiptBytes))
+	for i, rctBytes := range receiptBytes {
+		rct := new(types.Receipt)
+		if err := rct.UnmarshalBinary(rctBytes); err != nil {
+			return nil, err
+		}
+		rct.TxHash = txs[i]
+		rcts[i] = rct
+	}
+	return rcts, nil
+}
+
+// GetLogs returns all the logs for the given block hash
+func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
+	_, receiptBytes, txs, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(hash)
+	if err != nil {
+		return nil, err
+	}
+	logs := make([][]*types.Log, len(receiptBytes))
+	for i, rctBytes := range receiptBytes {
+		var rct types.Receipt
+		if err := rlp.DecodeBytes(rctBytes, &rct); err != nil {
+			return nil, err
+		}
+
 		for _, log := range rct.Logs {
-			if wanted := wantedLog(wantedTopics, log.Topics); wanted == true {
+			log.TxHash = txs[i]
-				logs = append(logs, log)
-			}
 		}
+
+		logs[i] = rct.Logs
 	}
 	return logs, nil
 }
 
-// returns true if the log matches on the filter
-func wantedLog(wantedTopics [][]string, actualTopics []common.Hash) bool {
-	// actualTopics will always have length <= 4
-	// wantedTopics will always have length 4
-	matches := 0
-	for i, actualTopic := range actualTopics {
-		// If we have topics in this filter slot, count as a match if the actualTopic matches one of the ones in this filter slot
-		if len(wantedTopics[i]) > 0 {
-			matches += sliceContainsHash(wantedTopics[i], actualTopic)
-		} else {
-			// Filter slot is empty, not matching any topics at this slot => counts as a match
-			matches++
-		}
-	}
-	if matches == len(actualTopics) {
-		return true
-	}
-	return false
-}
-
-// returns 1 if the slice contains the hash, 0 if it does not
-func sliceContainsHash(slice []string, hash common.Hash) int {
-	for _, str := range slice {
-		if str == hash.String() {
-			return 1
-		}
-	}
-	return 0
-}
-
-// rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires
-// a `PublicEthAPI`.
-func (pea *PublicEthAPI) rpcMarshalHeader(header *types.Header) (map[string]interface{}, error) {
-	fields := RPCMarshalHeader(header)
-	td, err := pea.B.GetTd(header.Hash())
-	if err != nil {
-		return nil, err
-	}
-	fields["totalDifficulty"] = (*hexutil.Big)(td)
-	return fields, nil
-}
-
-// RPCMarshalHeader converts the given header to the RPC output.
-// This function is eth/internal so we have to make our own version here...
-func RPCMarshalHeader(head *types.Header) map[string]interface{} {
-	return map[string]interface{}{
-		"number":           (*hexutil.Big)(head.Number),
-		"hash":             head.Hash(),
-		"parentHash":       head.ParentHash,
-		"nonce":            head.Nonce,
-		"mixHash":          head.MixDigest,
-		"sha3Uncles":       head.UncleHash,
-		"logsBloom":        head.Bloom,
-		"stateRoot":        head.Root,
-		"miner":            head.Coinbase,
-		"difficulty":       (*hexutil.Big)(head.Difficulty),
-		"extraData":        hexutil.Bytes(head.Extra),
-		"size":             hexutil.Uint64(head.Size()),
-		"gasLimit":         hexutil.Uint64(head.GasLimit),
-		"gasUsed":          hexutil.Uint64(head.GasUsed),
-		"timestamp":        hexutil.Uint64(head.Time),
-		"transactionsRoot": head.TxHash,
-		"receiptsRoot":     head.ReceiptHash,
-	}
-}
-
-// rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires
-// a `PublicBlockchainAPI`.
-func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
-	fields, err := RPCMarshalBlock(b, inclTx, fullTx)
-	if err != nil {
-		return nil, err
-	}
-	td, err := pea.B.GetTd(b.Hash())
-	if err != nil {
-		return nil, err
-	}
-	fields["totalDifficulty"] = (*hexutil.Big)(td)
-	return fields, err
-}
-
-// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are
-// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain
-// transaction hashes.
-func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
-	fields := RPCMarshalHeader(block.Header())
-	fields["size"] = hexutil.Uint64(block.Size())
-
-	if inclTx {
-		formatTx := func(tx *types.Transaction) (interface{}, error) {
-			return tx.Hash(), nil
-		}
-		if fullTx {
-			formatTx = func(tx *types.Transaction) (interface{}, error) {
-				return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil
-			}
-		}
-		txs := block.Transactions()
-		transactions := make([]interface{}, len(txs))
-		var err error
-		for i, tx := range txs {
-			if transactions[i], err = formatTx(tx); err != nil {
-				return nil, err
-			}
-		}
-		fields["transactions"] = transactions
-	}
-	uncles := block.Uncles()
-	uncleHashes := make([]common.Hash, len(uncles))
-	for i, uncle := range uncles {
-		uncleHashes[i] = uncle.Hash()
-	}
-	fields["uncles"] = uncleHashes
-
-	return fields, nil
-}
-
-// NewRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
-func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction {
-	for idx, tx := range b.Transactions() {
-		if tx.Hash() == hash {
-			return newRPCTransactionFromBlockIndex(b, uint64(idx))
-		}
-	}
-	return nil
-}
-
-// newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation.
-func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransaction {
-	txs := b.Transactions()
-	if index >= uint64(len(txs)) {
-		return nil
-	}
-	return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index)
-}
-
-// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
-type RPCTransaction struct {
-	BlockHash        *common.Hash    `json:"blockHash"`
-	BlockNumber      *hexutil.Big    `json:"blockNumber"`
-	From             common.Address  `json:"from"`
-	Gas              hexutil.Uint64  `json:"gas"`
-	GasPrice         *hexutil.Big    `json:"gasPrice"`
-	Hash             common.Hash     `json:"hash"`
-	Input            hexutil.Bytes   `json:"input"`
-	Nonce            hexutil.Uint64  `json:"nonce"`
-	To               *common.Address `json:"to"`
-	TransactionIndex *hexutil.Uint64 `json:"transactionIndex"`
-	Value            *hexutil.Big    `json:"value"`
-	V                *hexutil.Big    `json:"v"`
-	R                *hexutil.Big    `json:"r"`
-	S                *hexutil.Big    `json:"s"`
-}
-
-// NewRPCTransaction returns a transaction that will serialize to the RPC
-// representation, with the given location metadata set (if available).
-func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
-	var signer types.Signer = types.FrontierSigner{}
-	if tx.Protected() {
-		signer = types.NewEIP155Signer(tx.ChainId())
-	}
-	from, _ := types.Sender(signer, tx)
-	v, r, s := tx.RawSignatureValues()
-
-	result := &RPCTransaction{
-		From:     from,
-		Gas:      hexutil.Uint64(tx.Gas()),
-		GasPrice: (*hexutil.Big)(tx.GasPrice()),
-		Hash:     tx.Hash(),
-		Input:    hexutil.Bytes(tx.Data()), // somehow this is ending up `nil`
-		Nonce:    hexutil.Uint64(tx.Nonce()),
-		To:       tx.To(),
-		Value:    (*hexutil.Big)(tx.Value()),
-		V:        (*hexutil.Big)(v),
-		R:        (*hexutil.Big)(r),
-		S:        (*hexutil.Big)(s),
-	}
-	if blockHash != (common.Hash{}) {
-		result.BlockHash = &blockHash
-		result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber))
-		result.TransactionIndex = (*hexutil.Uint64)(&index)
-	}
-	return result
-}
-
 // StateAndHeaderByNumberOrHash returns the statedb and header for the provided block number or hash
 func (b *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) {
 	if blockNr, ok := blockNrOrHash.Number(); ok {
@@ -584,10 +568,14 @@ func (b *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHas
 		if header == nil {
 			return nil, nil, errors.New("header for hash not found")
 		}
-		if blockNrOrHash.RequireCanonical && b.GetCanonicalHash(header.Number.Uint64()) != hash {
+		canonicalHash, err := b.GetCanonicalHash(header.Number.Uint64())
+		if err != nil {
+			return nil, nil, err
+		}
+		if blockNrOrHash.RequireCanonical && canonicalHash != hash {
 			return nil, nil, errors.New("hash is not currently canonical")
 		}
-		stateDb, err := state.New(header.Root, b.StateDatabase)
+		stateDb, err := state.New(header.Root, b.StateDatabase, nil)
 		return stateDb, header, err
 	}
 	return nil, nil, errors.New("invalid arguments; neither block nor hash specified")
@@ -607,17 +595,17 @@ func (b *Backend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNu
 	if header == nil {
 		return nil, nil, errors.New("header not found")
 	}
-	stateDb, err := state.New(header.Root, b.StateDatabase)
+	stateDb, err := state.New(header.Root, b.StateDatabase, nil)
 	return stateDb, header, err
 }
 
 // GetCanonicalHash gets the canonical hash for the provided number, if there is one
-func (b *Backend) GetCanonicalHash(number uint64) common.Hash {
+func (b *Backend) GetCanonicalHash(number uint64) (common.Hash, error) {
 	var hashResult string
 	if err := b.DB.Get(&hashResult, RetrieveCanonicalBlockHashByNumber, number); err != nil {
-		return common.Hash{}
+		return common.Hash{}, err
 	}
-	return common.HexToHash(hashResult)
+	return common.HexToHash(hashResult), nil
 }
 
 type rowResult struct {
@@ -632,10 +620,195 @@ func (b *Backend) GetCanonicalHeader(number uint64) (string, []byte, error) {
 }
 
 // GetEVM constructs and returns a vm.EVM
-func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, error) {
+func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) {
-	state.SetBalance(msg.From(), math.MaxBig256)
+	vmError := func() error { return nil }
-	c := core.NewEVMContext(msg, header, b, nil)
+	txContext := core.NewEVMTxContext(msg)
-	return vm.NewEVM(c, state, b.Config.ChainConfig, b.Config.VmConfig), nil
+	context := core.NewEVMBlockContext(header, b, nil)
+	return vm.NewEVM(context, txContext, state, b.Config.ChainConfig, b.Config.VMConfig), vmError, nil
+}
+
+// GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash
+func (b *Backend) GetAccountByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*types.StateAccount, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.GetAccountByNumber(ctx, address, blockNr)
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		return b.GetAccountByHash(ctx, address, hash)
+	}
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
+}
+
+// GetAccountByNumber returns the account object for the provided address at the canonical block at the provided height
+func (b *Backend) GetAccountByNumber(ctx context.Context, address common.Address, blockNumber rpc.BlockNumber) (*types.StateAccount, error) {
+	var err error
+	number := blockNumber.Int64()
+	if blockNumber == rpc.LatestBlockNumber {
+		number, err = b.Retriever.RetrieveLastBlockNumber()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if blockNumber == rpc.EarliestBlockNumber {
+		number, err = b.Retriever.RetrieveFirstBlockNumber()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if blockNumber == rpc.PendingBlockNumber {
+		return nil, errPendingBlockNumber
+	}
+	hash, err := b.GetCanonicalHash(uint64(number))
+	if err == sql.ErrNoRows {
+		return nil, errHeaderNotFound
+	} else if err != nil {
+		return nil, err
+	}
+
+	return b.GetAccountByHash(ctx, address, hash)
+}
+
+// GetAccountByHash returns the account object for the provided address at the block with the provided hash
+func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address, hash common.Hash) (*types.StateAccount, error) {
+	_, err := b.HeaderByHash(context.Background(), hash)
+	if err == sql.ErrNoRows {
+		return nil, errHeaderHashNotFound
+	} else if err != nil {
+		return nil, err
+	}
+
+	_, accountRlp, err := b.IPLDRetriever.RetrieveAccountByAddressAndBlockHash(address, hash)
+	if err != nil {
+		return nil, err
+	}
+
+	acct := new(types.StateAccount)
+	return acct, rlp.DecodeBytes(accountRlp, acct)
+}
+
+// GetCodeByNumberOrHash returns the byte code for the contract deployed at the provided address at the block with the provided hash or block number
+func (b *Backend) GetCodeByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) ([]byte, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.GetCodeByNumber(ctx, address, blockNr)
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		return b.GetCodeByHash(ctx, address, hash)
+	}
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
+}
+
+// GetCodeByNumber returns the byte code for the contract deployed at the provided address at the canonical block with the provided block number
+func (b *Backend) GetCodeByNumber(ctx context.Context, address common.Address, blockNumber rpc.BlockNumber) ([]byte, error) {
+	var err error
+	number := blockNumber.Int64()
+	if blockNumber == rpc.LatestBlockNumber {
+		number, err = b.Retriever.RetrieveLastBlockNumber()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if blockNumber == rpc.EarliestBlockNumber {
+		number, err = b.Retriever.RetrieveFirstBlockNumber()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if blockNumber == rpc.PendingBlockNumber {
+		return nil, errPendingBlockNumber
+	}
+	hash, err := b.GetCanonicalHash(uint64(number))
+	if err != nil {
+		return nil, err
+	}
+	if hash == (common.Hash{}) {
+		return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number)
+	}
+	return b.GetCodeByHash(ctx, address, hash)
+}
+
+// GetCodeByHash returns the byte code for the contract deployed at the provided address at the block with the provided hash
+func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, hash common.Hash) ([]byte, error) {
+	codeHash := make([]byte, 0)
+	leafKey := crypto.Keccak256Hash(address.Bytes())
+	// Begin tx
+	tx, err := b.DB.Beginx()
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			shared.Rollback(tx)
+			panic(p)
+		} else if err != nil {
+			shared.Rollback(tx)
+		} else {
+			err = tx.Commit()
+		}
+	}()
+	err = tx.Get(&codeHash, RetrieveCodeHashByLeafKeyAndBlockHash, leafKey.Hex(), hash.Hex())
+	if err != nil {
+		return nil, err
+	}
+	var mhKey string
+	mhKey, err = ethServerShared.MultihashKeyFromKeccak256(common.BytesToHash(codeHash))
+	if err != nil {
+		return nil, err
+	}
+	code := make([]byte, 0)
+	err = tx.Get(&code, RetrieveCodeByMhKey, mhKey)
+	return code, err
+}
+
+// GetStorageByNumberOrHash returns the storage value for the provided contract address an storage key at the block corresponding to the provided number or hash
+func (b *Backend) GetStorageByNumberOrHash(ctx context.Context, address common.Address, key common.Hash, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.GetStorageByNumber(ctx, address, key, blockNr)
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		return b.GetStorageByHash(ctx, address, key, hash)
+	}
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
+}
+
+// GetStorageByNumber returns the storage value for the provided contract address an storage key at the block corresponding to the provided number
+func (b *Backend) GetStorageByNumber(ctx context.Context, address common.Address, key common.Hash, blockNumber rpc.BlockNumber) (hexutil.Bytes, error) {
+	var err error
+	number := blockNumber.Int64()
+	if blockNumber == rpc.LatestBlockNumber {
+		number, err = b.Retriever.RetrieveLastBlockNumber()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if blockNumber == rpc.EarliestBlockNumber {
+		number, err = b.Retriever.RetrieveFirstBlockNumber()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if blockNumber == rpc.PendingBlockNumber {
+		return nil, errPendingBlockNumber
+	}
+	hash, err := b.GetCanonicalHash(uint64(number))
+	if err == sql.ErrNoRows {
+		return nil, errHeaderNotFound
+	} else if err != nil {
+		return nil, err
+	}
+
+	return b.GetStorageByHash(ctx, address, key, hash)
+}
+
+// GetStorageByHash returns the storage value for the provided contract address an storage key at the block corresponding to the provided hash
+func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address, key, hash common.Hash) (hexutil.Bytes, error) {
+	_, err := b.HeaderByHash(context.Background(), hash)
+	if err == sql.ErrNoRows {
+		return nil, errHeaderHashNotFound
+	} else if err != nil {
+		return nil, err
+	}
+
+	_, _, storageRlp, err := b.IPLDRetriever.RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address, key, hash)
+	return storageRlp, err
 }
 
 // Engine satisfied the ChainContext interface
@@ -652,3 +825,56 @@ func (b *Backend) GetHeader(hash common.Hash, height uint64) *types.Header {
 	}
 	return header
 }
+
+// ValidateTrie validates the trie for the given stateRoot
+func (b *Backend) ValidateTrie(stateRoot common.Hash) error {
+	return validator.NewValidator(nil, b.EthDB).ValidateTrie(stateRoot)
+}
+
+// RPCGasCap returns the configured gas cap for the rpc server
+func (b *Backend) RPCGasCap() *big.Int {
+	return b.Config.RPCGasCap
+}
+
+func (b *Backend) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) BloomStatus() (uint64, uint64) {
+	panic("implement me")
+}
+
+func (b *Backend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {
+	panic("implement me")
+}
+
+func logStateDBStatsOnTimer(ethDB *ipfsethdb.Database, gcc *shared.GroupCacheConfig) {
+	// No stats logging if interval isn't a positive integer.
+	if gcc.StateDB.LogStatsIntervalInSecs <= 0 {
+		return
+	}
+
+	ticker := time.NewTicker(time.Duration(gcc.StateDB.LogStatsIntervalInSecs) * time.Second)
+
+	go func() {
+		for range ticker.C {
+			log.Infof("%s groupcache stats: %+v", StateDBGroupCacheName, ethDB.GetCacheStats())
+		}
+	}()
+}

pkg/eth/backend_utils.go

@@ -0,0 +1,297 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// RPCMarshalHeader converts the given header to the RPC output.
+// This function is eth/internal so we have to make our own version here...
+func RPCMarshalHeader(head *types.Header) map[string]interface{} {
+	headerMap := map[string]interface{}{
+		"number":           (*hexutil.Big)(head.Number),
+		"hash":             head.Hash(),
+		"parentHash":       head.ParentHash,
+		"nonce":            head.Nonce,
+		"mixHash":          head.MixDigest,
+		"sha3Uncles":       head.UncleHash,
+		"logsBloom":        head.Bloom,
+		"stateRoot":        head.Root,
+		"miner":            head.Coinbase,
+		"difficulty":       (*hexutil.Big)(head.Difficulty),
+		"extraData":        hexutil.Bytes(head.Extra),
+		"size":             hexutil.Uint64(head.Size()),
+		"gasLimit":         hexutil.Uint64(head.GasLimit),
+		"gasUsed":          hexutil.Uint64(head.GasUsed),
+		"timestamp":        hexutil.Uint64(head.Time),
+		"transactionsRoot": head.TxHash,
+		"receiptsRoot":     head.ReceiptHash,
+	}
+
+	if head.BaseFee != nil {
+		headerMap["baseFee"] = head.BaseFee
+	}
+	return headerMap
+}
+
+// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are
+// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain
+// transaction hashes.
+func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
+	fields := RPCMarshalHeader(block.Header())
+	fields["size"] = hexutil.Uint64(block.Size())
+
+	if inclTx {
+		formatTx := func(tx *types.Transaction) (interface{}, error) {
+			return tx.Hash(), nil
+		}
+		if fullTx {
+			formatTx = func(tx *types.Transaction) (interface{}, error) {
+				return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil
+			}
+		}
+		txs := block.Transactions()
+		transactions := make([]interface{}, len(txs))
+		var err error
+		for i, tx := range txs {
+			if transactions[i], err = formatTx(tx); err != nil {
+				return nil, err
+			}
+		}
+		fields["transactions"] = transactions
+	}
+	uncles := block.Uncles()
+	uncleHashes := make([]common.Hash, len(uncles))
+	for i, uncle := range uncles {
+		uncleHashes[i] = uncle.Hash()
+	}
+	fields["uncles"] = uncleHashes
+
+	return fields, nil
+}
+
+// RPCMarshalBlockWithUncleHashes marshals the block with the provided uncle hashes
+func RPCMarshalBlockWithUncleHashes(block *types.Block, uncleHashes []common.Hash, inclTx bool, fullTx bool) (map[string]interface{}, error) {
+	fields := RPCMarshalHeader(block.Header())
+	fields["size"] = hexutil.Uint64(block.Size())
+
+	if inclTx {
+		formatTx := func(tx *types.Transaction) (interface{}, error) {
+			return tx.Hash(), nil
+		}
+		if fullTx {
+			formatTx = func(tx *types.Transaction) (interface{}, error) {
+				return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil
+			}
+		}
+		txs := block.Transactions()
+		transactions := make([]interface{}, len(txs))
+		var err error
+		for i, tx := range txs {
+			if transactions[i], err = formatTx(tx); err != nil {
+				return nil, err
+			}
+		}
+		fields["transactions"] = transactions
+	}
+
+	fields["uncles"] = uncleHashes
+	return fields, nil
+}
+
+// NewRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
+func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction {
+	for idx, tx := range b.Transactions() {
+		if tx.Hash() == hash {
+			return newRPCTransactionFromBlockIndex(b, uint64(idx))
+		}
+	}
+	return nil
+}
+
+// NewRPCTransaction returns a transaction that will serialize to the RPC
+// representation, with the given location metadata set (if available).
+func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64, baseFee *big.Int) *RPCTransaction {
+	var signer types.Signer
+	if tx.Protected() {
+		signer = types.LatestSignerForChainID(tx.ChainId())
+	} else {
+		signer = types.HomesteadSigner{}
+	}
+	from, _ := types.Sender(signer, tx)
+	v, r, s := tx.RawSignatureValues()
+	result := &RPCTransaction{
+		From:     from,
+		Gas:      hexutil.Uint64(tx.Gas()),
+		GasPrice: (*hexutil.Big)(tx.GasPrice()),
+		Hash:     tx.Hash(),
+		Input:    hexutil.Bytes(tx.Data()), // somehow this is ending up `nil`
+		Nonce:    hexutil.Uint64(tx.Nonce()),
+		To:       tx.To(),
+		Value:    (*hexutil.Big)(tx.Value()),
+		Type:     hexutil.Uint64(tx.Type()),
+		V:        (*hexutil.Big)(v),
+		R:        (*hexutil.Big)(r),
+		S:        (*hexutil.Big)(s),
+	}
+	if blockHash != (common.Hash{}) {
+		result.BlockHash = &blockHash
+		result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber))
+		result.TransactionIndex = (*hexutil.Uint64)(&index)
+	}
+	switch tx.Type() {
+	case types.AccessListTxType:
+		al := tx.AccessList()
+		result.Accesses = &al
+		result.ChainID = (*hexutil.Big)(tx.ChainId())
+	case types.DynamicFeeTxType:
+		al := tx.AccessList()
+		result.Accesses = &al
+		result.ChainID = (*hexutil.Big)(tx.ChainId())
+		result.GasFeeCap = (*hexutil.Big)(tx.GasFeeCap())
+		result.GasTipCap = (*hexutil.Big)(tx.GasTipCap())
+		// if the transaction has been mined, compute the effective gas price
+		if baseFee != nil && blockHash != (common.Hash{}) {
+			// price = min(tip, gasFeeCap - baseFee) + baseFee
+			price := math.BigMin(new(big.Int).Add(tx.GasTipCap(), baseFee), tx.GasFeeCap())
+			result.GasPrice = (*hexutil.Big)(price)
+		} else {
+			result.GasPrice = nil
+		}
+	}
+	return result
+}
+
+type rpcBlock struct {
+	Hash         common.Hash      `json:"hash"`
+	Transactions []rpcTransaction `json:"transactions"`
+	UncleHashes  []common.Hash    `json:"uncles"`
+}
+
+type rpcTransaction struct {
+	tx *types.Transaction
+	txExtraInfo
+}
+
+type txExtraInfo struct {
+	BlockNumber *string         `json:"blockNumber,omitempty"`
+	BlockHash   *common.Hash    `json:"blockHash,omitempty"`
+	From        *common.Address `json:"from,omitempty"`
+}
+
+func (tx *rpcTransaction) UnmarshalJSON(msg []byte) error {
+	if err := json.Unmarshal(msg, &tx.tx); err != nil {
+		return err
+	}
+	return json.Unmarshal(msg, &tx.txExtraInfo)
+}
+
+func getBlockAndUncleHashes(cli *rpc.Client, ctx context.Context, method string, args ...interface{}) (*types.Block, []common.Hash, error) {
+	var raw json.RawMessage
+	err := cli.CallContext(ctx, &raw, method, args...)
+	if err != nil {
+		return nil, nil, err
+	} else if len(raw) == 0 {
+		return nil, nil, ethereum.NotFound
+	}
+	// Decode header and transactions.
+	var head *types.Header
+	var body rpcBlock
+	if err := json.Unmarshal(raw, &head); err != nil {
+		return nil, nil, err
+	}
+	if err := json.Unmarshal(raw, &body); err != nil {
+		return nil, nil, err
+	}
+	// Quick-verify transaction and uncle lists. This mostly helps with debugging the server.
+	if head.UncleHash == types.EmptyUncleHash && len(body.UncleHashes) > 0 {
+		return nil, nil, fmt.Errorf("server returned non-empty uncle list but block header indicates no uncles")
+	}
+	if head.UncleHash != types.EmptyUncleHash && len(body.UncleHashes) == 0 {
+		return nil, nil, fmt.Errorf("server returned empty uncle list but block header indicates uncles")
+	}
+	if head.TxHash == types.EmptyRootHash && len(body.Transactions) > 0 {
+		return nil, nil, fmt.Errorf("server returned non-empty transaction list but block header indicates no transactions")
+	}
+	if head.TxHash != types.EmptyRootHash && len(body.Transactions) == 0 {
+		return nil, nil, fmt.Errorf("server returned empty transaction list but block header indicates transactions")
+	}
+	txs := make([]*types.Transaction, len(body.Transactions))
+	for i, tx := range body.Transactions {
+		txs[i] = tx.tx
+	}
+	return types.NewBlockWithHeader(head).WithBody(txs, nil), body.UncleHashes, nil
+}
+
+// newRPCRawTransactionFromBlockIndex returns the bytes of a transaction given a block and a transaction index.
+func newRPCRawTransactionFromBlockIndex(b *types.Block, index uint64) hexutil.Bytes {
+	txs := b.Transactions()
+	if index >= uint64(len(txs)) {
+		return nil
+	}
+	blob, _ := rlp.EncodeToBytes(txs[index])
+	return blob
+}
+
+// newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation.
+func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransaction {
+	txs := b.Transactions()
+	if index >= uint64(len(txs)) {
+		return nil
+	}
+	return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index, b.BaseFee())
+}
+
+func toFilterArg(q ethereum.FilterQuery) (interface{}, error) {
+	arg := map[string]interface{}{
+		"address": q.Addresses,
+		"topics":  q.Topics,
+	}
+	if q.BlockHash != nil {
+		arg["blockHash"] = *q.BlockHash
+		if q.FromBlock != nil || q.ToBlock != nil {
+			return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock")
+		}
+	} else {
+		if q.FromBlock == nil {
+			arg["fromBlock"] = "0x0"
+		} else {
+			arg["fromBlock"] = toBlockNumArg(q.FromBlock)
+		}
+		arg["toBlock"] = toBlockNumArg(q.ToBlock)
+	}
+	return arg, nil
+}
+
+func toBlockNumArg(number *big.Int) string {
+	if number == nil {
+		return "latest"
+	}
+	return hexutil.EncodeBig(number)
+}

pkg/eth/cid_retriever.go

@@ -22,14 +22,14 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
 	"github.com/jmoiron/sqlx"
 	"github.com/lib/pq"
 	log "github.com/sirupsen/logrus"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
 // Retriever interface for substituting mocks in tests
@@ -41,13 +41,74 @@ type Retriever interface {
 
 // CIDRetriever satisfies the CIDRetriever interface for ethereum
 type CIDRetriever struct {
-	db *postgres.DB
+	db     *sqlx.DB
+	gormDB *gorm.DB
+}
+
+type IPLDModelRecord struct {
+	models.IPLDModel
+}
+
+// TableName overrides the table name used by IPLD
+func (IPLDModelRecord) TableName() string {
+	return "public.blocks"
+}
+
+type HeaderCIDRecord struct {
+	CID             string `gorm:"column:cid"`
+	BlockHash       string `gorm:"primaryKey"`
+	BlockNumber     string
+	ParentHash      string
+	Timestamp       uint64
+	StateRoot       string
+	TotalDifficulty string `gorm:"column:td"`
+	TxRoot          string
+	RctRoot         string `gorm:"column:receipt_root"`
+	UncleRoot       string
+	Bloom           []byte
+	MhKey           string
+
+	// gorm doesn't check if foreign key exists in database.
+	// It is required to eager load relations using preload.
+	TransactionCIDs []TransactionCIDRecord `gorm:"foreignKey:HeaderID;references:BlockHash"`
+	IPLD            IPLDModelRecord        `gorm:"foreignKey:MhKey;references:Key"`
+}
+
+// TableName overrides the table name used by HeaderCIDRecord
+func (HeaderCIDRecord) TableName() string {
+	return "eth.header_cids"
+}
+
+type TransactionCIDRecord struct {
+	CID      string `gorm:"column:cid"`
+	TxHash   string `gorm:"primaryKey"`
+	HeaderID string `gorm:"column:header_id"`
+	Index    int64
+	Src      string
+	Dst      string
+	MhKey    string
+	IPLD     IPLDModelRecord `gorm:"foreignKey:MhKey;references:Key"`
+}
+
+// TableName overrides the table name used by TransactionCIDRecord
+func (TransactionCIDRecord) TableName() string {
+	return "eth.transaction_cids"
 }
 
 // NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface
-func NewCIDRetriever(db *postgres.DB) *CIDRetriever {
+func NewCIDRetriever(db *sqlx.DB) *CIDRetriever {
+	gormDB, err := gorm.Open(postgres.New(postgres.Config{
+		Conn: db,
+	}), &gorm.Config{})
+
+	if err != nil {
+		log.Error(err)
+		return nil
+	}
+
 	return &CIDRetriever{
 		db:     db,
+		gormDB: gormDB,
 	}
 }
 
@@ -86,9 +147,10 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 	}()
 
 	// Retrieve cached header CIDs at this block height
-	headers, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
+	var headers []models.HeaderModel
+	headers, err = ecr.RetrieveHeaderCIDs(tx, blockNumber)
 	if err != nil {
-		log.Error("header cid retrieval error")
+		log.Error("header cid retrieval error", err)
 		return nil, true, err
 	}
 	cws := make([]CIDWrapper, len(headers))
@@ -101,7 +163,8 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 			empty = false
 			if filter.HeaderFilter.Uncles {
 				// Retrieve uncle cids for this header id
-				uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, header.ID)
+				var uncleCIDs []models.UncleModel
+				uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, header.BlockHash)
 				if err != nil {
 					log.Error("uncle cid retrieval error")
 					return nil, true, err
@@ -111,7 +174,7 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 		}
 		// Retrieve cached trx CIDs
 		if !filter.TxFilter.Off {
-			cw.Transactions, err = ecr.RetrieveTxCIDs(tx, filter.TxFilter, header.ID)
+			cw.Transactions, err = ecr.RetrieveTxCIDs(tx, filter.TxFilter, header.BlockHash)
 			if err != nil {
 				log.Error("transaction cid retrieval error")
 				return nil, true, err
@@ -120,13 +183,13 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 				empty = false
 			}
 		}
-		trxIds := make([]int64, len(cw.Transactions))
+		trxHashes := make([]string, len(cw.Transactions))
-		for j, tx := range cw.Transactions {
+		for j, t := range cw.Transactions {
-			trxIds[j] = tx.ID
+			trxHashes[j] = t.TxHash
 		}
 		// Retrieve cached receipt CIDs
 		if !filter.ReceiptFilter.Off {
-			cw.Receipts, err = ecr.RetrieveRctCIDsByHeaderID(tx, filter.ReceiptFilter, header.ID, trxIds)
+			cw.Receipts, err = ecr.RetrieveRctCIDsByHeaderID(tx, filter.ReceiptFilter, header.BlockHash, trxHashes)
 			if err != nil {
 				log.Error("receipt cid retrieval error")
 				return nil, true, err
@@ -137,7 +200,7 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 		}
 		// Retrieve cached state CIDs
 		if !filter.StateFilter.Off {
-			cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, filter.StateFilter, header.ID)
+			cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, filter.StateFilter, header.BlockHash)
 			if err != nil {
 				log.Error("state cid retrieval error")
 				return nil, true, err
@@ -148,7 +211,7 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 		}
 		// Retrieve cached storage CIDs
 		if !filter.StorageFilter.Off {
-			cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, filter.StorageFilter, header.ID)
+			cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, filter.StorageFilter, header.BlockHash)
 			if err != nil {
 				log.Error("storage cid retrieval error")
 				return nil, true, err
@@ -164,35 +227,36 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 }
 
 // RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
-func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]eth.HeaderModel, error) {
+func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]models.HeaderModel, error) {
 	log.Debug("retrieving header cids for block ", blockNumber)
-	headers := make([]eth.HeaderModel, 0)
+	headers := make([]models.HeaderModel, 0)
-	pgStr := `SELECT * FROM eth.header_cids
+	pgStr := `SELECT CAST(block_number as Text), block_hash,parent_hash,cid,mh_key,CAST(td as Text),node_id,
+				CAST(reward as Text), state_root, uncle_root, tx_root, receipt_root, bloom, timestamp, times_validated,
+				coinbase FROM eth.header_cids
 				WHERE block_number = $1`
 	return headers, tx.Select(&headers, pgStr, blockNumber)
 }
 
 // RetrieveUncleCIDsByHeaderID retrieves and returns all of the uncle cids for the provided header
-func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth.UncleModel, error) {
+func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID string) ([]models.UncleModel, error) {
 	log.Debug("retrieving uncle cids for block id ", headerID)
-	headers := make([]eth.UncleModel, 0)
+	headers := make([]models.UncleModel, 0)
-	pgStr := `SELECT * FROM eth.uncle_cids
+	pgStr := `SELECT header_id,block_hash,parent_hash,cid,mh_key, CAST(reward as text) FROM eth.uncle_cids
 				WHERE header_id = $1`
 	return headers, tx.Select(&headers, pgStr, headerID)
 }
 
 // RetrieveTxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters
 // also returns the ids for the returned transaction cids
-func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]eth.TxModel, error) {
+func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID string) ([]models.TxModel, error) {
 	log.Debug("retrieving transaction cids for header id ", headerID)
 	args := make([]interface{}, 0, 3)
-	results := make([]eth.TxModel, 0)
+	results := make([]models.TxModel, 0)
 	id := 1
-	pgStr := fmt.Sprintf(`SELECT transaction_cids.id, transaction_cids.header_id,
+	pgStr := fmt.Sprintf(`SELECT transaction_cids.tx_hash, transaction_cids.header_id,transaction_cids.cid, transaction_cids.mh_key,
- 			transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.mh_key,
 				transaction_cids.dst, transaction_cids.src, transaction_cids.index, transaction_cids.tx_data
- 			FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
+				FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash)
-			WHERE header_cids.id = $%d`, id)
+			WHERE header_cids.block_hash = $%d`, id)
 	args = append(args, headerID)
 	id++
 	if len(txFilter.Dst) > 0 {
@@ -208,97 +272,148 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID
 	return results, tx.Select(&results, pgStr, args...)
 }
 
-// RetrieveRctCIDsByHeaderID retrieves and returns all of the rct cids at the provided header ID that conform to the provided
+func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) {
-// filter parameters and correspond to the provided tx ids
+	for i, topicSet := range topics {
-func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]eth.ReceiptModel, error) {
+		if len(topicSet) == 0 {
-	log.Debug("retrieving receipt cids for header id ", headerID)
+			continue
-	args := make([]interface{}, 0, 4)
+		}
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
+
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
+		if !first {
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
+			pgStr += " AND"
- 			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+		} else {
-			WHERE receipt_cids.tx_id = transaction_cids.id 
+			first = false
-			AND transaction_cids.header_id = header_cids.id
+		}
-			AND header_cids.id = $1`
+		pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id)
-	id := 2
+		args = append(args, pq.Array(topicSet))
-	args = append(args, headerID)
+		*id++
+	}
+	return pgStr, args
+}
+
+func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) {
 	if len(rctFilter.LogAddresses) > 0 {
-		// Filter on log contract addresses if there are any
+		pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id)
-		pgStr += fmt.Sprintf(` AND ((receipt_cids.log_contracts && $%d::VARCHAR(66)[]`, id)
 		args = append(args, pq.Array(rctFilter.LogAddresses))
-		id++
+		*id++
+	}
+
 	// Filter on topics if there are any
 	if hasTopics(rctFilter.Topics) {
-			pgStr += " AND ("
+		pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
 	}
-					args = append(args, pq.Array(topicSet))
+
-					id++
+	return pgStr, args
 }
+
+func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) {
+	rctCond := " AND (receipt_cids.tx_id = ANY ( "
+	logQuery := "SELECT rct_id FROM eth.log_cids WHERE"
+	if len(rctFilter.LogAddresses) > 0 {
+		// Filter on log contract addresses if there are any
+		pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id)
+		args = append(args, pq.Array(rctFilter.LogAddresses))
+		*id++
+
+		// Filter on topics if there are any
+		if hasTopics(rctFilter.Topics) {
+			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
 		}
+
 		pgStr += ")"
-		}
+
-		pgStr += ")"
+		// Filter on txHashes if there are any, and we are matching txs
-		// Filter on txIDs if there are any and we are matching txs
+		if rctFilter.MatchTxs && len(txHashes) > 0 {
-		if rctFilter.MatchTxs && len(trxIds) > 0 {
+			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
-			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
+			args = append(args, pq.Array(txHashes))
-			args = append(args, pq.Array(trxIds))
 		}
 		pgStr += ")"
 	} else { // If there are no contract addresses to filter on
 		// Filter on topics if there are any
 		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND (("
+			pgStr += rctCond + logQuery
-			first := true
+			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true)
-			for i, topicSet := range rctFilter.Topics {
+			pgStr += ")"
-				if i < 4 && len(topicSet) > 0 {
+			// Filter on txHashes if there are any, and we are matching txs
-					if first {
+			if rctFilter.MatchTxs && len(txHashes) > 0 {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
+				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
-						first = false
+				args = append(args, pq.Array(txHashes))
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
 			}
 			pgStr += ")"
-			// Filter on txIDs if there are any and we are matching txs
+		} else if rctFilter.MatchTxs && len(txHashes) > 0 {
-			if rctFilter.MatchTxs && len(trxIds) > 0 {
-				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-				args = append(args, pq.Array(trxIds))
-			}
-			pgStr += ")"
-		} else if rctFilter.MatchTxs && len(trxIds) > 0 {
 			// If there are no contract addresses or topics to filter on,
-			// Filter on txIDs if there are any and we are matching txs
+			// Filter on txHashes if there are any, and we are matching txs
-			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
+			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id)
-			args = append(args, pq.Array(trxIds))
+			args = append(args, pq.Array(txHashes))
 		}
 	}
-	pgStr += ` ORDER BY transaction_cids.index`
-	receiptCids := make([]eth.ReceiptModel, 0)
-	return receiptCids, tx.Select(&receiptCids, pgStr, args...)
-}
 
-// RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight or block hash that conform to the provided
+	return pgStr, args
+}
+
+// RetrieveRctCIDsByHeaderID retrieves and returns all of the rct cids at the provided header ID that conform to the provided
 // filter parameters and correspond to the provided tx ids
-func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]eth.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID string, trxHashes []string) ([]models.ReceiptModel, error) {
-	log.Debug("retrieving receipt cids for block ", blockNumber)
+	log.Debug("retrieving receipt cids for header id ", headerID)
-	args := make([]interface{}, 0, 5)
+	args := make([]interface{}, 0, 4)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
+	pgStr := `SELECT receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
+ 			receipt_cids.contract, receipt_cids.contract_hash
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
  			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
-			WHERE receipt_cids.tx_id = transaction_cids.id 
+			WHERE receipt_cids.tx_id = transaction_cids.tx_hash
-			AND transaction_cids.header_id = header_cids.id`
+			AND transaction_cids.header_id = header_cids.block_hash
+			AND header_cids.block_hash = $1`
+	id := 2
+	args = append(args, headerID)
+
+	pgStr, args = receiptFilterConditions(&id, pgStr, args, rctFilter, trxHashes)
+
+	pgStr += ` ORDER BY transaction_cids.index`
+	receiptCIDs := make([]models.ReceiptModel, 0)
+	return receiptCIDs, tx.Select(&receiptCIDs, pgStr, args...)
+}
+
+// RetrieveFilteredGQLLogs retrieves and returns all the log cIDs provided blockHash that conform to the provided
+// filter parameters.
+func (ecr *CIDRetriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash) ([]LogResult, error) {
+	log.Debug("retrieving log cids for receipt ids")
+	args := make([]interface{}, 0, 4)
+	id := 1
+	pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id,
+       			eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
+       			eth.log_cids.log_data, eth.transaction_cids.tx_hash, data, eth.receipt_cids.leaf_cid as cid, eth.receipt_cids.post_status
+				FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks
+				WHERE eth.log_cids.rct_id = receipt_cids.tx_id
+				AND receipt_cids.tx_id = transaction_cids.tx_hash
+ 				AND transaction_cids.header_id = header_cids.block_hash
+ 				AND log_cids.leaf_mh_key = blocks.key AND header_cids.block_hash = $1`
+
+	args = append(args, blockHash.String())
+	id++
+
+	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
+	pgStr += ` ORDER BY log_cids.index`
+
+	logCIDs := make([]LogResult, 0)
+	err := tx.Select(&logCIDs, pgStr, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	return logCIDs, nil
+}
+
+// RetrieveFilteredLog retrieves and returns all the log cIDs provided blockHeight or blockHash that conform to the provided
+// filter parameters.
+func (ecr *CIDRetriever) RetrieveFilteredLog(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash) ([]LogResult, error) {
+	log.Debug("retrieving log cids for receipt ids")
+	args := make([]interface{}, 0, 4)
+	pgStr := `SELECT eth.log_cids.leaf_cid, eth.log_cids.index, eth.log_cids.rct_id,
+       			eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
+       			eth.log_cids.log_data, eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index,
+       			header_cids.block_hash, CAST(header_cids.block_number as Text)
+							FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, eth.header_cids
+							WHERE eth.log_cids.rct_id = receipt_cids.tx_id
+							AND receipt_cids.tx_id = transaction_cids.tx_hash
+							AND transaction_cids.header_id = header_cids.block_hash`
 	id := 1
 	if blockNumber > 0 {
 		pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id)
@@ -310,70 +425,45 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b
 		args = append(args, blockHash.String())
 		id++
 	}
-	if len(rctFilter.LogAddresses) > 0 {
+
-		// Filter on log contract addresses if there are any
+	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
-		pgStr += fmt.Sprintf(` AND ((receipt_cids.log_contracts && $%d::VARCHAR(66)[]`, id)
+	pgStr += ` ORDER BY log_cids.index`
-		args = append(args, pq.Array(rctFilter.LogAddresses))
+
-		id++
+	logCIDs := make([]LogResult, 0)
-		// Filter on topics if there are any
+	err := tx.Select(&logCIDs, pgStr, args...)
-		if hasTopics(rctFilter.Topics) {
+	if err != nil {
-			pgStr += " AND ("
+		return nil, err
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
 	}
-					args = append(args, pq.Array(topicSet))
+
+	return logCIDs, nil
+}
+
+// RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight or block hash that conform to the provided
+// filter parameters and correspond to the provided tx ids
+func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, txHashes []string) ([]models.ReceiptModel, error) {
+	log.Debug("retrieving receipt cids for block ", blockNumber)
+	args := make([]interface{}, 0, 5)
+	pgStr := `SELECT receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key, receipt_cids.tx_id
+ 			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+			WHERE receipt_cids.tx_id = transaction_cids.tx_hash
+			AND transaction_cids.header_id = header_cids.block_hash`
+	id := 1
+	if blockNumber > 0 {
+		pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id)
+		args = append(args, blockNumber)
 		id++
 	}
-			}
+	if blockHash != nil {
-			pgStr += ")"
+		pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id)
-		}
+		args = append(args, blockHash.String())
-		pgStr += ")"
-		// Filter on txIDs if there are any and we are matching txs
-		if rctFilter.MatchTxs && len(trxIds) > 0 {
-			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-		pgStr += ")"
-	} else { // If there are no contract addresses to filter on
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND (("
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
 		id++
 	}
-			}
+
-			pgStr += ")"
+	pgStr, args = receiptFilterConditions(&id, pgStr, args, rctFilter, txHashes)
-			// Filter on txIDs if there are any and we are matching txs
+
-			if rctFilter.MatchTxs && len(trxIds) > 0 {
-				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-				args = append(args, pq.Array(trxIds))
-			}
-			pgStr += ")"
-		} else if rctFilter.MatchTxs && len(trxIds) > 0 {
-			// If there are no contract addresses or topics to filter on,
-			// Filter on txIDs if there are any and we are matching txs
-			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-	}
 	pgStr += ` ORDER BY transaction_cids.index`
-	receiptCids := make([]eth.ReceiptModel, 0)
+	receiptCIDs := make([]models.ReceiptModel, 0)
-	return receiptCids, tx.Select(&receiptCids, pgStr, args...)
+	return receiptCIDs, tx.Select(&receiptCIDs, pgStr, args...)
 }
 
 func hasTopics(topics [][]string) bool {
@@ -386,13 +476,13 @@ func hasTopics(topics [][]string) bool {
 }
 
 // RetrieveStateCIDs retrieves and returns all of the state node cids at the provided header ID that conform to the provided filter parameters
-func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID int64) ([]eth.StateNodeModel, error) {
+func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID string) ([]models.StateNodeModel, error) {
 	log.Debug("retrieving state cids for header id ", headerID)
 	args := make([]interface{}, 0, 2)
-	pgStr := `SELECT state_cids.id, state_cids.header_id,
+	pgStr := `SELECT state_cids.header_id,
 			state_cids.state_leaf_key, state_cids.node_type, state_cids.cid, state_cids.mh_key, state_cids.state_path
-			FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+			FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
-			WHERE header_cids.id = $1`
+			WHERE header_cids.block_hash = $1`
 	args = append(args, headerID)
 	addrLen := len(stateFilter.Addresses)
 	if addrLen > 0 {
@@ -406,20 +496,20 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
 	if !stateFilter.IntermediateNodes {
 		pgStr += ` AND state_cids.node_type = 2`
 	}
-	stateNodeCIDs := make([]eth.StateNodeModel, 0)
+	stateNodeCIDs := make([]models.StateNodeModel, 0)
 	return stateNodeCIDs, tx.Select(&stateNodeCIDs, pgStr, args...)
 }
 
 // RetrieveStorageCIDs retrieves and returns all of the storage node cids at the provided header id that conform to the provided filter parameters
-func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID int64) ([]eth.StorageNodeWithStateKeyModel, error) {
+func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID string) ([]models.StorageNodeWithStateKeyModel, error) {
 	log.Debug("retrieving storage cids for header id ", headerID)
 	args := make([]interface{}, 0, 3)
-	pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_leaf_key, storage_cids.node_type,
+	pgStr := `SELECT storage_cids.header_id, storage_cids.storage_leaf_key, storage_cids.node_type,
- 			storage_cids.cid, storage_cids.mh_key, storage_cids.storage_path, state_cids.state_leaf_key
+ 			storage_cids.cid, storage_cids.mh_key, storage_cids.storage_path, storage_cids.state_path, state_cids.state_leaf_key
  			FROM eth.storage_cids, eth.state_cids, eth.header_cids
-			WHERE storage_cids.state_id = state_cids.id 
+			WHERE storage_cids.header_id = state_cids.header_id AND storage_cids.state_path = state_cids.state_path
-			AND state_cids.header_id = header_cids.id
+			AND state_cids.header_id = header_cids.block_hash
-			AND header_cids.id = $1`
+			AND header_cids.block_hash = $1`
 	args = append(args, headerID)
 	id := 2
 	addrLen := len(storageFilter.Addresses)
@@ -439,18 +529,18 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF
 	if !storageFilter.IntermediateNodes {
 		pgStr += ` AND storage_cids.node_type = 2`
 	}
-	storageNodeCIDs := make([]eth.StorageNodeWithStateKeyModel, 0)
+	storageNodeCIDs := make([]models.StorageNodeWithStateKeyModel, 0)
 	return storageNodeCIDs, tx.Select(&storageNodeCIDs, pgStr, args...)
 }
 
 // RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash
-func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth.HeaderModel, []eth.UncleModel, []eth.TxModel, []eth.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (models.HeaderModel, []models.UncleModel, []models.TxModel, []models.ReceiptModel, error) {
 	log.Debug("retrieving block cids for block hash ", blockHash.String())
 
 	// Begin new db tx
 	tx, err := ecr.db.Beginx()
 	if err != nil {
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
 	defer func() {
 		if p := recover(); p != nil {
@@ -463,26 +553,30 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth.HeaderM
 		}
 	}()
 
-	headerCID, err := ecr.RetrieveHeaderCIDByHash(tx, blockHash)
+	var headerCID models.HeaderModel
+	headerCID, err = ecr.RetrieveHeaderCIDByHash(tx, blockHash)
 	if err != nil {
 		log.Error("header cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
-	uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.ID)
+	var uncleCIDs []models.UncleModel
+	uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.BlockHash)
 	if err != nil {
 		log.Error("uncle cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
-	txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.ID)
+	var txCIDs []models.TxModel
+	txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.BlockHash)
 	if err != nil {
 		log.Error("tx cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
-	txIDs := make([]int64, len(txCIDs))
+	txHashes := make([]string, len(txCIDs))
 	for i, txCID := range txCIDs {
-		txIDs[i] = txCID.ID
+		txHashes[i] = txCID.TxHash
 	}
-	rctCIDs, err := ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
+	var rctCIDs []models.ReceiptModel
+	rctCIDs, err = ecr.RetrieveReceiptCIDsByTxIDs(tx, txHashes)
 	if err != nil {
 		log.Error("rct cid retrieval error")
 	}
@@ -490,13 +584,13 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth.HeaderM
 }
 
 // RetrieveBlockByNumber returns all of the CIDs needed to compose an entire block, for a given block number
-func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth.HeaderModel, []eth.UncleModel, []eth.TxModel, []eth.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (models.HeaderModel, []models.UncleModel, []models.TxModel, []models.ReceiptModel, error) {
 	log.Debug("retrieving block cids for block number ", blockNumber)
 
 	// Begin new db tx
 	tx, err := ecr.db.Beginx()
 	if err != nil {
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
 	defer func() {
 		if p := recover(); p != nil {
@@ -509,29 +603,33 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth.HeaderMod
 		}
 	}()
 
-	headerCID, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
+	var headerCID []models.HeaderModel
+	headerCID, err = ecr.RetrieveHeaderCIDs(tx, blockNumber)
 	if err != nil {
 		log.Error("header cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
 	if len(headerCID) < 1 {
-		return eth.HeaderModel{}, nil, nil, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber)
+		return models.HeaderModel{}, nil, nil, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber)
 	}
-	uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID[0].ID)
+	var uncleCIDs []models.UncleModel
+	uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID[0].BlockHash)
 	if err != nil {
 		log.Error("uncle cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
-	txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].ID)
+	var txCIDs []models.TxModel
+	txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].BlockHash)
 	if err != nil {
 		log.Error("tx cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
+		return models.HeaderModel{}, nil, nil, nil, err
 	}
-	txIDs := make([]int64, len(txCIDs))
+	txHashes := make([]string, len(txCIDs))
 	for i, txCID := range txCIDs {
-		txIDs[i] = txCID.ID
+		txHashes[i] = txCID.TxHash
 	}
-	rctCIDs, err := ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
+	var rctCIDs []models.ReceiptModel
+	rctCIDs, err = ecr.RetrieveReceiptCIDsByTxIDs(tx, txHashes)
 	if err != nil {
 		log.Error("rct cid retrieval error")
 	}
@@ -539,34 +637,89 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth.HeaderMod
 }
 
 // RetrieveHeaderCIDByHash returns the header for the given block hash
-func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (eth.HeaderModel, error) {
+func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (models.HeaderModel, error) {
 	log.Debug("retrieving header cids for block hash ", blockHash.String())
-	pgStr := `SELECT * FROM eth.header_cids
+	pgStr := `SELECT block_hash, CAST(block_number as Text), parent_hash, cid, mh_key, CAST(td as Text),
+			state_root, uncle_root, tx_root, receipt_root, bloom, timestamp FROM eth.header_cids
 			WHERE block_hash = $1`
-	var headerCID eth.HeaderModel
+	var headerCID models.HeaderModel
 	return headerCID, tx.Get(&headerCID, pgStr, blockHash.String())
 }
 
 // RetrieveTxCIDsByHeaderID retrieves all tx CIDs for the given header id
-func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth.TxModel, error) {
+func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID string) ([]models.TxModel, error) {
 	log.Debug("retrieving tx cids for block id ", headerID)
 	pgStr := `SELECT * FROM eth.transaction_cids
 			WHERE header_id = $1
 			ORDER BY index`
-	var txCIDs []eth.TxModel
+	var txCIDs []models.TxModel
 	return txCIDs, tx.Select(&txCIDs, pgStr, headerID)
 }
 
 // RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs
-func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]eth.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txHashes []string) ([]models.ReceiptModel, error) {
-	log.Debugf("retrieving receipt cids for tx ids %v", txIDs)
+	log.Debugf("retrieving receipt cids for tx hashes %v", txHashes)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
+	pgStr := `SELECT receipt_cids.tx_id, receipt_cids.leaf_cid, receipt_cids.leaf_mh_key,
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
+ 			receipt_cids.contract, receipt_cids.contract_hash
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
 			FROM eth.receipt_cids, eth.transaction_cids
-			WHERE tx_id = ANY($1::INTEGER[])
+			WHERE tx_id = ANY($1)
-			AND receipt_cids.tx_id = transaction_cids.id
+			AND receipt_cids.tx_id = transaction_cids.tx_hash
 			ORDER BY transaction_cids.index`
-	var rctCIDs []eth.ReceiptModel
+	var rctCIDs []models.ReceiptModel
-	return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txIDs))
+	return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txHashes))
+}
+
+// RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number
+func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCIDRecord, error) {
+	log.Debug("retrieving header cids and tx cids for block number ", blockNumber)
+
+	var headerCIDs []HeaderCIDRecord
+
+	// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
+	// Will use join for TransactionCIDs once preload for 1:N is supported.
+	err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
+		return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id")
+	}).Joins("IPLD").Find(&headerCIDs, "block_number = ?", blockNumber).Error
+
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return nil, err
+	}
+
+	return headerCIDs, nil
+}
+
+// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash
+func (ecr *CIDRetriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash) (HeaderCIDRecord, error) {
+	log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String())
+
+	var headerCID HeaderCIDRecord
+
+	// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
+	// Will use join for TransactionCIDs once preload for 1:N is supported.
+	err := ecr.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
+		return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id")
+	}).Joins("IPLD").First(&headerCID, "block_hash = ?", blockHash.String()).Error
+
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return headerCID, err
+	}
+
+	return headerCID, nil
+}
+
+// RetrieveTxCIDByHash returns the tx for the given tx hash
+func (ecr *CIDRetriever) RetrieveTxCIDByHash(txHash string) (TransactionCIDRecord, error) {
+	log.Debug("retrieving tx cid for tx hash ", txHash)
+
+	var txCID TransactionCIDRecord
+
+	err := ecr.gormDB.Joins("IPLD").First(&txCID, "tx_hash = ?", txHash).Error
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return txCID, err
+	}
+
+	return txCID, nil
 }

pkg/eth/cid_retriever_test.go

@@ -21,15 +21,17 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/jmoiron/sqlx"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth/test_helpers"
-
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
 var (
@@ -209,45 +211,64 @@ var (
 
 var _ = Describe("Retriever", func() {
 	var (
-		db        *postgres.DB
+		db          *sqlx.DB
-		repo      *eth2.IPLDPublisher
+		diffIndexer interfaces.StateDiffIndexer
 		retriever   *eth.CIDRetriever
 	)
 	BeforeEach(func() {
-		var err error
+		db = shared.SetupDB()
-		db, err = shared.SetupDB()
+		diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash())
-		Expect(err).ToNot(HaveOccurred())
+
-		repo = eth2.NewIPLDPublisher(db)
 		retriever = eth.NewCIDRetriever(db)
 	})
 	AfterEach(func() {
-		eth.TearDownDB(db)
+		shared.TearDownDB(db)
 	})
 
 	Describe("Retrieve", func() {
 		BeforeEach(func() {
-			err := repo.Publish(test_helpers.MockConvertedPayload)
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			for _, node := range test_helpers.MockStateNodes {
+				err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
+			type rctCIDAndMHKeyResult struct {
+				LeafCID   string `db:"leaf_cid"`
+				LeafMhKey string `db:"leaf_mh_key"`
+			}
+			expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0)
+			pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+				WHERE receipt_cids.tx_id = transaction_cids.tx_hash
+				AND transaction_cids.header_id = header_cids.block_hash
+				AND header_cids.block_number = $1
+				ORDER BY transaction_cids.index`
+			err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
 			cids, empty, err := retriever.Retrieve(openFilter, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids)).To(Equal(1))
 			Expect(cids[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
+
 			expectedHeaderCID := test_helpers.MockCIDWrapper.Header
-			expectedHeaderCID.ID = cids[0].Header.ID
+			expectedHeaderCID.BlockHash = cids[0].Header.BlockHash
 			expectedHeaderCID.NodeID = cids[0].Header.NodeID
 			Expect(cids[0].Header).To(Equal(expectedHeaderCID))
-			Expect(len(cids[0].Transactions)).To(Equal(3))
+			Expect(len(cids[0].Transactions)).To(Equal(4))
 			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[0].CID)).To(BeTrue())
 			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
 			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[2].CID)).To(BeTrue())
-			Expect(len(cids[0].Receipts)).To(Equal(3))
+			Expect(len(cids[0].Receipts)).To(Equal(4))
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[0].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[2].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue())
 			Expect(len(cids[0].StateNodes)).To(Equal(2))
+
 			for _, stateNode := range cids[0].StateNodes {
 				if stateNode.CID == test_helpers.State1CID.String() {
 					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(test_helpers.ContractLeafKey).Hex()))
@@ -262,25 +283,37 @@ var _ = Describe("Retriever", func() {
 			}
 			Expect(len(cids[0].StorageNodes)).To(Equal(1))
 			expectedStorageNodeCIDs := test_helpers.MockCIDWrapper.StorageNodes
-			expectedStorageNodeCIDs[0].ID = cids[0].StorageNodes[0].ID
+			expectedStorageNodeCIDs[0].HeaderID = cids[0].StorageNodes[0].HeaderID
-			expectedStorageNodeCIDs[0].StateID = cids[0].StorageNodes[0].StateID
+			expectedStorageNodeCIDs[0].StatePath = cids[0].StorageNodes[0].StatePath
 			Expect(cids[0].StorageNodes).To(Equal(expectedStorageNodeCIDs))
 		})
 
 		It("Applies filters from the provided config.Subscription", func() {
+			type rctCIDAndMHKeyResult struct {
+				LeafCID   string `db:"leaf_cid"`
+				LeafMhKey string `db:"leaf_mh_key"`
+			}
+			expectedRctCIDsAndLeafNodes := make([]rctCIDAndMHKeyResult, 0)
+			pgStr := `SELECT receipt_cids.leaf_cid, receipt_cids.leaf_mh_key FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
+				WHERE receipt_cids.tx_id = transaction_cids.tx_hash
+				AND transaction_cids.header_id = header_cids.block_hash
+				AND header_cids.block_number = $1
+				ORDER BY transaction_cids.index`
+			err := db.Select(&expectedRctCIDsAndLeafNodes, pgStr, test_helpers.BlockNumber.Uint64())
 			cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids1)).To(Equal(1))
 			Expect(cids1[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids1[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids1[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids1[0].Transactions)).To(Equal(0))
 			Expect(len(cids1[0].StateNodes)).To(Equal(0))
 			Expect(len(cids1[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids1[0].Receipts)).To(Equal(1))
 			expectedReceiptCID := test_helpers.MockCIDWrapper.Receipts[0]
-			expectedReceiptCID.ID = cids1[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids1[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
 			Expect(cids1[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1)
@@ -288,14 +321,15 @@ var _ = Describe("Retriever", func() {
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids2)).To(Equal(1))
 			Expect(cids2[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids2[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids2[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids2[0].Transactions)).To(Equal(0))
 			Expect(len(cids2[0].StateNodes)).To(Equal(0))
 			Expect(len(cids2[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids2[0].Receipts)).To(Equal(1))
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
-			expectedReceiptCID.ID = cids2[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids2[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
 			Expect(cids2[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids3, empty, err := retriever.Retrieve(rctTopicsAndAddressFilter, 1)
@@ -303,14 +337,15 @@ var _ = Describe("Retriever", func() {
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids3)).To(Equal(1))
 			Expect(cids3[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids3[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids3[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids3[0].Transactions)).To(Equal(0))
 			Expect(len(cids3[0].StateNodes)).To(Equal(0))
 			Expect(len(cids3[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids3[0].Receipts)).To(Equal(1))
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
-			expectedReceiptCID.ID = cids3[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids3[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[0].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[0].LeafMhKey
 			Expect(cids3[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids4, empty, err := retriever.Retrieve(rctAddressesAndTopicFilter, 1)
@@ -318,14 +353,15 @@ var _ = Describe("Retriever", func() {
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids4)).To(Equal(1))
 			Expect(cids4[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids4[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids4[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids4[0].Transactions)).To(Equal(0))
 			Expect(len(cids4[0].StateNodes)).To(Equal(0))
 			Expect(len(cids4[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids4[0].Receipts)).To(Equal(1))
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
-			expectedReceiptCID.ID = cids4[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids4[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey
 			Expect(cids4[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1)
@@ -333,35 +369,36 @@ var _ = Describe("Retriever", func() {
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids5)).To(Equal(1))
 			Expect(cids5[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids5[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids5[0].Header).To(Equal(models.HeaderModel{}))
-			Expect(len(cids5[0].Transactions)).To(Equal(3))
+			Expect(len(cids5[0].Transactions)).To(Equal(4))
 			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx1CID.String())).To(BeTrue())
 			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx2CID.String())).To(BeTrue())
 			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx3CID.String())).To(BeTrue())
 			Expect(len(cids5[0].StateNodes)).To(Equal(0))
 			Expect(len(cids5[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids5[0].Receipts)).To(Equal(3))
+			Expect(len(cids5[0].Receipts)).To(Equal(4))
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct1CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[0].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct2CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[1].LeafCID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct3CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, expectedRctCIDsAndLeafNodes[2].LeafCID)).To(BeTrue())
 
 			cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids6)).To(Equal(1))
 			Expect(cids6[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids6[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids6[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids6[0].Transactions)).To(Equal(1))
 			expectedTxCID := test_helpers.MockCIDWrapper.Transactions[1]
-			expectedTxCID.ID = cids6[0].Transactions[0].ID
+			expectedTxCID.TxHash = cids6[0].Transactions[0].TxHash
 			expectedTxCID.HeaderID = cids6[0].Transactions[0].HeaderID
 			Expect(cids6[0].Transactions[0]).To(Equal(expectedTxCID))
 			Expect(len(cids6[0].StateNodes)).To(Equal(0))
 			Expect(len(cids6[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids6[0].Receipts)).To(Equal(1))
 			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
-			expectedReceiptCID.ID = cids6[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids6[0].Receipts[0].TxID
+			expectedReceiptCID.LeafCID = expectedRctCIDsAndLeafNodes[1].LeafCID
+			expectedReceiptCID.LeafMhKey = expectedRctCIDsAndLeafNodes[1].LeafMhKey
 			Expect(cids6[0].Receipts[0]).To(Equal(expectedReceiptCID))
 
 			cids7, empty, err := retriever.Retrieve(stateFilter, 1)
@@ -369,13 +406,12 @@ var _ = Describe("Retriever", func() {
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids7)).To(Equal(1))
 			Expect(cids7[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids7[0].Header).To(Equal(eth2.HeaderModel{}))
+			Expect(cids7[0].Header).To(Equal(models.HeaderModel{}))
 			Expect(len(cids7[0].Transactions)).To(Equal(0))
 			Expect(len(cids7[0].Receipts)).To(Equal(0))
 			Expect(len(cids7[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids7[0].StateNodes)).To(Equal(1))
-			Expect(cids7[0].StateNodes[0]).To(Equal(eth2.StateNodeModel{
+			Expect(cids7[0].StateNodes[0]).To(Equal(models.StateNodeModel{
-				ID:       cids7[0].StateNodes[0].ID,
 				HeaderID: cids7[0].StateNodes[0].HeaderID,
 				NodeType: 2,
 				StateKey: common.BytesToHash(test_helpers.AccountLeafKey).Hex(),
@@ -396,8 +432,12 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the first block that has data in the database", func() {
-			err := repo.Publish(test_helpers.MockConvertedPayload)
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1)))
@@ -406,8 +446,12 @@ var _ = Describe("Retriever", func() {
 		It("Gets the number of the first block that has data in the database", func() {
 			payload := test_helpers.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
-			err := repo.Publish(payload)
+			tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1010101)))
@@ -418,10 +462,16 @@ var _ = Describe("Retriever", func() {
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
-			err := repo.Publish(payload1)
+			tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
-			err = repo.Publish(payload2)
+			err = tx.Submit(err)
 			Expect(err).ToNot(HaveOccurred())
+
+			tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(5)))
@@ -434,8 +484,11 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the latest block that has data in the database", func() {
-			err := repo.Publish(test_helpers.MockConvertedPayload)
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1)))
@@ -444,8 +497,12 @@ var _ = Describe("Retriever", func() {
 		It("Gets the number of the latest block that has data in the database", func() {
 			payload := test_helpers.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
-			err := repo.Publish(payload)
+			tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1010101)))
@@ -456,10 +513,16 @@ var _ = Describe("Retriever", func() {
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
-			err := repo.Publish(payload1)
+			tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty())
 			Expect(err).ToNot(HaveOccurred())
-			err = repo.Publish(payload2)
+			err = tx.Submit(err)
 			Expect(err).ToNot(HaveOccurred())
+
+			tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(num).To(Equal(int64(1010101)))
@@ -470,5 +533,5 @@ var _ = Describe("Retriever", func() {
 func newMockBlock(blockNumber uint64) *types.Block {
 	header := test_helpers.MockHeader
 	header.Number.SetUint64(blockNumber)
-	return types.NewBlock(&test_helpers.MockHeader, test_helpers.MockTransactions, nil, test_helpers.MockReceipts)
+	return types.NewBlock(&test_helpers.MockHeader, test_helpers.MockTransactions, nil, test_helpers.MockReceipts, new(trie.Trie))
 }

pkg/eth/eth_call_test.go

@@ -1,165 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	"bytes"
-	"context"
-	"io/ioutil"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/statediff"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-var (
-	parsedABI abi.ABI
-)
-
-func init() {
-	// load abi
-	abiBytes, err := ioutil.ReadFile("./test_helpers/abi.json")
-	if err != nil {
-		panic(err)
-	}
-	parsedABI, err = abi.JSON(bytes.NewReader(abiBytes))
-	if err != nil {
-		panic(err)
-	}
-}
-
-var _ = Describe("eth_call", func() {
-	var (
-		blocks      []*types.Block
-		receipts    []types.Receipts
-		chain       *core.BlockChain
-		db          *postgres.DB
-		transformer *eth2.StateDiffTransformer
-		backend     *eth.Backend
-		api         *eth.PublicEthAPI
-		builder     statediff.Builder
-		pams        statediff.Params
-		chainConfig = params.TestChainConfig
-		mockTD      = big.NewInt(1337)
-	)
-
-	BeforeEach(func() {
-		// db and type initializations
-		var err error
-		db, err = shared.SetupDB()
-		Expect(err).ToNot(HaveOccurred())
-		transformer = eth2.NewStateDiffTransformer(chainConfig, db)
-		backend, err = eth.NewEthBackend(db, &eth.Config{
-			ChainConfig: chainConfig,
-			VmConfig:    vm.Config{},
-			RPCGasCap:   big.NewInt(10000000000),
-		})
-		Expect(err).ToNot(HaveOccurred())
-		api = eth.NewPublicEthAPI(backend)
-
-		// make the test blockchain (and state)
-		blocks, receipts, chain = test_helpers.MakeChain(4, test_helpers.Genesis, test_helpers.TestChainGen)
-		pams = statediff.Params{
-			IntermediateStateNodes:   true,
-			IntermediateStorageNodes: true,
-		}
-		// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
-		builder = statediff.NewBuilder(chain.StateCache())
-		for i, block := range blocks {
-			var args statediff.Args
-			var rcts types.Receipts
-			if i == 0 {
-				args = statediff.Args{
-					OldStateRoot: common.Hash{},
-					NewStateRoot: block.Root(),
-					BlockNumber:  block.Number(),
-					BlockHash:    block.Hash(),
-				}
-			} else {
-				args = statediff.Args{
-					OldStateRoot: blocks[i-1].Root(),
-					NewStateRoot: block.Root(),
-					BlockNumber:  block.Number(),
-					BlockHash:    block.Hash(),
-				}
-				rcts = receipts[i-1]
-			}
-			diff, err := builder.BuildStateDiffObject(args, pams)
-			Expect(err).ToNot(HaveOccurred())
-			diffRlp, err := rlp.EncodeToBytes(diff)
-			Expect(err).ToNot(HaveOccurred())
-			blockRlp, err := rlp.EncodeToBytes(block)
-			Expect(err).ToNot(HaveOccurred())
-			receiptsRlp, err := rlp.EncodeToBytes(rcts)
-			Expect(err).ToNot(HaveOccurred())
-			payload := statediff.Payload{
-				StateObjectRlp:  diffRlp,
-				BlockRlp:        blockRlp,
-				ReceiptsRlp:     receiptsRlp,
-				TotalDifficulty: mockTD,
-			}
-			_, err = transformer.Transform(0, payload)
-			Expect(err).ToNot(HaveOccurred())
-		}
-	})
-	AfterEach(func() {
-		eth.TearDownDB(db)
-		chain.Stop()
-	})
-	Describe("eth_call", func() {
-		It("Applies call args (tx data) on top of state, returning the result (e.g. a Getter method call)", func() {
-			data, err := parsedABI.Pack("data")
-			Expect(err).ToNot(HaveOccurred())
-			bdata := hexutil.Bytes(data)
-			callArgs := eth.CallArgs{
-				To:   &test_helpers.ContractAddr,
-				Data: &bdata,
-			}
-			res, err := api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(2), nil)
-			Expect(err).ToNot(HaveOccurred())
-			expectedRes := hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
-			Expect(res).To(Equal(expectedRes))
-
-			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(3), nil)
-			Expect(err).ToNot(HaveOccurred())
-			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003"))
-			Expect(res).To(Equal(expectedRes))
-
-			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(4), nil)
-			Expect(err).ToNot(HaveOccurred())
-			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009"))
-			Expect(res).To(Equal(expectedRes))
-		})
-	})
-})

pkg/eth/eth_state_test.go

@@ -0,0 +1,533 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"math/big"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth/test_helpers"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+	ethServerShared "github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+)
+
+var (
+	parsedABI abi.ABI
+)
+
+func init() {
+	// load abi
+	abiBytes, err := ioutil.ReadFile("./test_helpers/abi.json")
+	if err != nil {
+		panic(err)
+	}
+	parsedABI, err = abi.JSON(bytes.NewReader(abiBytes))
+	if err != nil {
+		panic(err)
+	}
+}
+
+var _ = Describe("eth state reading tests", func() {
+	const chainLength = 5
+	var (
+		blocks                  []*types.Block
+		receipts                []types.Receipts
+		chain                   *core.BlockChain
+		db                      *sqlx.DB
+		api                     *eth.PublicEthAPI
+		backend                 *eth.Backend
+		chainConfig             = params.TestChainConfig
+		mockTD                  = big.NewInt(1337)
+		expectedCanonicalHeader map[string]interface{}
+	)
+	It("test init", func() {
+		// db and type initializations
+		var err error
+		db = shared.SetupDB()
+		transformer := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
+
+		backend, err = eth.NewEthBackend(db, &eth.Config{
+			ChainConfig: chainConfig,
+			VMConfig:    vm.Config{},
+			RPCGasCap:   big.NewInt(10000000000), // Max gas capacity for a rpc call.
+			GroupCacheConfig: &ethServerShared.GroupCacheConfig{
+				StateDB: ethServerShared.GroupConfig{
+					Name:                   "eth_state_test",
+					CacheSizeInMB:          8,
+					CacheExpiryInMins:      60,
+					LogStatsIntervalInSecs: 0,
+				},
+			},
+		})
+		Expect(err).ToNot(HaveOccurred())
+		api, _ = eth.NewPublicEthAPI(backend, nil, false, false, false)
+
+		// make the test blockchain (and state)
+		blocks, receipts, chain = test_helpers.MakeChain(chainLength, test_helpers.Genesis, test_helpers.TestChainGen)
+		params := statediff.Params{
+			IntermediateStateNodes:   true,
+			IntermediateStorageNodes: true,
+		}
+		canonicalHeader := blocks[1].Header()
+		expectedCanonicalHeader = map[string]interface{}{
+			"number":           (*hexutil.Big)(canonicalHeader.Number),
+			"hash":             canonicalHeader.Hash(),
+			"parentHash":       canonicalHeader.ParentHash,
+			"nonce":            canonicalHeader.Nonce,
+			"mixHash":          canonicalHeader.MixDigest,
+			"sha3Uncles":       canonicalHeader.UncleHash,
+			"logsBloom":        canonicalHeader.Bloom,
+			"stateRoot":        canonicalHeader.Root,
+			"miner":            canonicalHeader.Coinbase,
+			"difficulty":       (*hexutil.Big)(canonicalHeader.Difficulty),
+			"extraData":        hexutil.Bytes([]byte{}),
+			"size":             hexutil.Uint64(canonicalHeader.Size()),
+			"gasLimit":         hexutil.Uint64(canonicalHeader.GasLimit),
+			"gasUsed":          hexutil.Uint64(canonicalHeader.GasUsed),
+			"timestamp":        hexutil.Uint64(canonicalHeader.Time),
+			"transactionsRoot": canonicalHeader.TxHash,
+			"receiptsRoot":     canonicalHeader.ReceiptHash,
+			"totalDifficulty":  (*hexutil.Big)(mockTD),
+		}
+		// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
+		builder := statediff.NewBuilder(chain.StateCache())
+		for i, block := range blocks {
+			var args statediff.Args
+			var rcts types.Receipts
+			if i == 0 {
+				args = statediff.Args{
+					OldStateRoot: common.Hash{},
+					NewStateRoot: block.Root(),
+					BlockNumber:  block.Number(),
+					BlockHash:    block.Hash(),
+				}
+			} else {
+				args = statediff.Args{
+					OldStateRoot: blocks[i-1].Root(),
+					NewStateRoot: block.Root(),
+					BlockNumber:  block.Number(),
+					BlockHash:    block.Hash(),
+				}
+				rcts = receipts[i-1]
+			}
+			diff, err := builder.BuildStateDiffObject(args, params)
+			Expect(err).ToNot(HaveOccurred())
+			tx, err := transformer.PushBlock(block, rcts, mockTD)
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range diff.Nodes {
+				err = transformer.PushStateNode(tx, node, block.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		// Insert some non-canonical data into the database so that we test our ability to discern canonicity
+		indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
+
+		tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+		Expect(err).ToNot(HaveOccurred())
+
+		err = tx.Submit(err)
+		Expect(err).ToNot(HaveOccurred())
+
+		// The non-canonical header has a child
+		tx, err = indexAndPublisher.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
+		Expect(err).ToNot(HaveOccurred())
+
+		hash := sdtypes.CodeAndCodeHash{
+			Hash: test_helpers.CodeHash,
+			Code: test_helpers.ContractCode,
+		}
+
+		err = indexAndPublisher.PushCodeAndCodeHash(tx, hash)
+		Expect(err).ToNot(HaveOccurred())
+
+		// wait for tx batch process to complete.
+		time.Sleep(10000 * time.Millisecond)
+		err = tx.Submit(err)
+		Expect(err).ToNot(HaveOccurred())
+	})
+	defer It("test teardown", func() {
+		shared.TearDownDB(db)
+		chain.Stop()
+	})
+
+	Describe("eth_call", func() {
+		It("Applies call args (tx data) on top of state, returning the result (e.g. a Getter method call)", func() {
+			data, err := parsedABI.Pack("data")
+			Expect(err).ToNot(HaveOccurred())
+			bdata := hexutil.Bytes(data)
+			callArgs := eth.CallArgs{
+				To:   &test_helpers.ContractAddr,
+				Data: &bdata,
+			}
+			// Before contract deployment, returns nil
+			res, err := api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(0), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(BeNil())
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(1), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(BeNil())
+
+			// After deployment
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(2), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes := hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(3), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(4), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(5), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000"))
+			Expect(res).To(Equal(expectedRes))
+		})
+	})
+
+	var (
+		expectedContractBalance   = (*hexutil.Big)(common.Big0)
+		expectedBankBalanceBlock0 = (*hexutil.Big)(test_helpers.TestBankFunds)
+
+		expectedAcct1BalanceBlock1 = (*hexutil.Big)(big.NewInt(10000))
+		expectedBankBalanceBlock1  = (*hexutil.Big)(new(big.Int).Sub(test_helpers.TestBankFunds, big.NewInt(10000)))
+
+		expectedAcct2BalanceBlock2 = (*hexutil.Big)(big.NewInt(1000))
+		expectedBankBalanceBlock2  = (*hexutil.Big)(new(big.Int).Sub(expectedBankBalanceBlock1.ToInt(), big.NewInt(1000)))
+
+		expectedAcct2BalanceBlock3 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock2.ToInt(), test_helpers.MiningReward))
+
+		expectedAcct2BalanceBlock4 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock3.ToInt(), test_helpers.MiningReward))
+
+		expectedAcct1BalanceBlock5 = (*hexutil.Big)(new(big.Int).Add(expectedAcct1BalanceBlock1.ToInt(), test_helpers.MiningReward))
+	)
+
+	Describe("eth_getBalance", func() {
+		It("Retrieves the eth balance for the provided account address at the block with the provided number", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock3))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock5))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+		})
+		It("Retrieves the eth balance for the provided account address at the block with the provided hash", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			_, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock2))
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock3))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock5))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+		})
+		It("Returns `0` for an account it cannot find the balance for an account at the provided block number", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+		})
+		It("Returns `0` for an error for an account it cannot find the balance for an account at the provided block hash", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+		})
+	})
+
+	Describe("eth_getCode", func() {
+		It("Retrieves the code for the provided contract address at the block with the provided number", func() {
+			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+
+			code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+		})
+		It("Retrieves the code for the provided contract address at the block with the provided hash", func() {
+			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+
+			code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+		})
+		It("Returns `nil` for an account it cannot find the code for", func() {
+			code, err := api.GetCode(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(BeEmpty())
+		})
+		It("Returns `nil`  for a contract that doesn't exist at this height", func() {
+			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(BeEmpty())
+		})
+	})
+
+	Describe("eth_getStorageAt", func() {
+		It("Returns empty slice if it tries to access a contract which does not exist", func() {
+			storage, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+
+			storage, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+		})
+		It("Returns empty slice if it tries to access a contract slot which does not exist", func() {
+			storage, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, randomHash.Hex(), rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+		})
+		It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() {
+			// After deployment
+			val, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes := hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+		})
+		It("Throws an error for a non-existing block hash", func() {
+			_, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithHash(randomHash, true))
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError("header for hash not found"))
+		})
+		It("Throws an error for a non-existing block number", func() {
+			_, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(chainLength+1))
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError("header not found"))
+		})
+	})
+
+	Describe("eth_getHeaderByNumber", func() {
+		It("Finds the canonical header based on the header's weight relative to others at the provided height", func() {
+			header, err := api.GetHeaderByNumber(ctx, number)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(header).To(Equal(expectedCanonicalHeader))
+		})
+	})
+})

pkg/eth/eth_suite_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func TestETHWatcher(t *testing.T) {
+func TestETHSuite(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "eth ipld server eth suite test")
 }

pkg/eth/filterer.go

@@ -23,17 +23,16 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
 )
 
 // Filterer interface for substituing mocks in tests
 type Filterer interface {
-	Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error)
+	Filter(filter SubscriptionSettings, payload ConvertedPayload) (*IPLDs, error)
 }
 
 // ResponseFilterer satisfies the ResponseFilterer interface for ethereum
@@ -45,7 +44,7 @@ func NewResponseFilterer() *ResponseFilterer {
 }
 
 // Filter is used to filter through eth data to extract and package requested data into a Payload
-func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error) {
+func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload ConvertedPayload) (*IPLDs, error) {
 	if checkRange(filter.Start.Int64(), filter.End.Int64(), payload.Block.Number().Int64()) {
 		response := new(IPLDs)
 		response.TotalDifficulty = payload.TotalDifficulty
@@ -72,7 +71,7 @@ func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.Conve
 	return nil, nil
 }
 
-func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IPLDs, payload eth.ConvertedPayload) error {
+func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IPLDs, payload ConvertedPayload) error {
 	if !headerFilter.Off {
 		headerRLP, err := rlp.EncodeToBytes(payload.Block.Header())
 		if err != nil {
@@ -82,12 +81,12 @@ func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IP
 		if err != nil {
 			return err
 		}
-		response.Header = ipfs.BlockModel{
+		response.Header = models.IPLDModel{
 			Data: headerRLP,
-			CID:  cid.String(),
+			Key:  cid.String(),
 		}
 		if headerFilter.Uncles {
-			response.Uncles = make([]ipfs.BlockModel, len(payload.Block.Body().Uncles))
+			response.Uncles = make([]models.IPLDModel, len(payload.Block.Body().Uncles))
 			for i, uncle := range payload.Block.Body().Uncles {
 				uncleRlp, err := rlp.EncodeToBytes(uncle)
 				if err != nil {
@@ -97,9 +96,9 @@ func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IP
 				if err != nil {
 					return err
 				}
-				response.Uncles[i] = ipfs.BlockModel{
+				response.Uncles[i] = models.IPLDModel{
 					Data: uncleRlp,
-					CID:  cid.String(),
+					Key:  cid.String(),
 				}
 			}
 		}
@@ -114,12 +113,12 @@ func checkRange(start, end, actual int64) bool {
 	return false
 }
 
-func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload eth.ConvertedPayload) ([]common.Hash, error) {
+func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload ConvertedPayload) ([]common.Hash, error) {
 	var trxHashes []common.Hash
 	if !trxFilter.Off {
 		trxLen := len(payload.Block.Body().Transactions)
 		trxHashes = make([]common.Hash, 0, trxLen)
-		response.Transactions = make([]ipfs.BlockModel, 0, trxLen)
+		response.Transactions = make([]models.IPLDModel, 0, trxLen)
 		for i, trx := range payload.Block.Body().Transactions {
 			// TODO: check if want corresponding receipt and if we do we must include this transaction
 			if checkTransactionAddrs(trxFilter.Src, trxFilter.Dst, payload.TxMetaData[i].Src, payload.TxMetaData[i].Dst) {
@@ -132,9 +131,9 @@ func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLD
 				if err != nil {
 					return nil, err
 				}
-				response.Transactions = append(response.Transactions, ipfs.BlockModel{
+				response.Transactions = append(response.Transactions, models.IPLDModel{
 					Data: data,
-					CID:  cid.String(),
+					Key:  cid.String(),
 				})
 				trxHashes = append(trxHashes, trx.Hash())
 			}
@@ -162,25 +161,30 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s
 	return false
 }
 
-func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload eth.ConvertedPayload, trxHashes []common.Hash) error {
+func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload ConvertedPayload, trxHashes []common.Hash) error {
 	if !receiptFilter.Off {
-		response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
+		response.Receipts = make([]models.IPLDModel, 0, len(payload.Receipts))
-		for i, receipt := range payload.Receipts {
+		rctLeafCID, rctIPLDData, err := GetRctLeafNodeData(payload.Receipts)
-			// topics is always length 4
-			topics := [][]string{payload.ReceiptMetaData[i].Topic0s, payload.ReceiptMetaData[i].Topic1s, payload.ReceiptMetaData[i].Topic2s, payload.ReceiptMetaData[i].Topic3s}
-			if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, payload.ReceiptMetaData[i].LogContracts, trxHashes) {
-				receiptBuffer := new(bytes.Buffer)
-				if err := receipt.EncodeRLP(receiptBuffer); err != nil {
-					return err
-				}
-				data := receiptBuffer.Bytes()
-				cid, err := ipld.RawdataToCid(ipld.MEthTxReceipt, data, multihash.KECCAK_256)
 		if err != nil {
 			return err
 		}
-				response.Receipts = append(response.Receipts, ipfs.BlockModel{
+
-					Data: data,
+		for idx, receipt := range payload.Receipts {
-					CID:  cid.String(),
+			// topics is always length 4
+			topics := make([][]string, 4)
+			contracts := make([]string, len(receipt.Logs))
+			for _, l := range receipt.Logs {
+				contracts = append(contracts, l.Address.String())
+				for idx, t := range l.Topics {
+					topics[idx] = append(topics[idx], t.String())
+				}
+			}
+
+			// TODO: Verify this filter logic.
+			if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, contracts, trxHashes) {
+				response.Receipts = append(response.Receipts, models.IPLDModel{
+					Data: rctIPLDData[idx],
+					Key:  rctLeafCID[idx].String(),
 				})
 			}
 		}
@@ -252,7 +256,7 @@ func slicesShareString(slice1, slice2 []string) int {
 }
 
 // filterStateAndStorage filters state and storage nodes into the response according to the provided filters
-func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload eth.ConvertedPayload) error {
+func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload ConvertedPayload) error {
 	response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
 	response.StorageNodes = make([]StorageNode, 0)
 	stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses))
@@ -269,37 +273,37 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
 	}
 	for _, stateNode := range payload.StateNodes {
 		if !stateFilter.Off && checkNodeKeys(stateAddressFilters, stateNode.LeafKey) {
-			if stateNode.Type == statediff.Leaf || stateFilter.IntermediateNodes {
+			if stateNode.NodeType == sdtypes.Leaf || stateFilter.IntermediateNodes {
-				cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
+				cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.NodeValue, multihash.KECCAK_256)
 				if err != nil {
 					return err
 				}
 				response.StateNodes = append(response.StateNodes, StateNode{
-					StateLeafKey: stateNode.LeafKey,
+					StateLeafKey: common.BytesToHash(stateNode.LeafKey),
 					Path:         stateNode.Path,
-					IPLD: ipfs.BlockModel{
+					IPLD: models.IPLDModel{
-						Data: stateNode.Value,
+						Data: stateNode.NodeValue,
-						CID:  cid.String(),
+						Key:  cid.String(),
 					},
-					Type: stateNode.Type,
+					Type: stateNode.NodeType,
 				})
 			}
 		}
 		if !storageFilter.Off && checkNodeKeys(storageAddressFilters, stateNode.LeafKey) {
 			for _, storageNode := range payload.StorageNodes[common.Bytes2Hex(stateNode.Path)] {
 				if checkNodeKeys(storageKeyFilters, storageNode.LeafKey) {
-					cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
+					cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.NodeValue, multihash.KECCAK_256)
 					if err != nil {
 						return err
 					}
 					response.StorageNodes = append(response.StorageNodes, StorageNode{
-						StateLeafKey:   stateNode.LeafKey,
+						StateLeafKey:   common.BytesToHash(stateNode.LeafKey),
-						StorageLeafKey: storageNode.LeafKey,
+						StorageLeafKey: common.BytesToHash(storageNode.LeafKey),
-						IPLD: ipfs.BlockModel{
+						IPLD: models.IPLDModel{
-							Data: storageNode.Value,
+							Data: storageNode.NodeValue,
-							CID:  cid.String(),
+							Key:  cid.String(),
 						},
-						Type: storageNode.Type,
+						Type: storageNode.NodeType,
 						Path: storageNode.Path,
 					})
 				}
@@ -309,15 +313,52 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
 	return nil
 }
 
-func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
+func checkNodeKeys(wantedKeys []common.Hash, actualKey []byte) bool {
 	// If we aren't filtering for any specific keys, all nodes are a go
 	if len(wantedKeys) == 0 {
 		return true
 	}
 	for _, key := range wantedKeys {
-		if bytes.Equal(key.Bytes(), actualKey.Bytes()) {
+		if bytes.Equal(key.Bytes(), actualKey) {
 			return true
 		}
 	}
 	return false
 }
+
+// GetRctLeafNodeData converts the receipts to receipt trie and returns the receipt leaf node IPLD data and
+// corresponding CIDs
+func GetRctLeafNodeData(rcts types.Receipts) ([]cid.Cid, [][]byte, error) {
+	receiptTrie := ipld.NewRctTrie()
+	for idx, rct := range rcts {
+		ethRct, err := ipld.NewReceipt(rct)
+		if err != nil {
+			return nil, nil, err
+		}
+		if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	rctLeafNodes, keys, err := receiptTrie.GetLeafNodes()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
+	ethRctleafNodeData := make([][]byte, len(rctLeafNodes))
+	for i, rln := range rctLeafNodes {
+		var idx uint
+
+		r := bytes.NewReader(keys[i].TrieKey)
+		err = rlp.Decode(r, &idx)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		ethRctleafNodeCids[idx] = rln.Cid()
+		ethRctleafNodeData[idx] = rln.RawData()
+	}
+
+	return ethRctleafNodeCids, ethRctleafNodeData, nil
+}

pkg/eth/filterer_test.go

@@ -19,15 +19,15 @@ package eth_test
 import (
 	"bytes"
 
-	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
 var (
@@ -46,29 +46,29 @@ var _ = Describe("Filterer", func() {
 			Expect(iplds).ToNot(BeNil())
 			Expect(iplds.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds.Header).To(Equal(test_helpers.MockIPLDs.Header))
-			var expectedEmptyUncles []ipfs.BlockModel
+			var expectedEmptyUncles []models.IPLDModel
 			Expect(iplds.Uncles).To(Equal(expectedEmptyUncles))
-			Expect(len(iplds.Transactions)).To(Equal(3))
+			Expect(len(iplds.Transactions)).To(Equal(4))
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx1)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx2)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.MockTransactions.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.Tx3)).To(BeTrue())
-			Expect(len(iplds.Receipts)).To(Equal(3))
+			Expect(len(iplds.Receipts)).To(Equal(4))
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.MockReceipts.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct1IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.MockReceipts.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct2IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.MockReceipts.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.Rct3IPLD)).To(BeTrue())
 			Expect(len(iplds.StateNodes)).To(Equal(2))
 			for _, stateNode := range iplds.StateNodes {
-				Expect(stateNode.Type).To(Equal(statediff.Leaf))
+				Expect(stateNode.Type).To(Equal(sdtypes.Leaf))
 				if bytes.Equal(stateNode.StateLeafKey.Bytes(), test_helpers.AccountLeafKey) {
-					Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
+					Expect(stateNode.IPLD).To(Equal(models.IPLDModel{
 						Data: test_helpers.State2IPLD.RawData(),
-						CID:  test_helpers.State2IPLD.Cid().String(),
+						Key:  test_helpers.State2IPLD.Cid().String(),
 					}))
 				}
 				if bytes.Equal(stateNode.StateLeafKey.Bytes(), test_helpers.ContractLeafKey) {
-					Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
+					Expect(stateNode.IPLD).To(Equal(models.IPLDModel{
 						Data: test_helpers.State1IPLD.RawData(),
-						CID:  test_helpers.State1IPLD.Cid().String(),
+						Key:  test_helpers.State1IPLD.Cid().String(),
 					}))
 				}
 			}
@@ -80,116 +80,116 @@ var _ = Describe("Filterer", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds1).ToNot(BeNil())
 			Expect(iplds1.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds1.Header).To(Equal(ipfs.BlockModel{}))
+			Expect(iplds1.Header).To(Equal(models.IPLDModel{}))
 			Expect(len(iplds1.Uncles)).To(Equal(0))
 			Expect(len(iplds1.Transactions)).To(Equal(0))
 			Expect(len(iplds1.StorageNodes)).To(Equal(0))
 			Expect(len(iplds1.StateNodes)).To(Equal(0))
 			Expect(len(iplds1.Receipts)).To(Equal(1))
-			Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{
+			Expect(iplds1.Receipts[0]).To(Equal(models.IPLDModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
+				Data: test_helpers.Rct1IPLD,
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
+				Key:  test_helpers.Rct1CID.String(),
 			}))
 
 			iplds2, err := filterer.Filter(rctTopicsFilter, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds2).ToNot(BeNil())
 			Expect(iplds2.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds2.Header).To(Equal(ipfs.BlockModel{}))
+			Expect(iplds2.Header).To(Equal(models.IPLDModel{}))
 			Expect(len(iplds2.Uncles)).To(Equal(0))
 			Expect(len(iplds2.Transactions)).To(Equal(0))
 			Expect(len(iplds2.StorageNodes)).To(Equal(0))
 			Expect(len(iplds2.StateNodes)).To(Equal(0))
 			Expect(len(iplds2.Receipts)).To(Equal(1))
-			Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{
+			Expect(iplds2.Receipts[0]).To(Equal(models.IPLDModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
+				Data: test_helpers.Rct1IPLD,
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
+				Key:  test_helpers.Rct1CID.String(),
 			}))
 
 			iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds3).ToNot(BeNil())
 			Expect(iplds3.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds3.Header).To(Equal(ipfs.BlockModel{}))
+			Expect(iplds3.Header).To(Equal(models.IPLDModel{}))
 			Expect(len(iplds3.Uncles)).To(Equal(0))
 			Expect(len(iplds3.Transactions)).To(Equal(0))
 			Expect(len(iplds3.StorageNodes)).To(Equal(0))
 			Expect(len(iplds3.StateNodes)).To(Equal(0))
 			Expect(len(iplds3.Receipts)).To(Equal(1))
-			Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{
+			Expect(iplds3.Receipts[0]).To(Equal(models.IPLDModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
+				Data: test_helpers.Rct1IPLD,
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
+				Key:  test_helpers.Rct1CID.String(),
 			}))
 
 			iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds4).ToNot(BeNil())
 			Expect(iplds4.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds4.Header).To(Equal(ipfs.BlockModel{}))
+			Expect(iplds4.Header).To(Equal(models.IPLDModel{}))
 			Expect(len(iplds4.Uncles)).To(Equal(0))
 			Expect(len(iplds4.Transactions)).To(Equal(0))
 			Expect(len(iplds4.StorageNodes)).To(Equal(0))
 			Expect(len(iplds4.StateNodes)).To(Equal(0))
 			Expect(len(iplds4.Receipts)).To(Equal(1))
-			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
+			Expect(iplds4.Receipts[0]).To(Equal(models.IPLDModel{
-				Data: test_helpers.Rct2IPLD.RawData(),
+				Data: test_helpers.Rct2IPLD,
-				CID:  test_helpers.Rct2IPLD.Cid().String(),
+				Key:  test_helpers.Rct2CID.String(),
 			}))
 
 			iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds5).ToNot(BeNil())
 			Expect(iplds5.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds5.Header).To(Equal(ipfs.BlockModel{}))
+			Expect(iplds5.Header).To(Equal(models.IPLDModel{}))
 			Expect(len(iplds5.Uncles)).To(Equal(0))
-			Expect(len(iplds5.Transactions)).To(Equal(3))
+			Expect(len(iplds5.Transactions)).To(Equal(4))
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx1)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx2)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx3)).To(BeTrue())
 			Expect(len(iplds5.StorageNodes)).To(Equal(0))
 			Expect(len(iplds5.StateNodes)).To(Equal(0))
-			Expect(len(iplds5.Receipts)).To(Equal(3))
+			Expect(len(iplds5.Receipts)).To(Equal(4))
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.MockReceipts.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct1IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.MockReceipts.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct2IPLD)).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.MockReceipts.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.Rct3IPLD)).To(BeTrue())
 
 			iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds6).ToNot(BeNil())
 			Expect(iplds6.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds6.Header).To(Equal(ipfs.BlockModel{}))
+			Expect(iplds6.Header).To(Equal(models.IPLDModel{}))
 			Expect(len(iplds6.Uncles)).To(Equal(0))
 			Expect(len(iplds6.Transactions)).To(Equal(1))
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.Tx2)).To(BeTrue())
 			Expect(len(iplds6.StorageNodes)).To(Equal(0))
 			Expect(len(iplds6.StateNodes)).To(Equal(0))
 			Expect(len(iplds6.Receipts)).To(Equal(1))
-			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
+			Expect(iplds4.Receipts[0]).To(Equal(models.IPLDModel{
-				Data: test_helpers.Rct2IPLD.RawData(),
+				Data: test_helpers.Rct2IPLD,
-				CID:  test_helpers.Rct2IPLD.Cid().String(),
+				Key:  test_helpers.Rct2CID.String(),
 			}))
 
 			iplds7, err := filterer.Filter(stateFilter, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds7).ToNot(BeNil())
 			Expect(iplds7.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds7.Header).To(Equal(ipfs.BlockModel{}))
+			Expect(iplds7.Header).To(Equal(models.IPLDModel{}))
 			Expect(len(iplds7.Uncles)).To(Equal(0))
 			Expect(len(iplds7.Transactions)).To(Equal(0))
 			Expect(len(iplds7.StorageNodes)).To(Equal(0))
 			Expect(len(iplds7.Receipts)).To(Equal(0))
 			Expect(len(iplds7.StateNodes)).To(Equal(1))
 			Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(test_helpers.AccountLeafKey))
-			Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
+			Expect(iplds7.StateNodes[0].IPLD).To(Equal(models.IPLDModel{
 				Data: test_helpers.State2IPLD.RawData(),
-				CID:  test_helpers.State2IPLD.Cid().String(),
+				Key:  test_helpers.State2IPLD.Cid().String(),
 			}))
 
 			iplds8, err := filterer.Filter(rctTopicsAndAddressFilterFail, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds8).ToNot(BeNil())
 			Expect(iplds8.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds8.Header).To(Equal(ipfs.BlockModel{}))
+			Expect(iplds8.Header).To(Equal(models.IPLDModel{}))
 			Expect(len(iplds8.Uncles)).To(Equal(0))
 			Expect(len(iplds8.Transactions)).To(Equal(0))
 			Expect(len(iplds8.StorageNodes)).To(Equal(0))

pkg/eth/helpers.go

@@ -17,39 +17,20 @@
 package eth
 
 import (
-	"fmt"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
-
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/statediff"
 )
 
-func ResolveToNodeType(nodeType int) statediff.NodeType {
+func ResolveToNodeType(nodeType int) sdtypes.NodeType {
 	switch nodeType {
 	case 0:
-		return statediff.Branch
+		return sdtypes.Branch
 	case 1:
-		return statediff.Extension
+		return sdtypes.Extension
 	case 2:
-		return statediff.Leaf
+		return sdtypes.Leaf
 	case 3:
-		return statediff.Removed
+		return sdtypes.Removed
 	default:
-		return statediff.Unknown
+		return sdtypes.Unknown
-	}
-}
-
-// ChainConfig returns the appropriate ethereum chain config for the provided chain id
-func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
-	switch chainID {
-	case 1:
-		return params.MainnetChainConfig, nil
-	case 3:
-		return params.TestnetChainConfig, nil // Ropsten
-	case 4:
-		return params.RinkebyChainConfig, nil
-	case 5:
-		return params.GoerliChainConfig, nil
-	default:
-		return nil, fmt.Errorf("chain config for chainid %d not available", chainID)
 	}
 }

pkg/eth/interfaces.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/bloombits"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// FilterBackend is the geth interface we need to satisfy to use their filters
+type FilterBackend interface {
+	ChainDb() ethdb.Database
+	HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error)
+	HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error)
+	GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
+	GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error)
+
+	SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
+	SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
+	SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
+	SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
+	SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
+
+	BloomStatus() (uint64, uint64)
+	ServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
+}

pkg/eth/ipld_fetcher.go

@@ -22,14 +22,10 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
 	"github.com/jmoiron/sqlx"
 	log "github.com/sirupsen/logrus"
-
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
 // Fetcher interface for substituting mocks in tests
@@ -40,11 +36,11 @@ type Fetcher interface {
 // IPLDFetcher satisfies the IPLDFetcher interface for ethereum
 // It interfaces directly with PG-IPFS
 type IPLDFetcher struct {
-	db *postgres.DB
+	db *sqlx.DB
 }
 
 // NewIPLDFetcher creates a pointer to a new IPLDFetcher
-func NewIPLDFetcher(db *postgres.DB) *IPLDFetcher {
+func NewIPLDFetcher(db *sqlx.DB) *IPLDFetcher {
 	return &IPLDFetcher{
 		db: db,
 	}
@@ -103,72 +99,73 @@ func (f *IPLDFetcher) Fetch(cids CIDWrapper) (*IPLDs, error) {
 	return iplds, err
 }
 
-// FetchHeaders fetches headers
+// FetchHeader fetches header
-func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c eth.HeaderModel) (ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c models.HeaderModel) (models.IPLDModel, error) {
 	log.Debug("fetching header ipld")
 	headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
 	if err != nil {
-		return ipfs.BlockModel{}, err
+		return models.IPLDModel{}, err
 	}
-	return ipfs.BlockModel{
+	return models.IPLDModel{
 		Data: headerBytes,
-		CID:  c.CID,
+		Key:  c.CID,
 	}, nil
 }
 
 // FetchUncles fetches uncles
-func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []eth.UncleModel) ([]ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []models.UncleModel) ([]models.IPLDModel, error) {
 	log.Debug("fetching uncle iplds")
-	uncleIPLDs := make([]ipfs.BlockModel, len(cids))
+	uncleIPLDs := make([]models.IPLDModel, len(cids))
 	for i, c := range cids {
 		uncleBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
 		if err != nil {
 			return nil, err
 		}
-		uncleIPLDs[i] = ipfs.BlockModel{
+		uncleIPLDs[i] = models.IPLDModel{
 			Data: uncleBytes,
-			CID:  c.CID,
+			Key:  c.CID,
 		}
 	}
 	return uncleIPLDs, nil
 }
 
 // FetchTrxs fetches transactions
-func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []eth.TxModel) ([]ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []models.TxModel) ([]models.IPLDModel, error) {
 	log.Debug("fetching transaction iplds")
-	trxIPLDs := make([]ipfs.BlockModel, len(cids))
+	trxIPLDs := make([]models.IPLDModel, len(cids))
 	for i, c := range cids {
 		txBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
 		if err != nil {
 			return nil, err
 		}
-		trxIPLDs[i] = ipfs.BlockModel{
+		trxIPLDs[i] = models.IPLDModel{
 			Data: txBytes,
-			CID:  c.CID,
+			Key:  c.CID,
 		}
 	}
 	return trxIPLDs, nil
 }
 
 // FetchRcts fetches receipts
-func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []eth.ReceiptModel) ([]ipfs.BlockModel, error) {
+func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []models.ReceiptModel) ([]models.IPLDModel, error) {
 	log.Debug("fetching receipt iplds")
-	rctIPLDs := make([]ipfs.BlockModel, len(cids))
+	rctIPLDs := make([]models.IPLDModel, len(cids))
 	for i, c := range cids {
-		rctBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
+		rctBytes, err := shared.FetchIPLDByMhKey(tx, c.LeafMhKey)
 		if err != nil {
 			return nil, err
 		}
-		rctIPLDs[i] = ipfs.BlockModel{
+		//nodeVal, err := DecodeLeafNode(rctBytes)
+		rctIPLDs[i] = models.IPLDModel{
 			Data: rctBytes,
-			CID:  c.CID,
+			Key:  c.LeafCID,
 		}
 	}
 	return rctIPLDs, nil
 }
 
 // FetchState fetches state nodes
-func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]StateNode, error) {
+func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []models.StateNodeModel) ([]StateNode, error) {
 	log.Debug("fetching state iplds")
 	stateNodes := make([]StateNode, 0, len(cids))
 	for _, stateNode := range cids {
@@ -180,9 +177,9 @@ func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]Stat
 			return nil, err
 		}
 		stateNodes = append(stateNodes, StateNode{
-			IPLD: ipfs.BlockModel{
+			IPLD: models.IPLDModel{
 				Data: stateBytes,
-				CID:  stateNode.CID,
+				Key:  stateNode.CID,
 			},
 			StateLeafKey: common.HexToHash(stateNode.StateKey),
 			Type:         ResolveToNodeType(stateNode.NodeType),
@@ -193,7 +190,7 @@ func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]Stat
 }
 
 // FetchStorage fetches storage nodes
-func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateKeyModel) ([]StorageNode, error) {
+func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []models.StorageNodeWithStateKeyModel) ([]StorageNode, error) {
 	log.Debug("fetching storage iplds")
 	storageNodes := make([]StorageNode, 0, len(cids))
 	for _, storageNode := range cids {
@@ -205,9 +202,9 @@ func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateK
 			return nil, err
 		}
 		storageNodes = append(storageNodes, StorageNode{
-			IPLD: ipfs.BlockModel{
+			IPLD: models.IPLDModel{
 				Data: storageBytes,
-				CID:  storageNode.CID,
+				Key:  storageNode.CID,
 			},
 			StateLeafKey:   common.HexToHash(storageNode.StateKey),
 			StorageLeafKey: common.HexToHash(storageNode.StorageKey),

pkg/eth/ipld_fetcher_test.go

@@ -17,35 +17,45 @@
 package eth_test
 
 import (
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/jmoiron/sqlx"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth/test_helpers"
-
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
 var _ = Describe("IPLDFetcher", func() {
 	var (
-		db            *postgres.DB
+		db            *sqlx.DB
-		pubAndIndexer *eth2.IPLDPublisher
+		pubAndIndexer interfaces.StateDiffIndexer
 		fetcher       *eth.IPLDFetcher
 	)
 	Describe("Fetch", func() {
 		BeforeEach(func() {
-			var err error
+			var (
-			db, err = shared.SetupDB()
+				err error
+				tx  interfaces.Batch
+			)
+			db = shared.SetupDB()
+			pubAndIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash())
+
+			tx, err = pubAndIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			for _, node := range test_helpers.MockStateNodes {
+				err = pubAndIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
 				Expect(err).ToNot(HaveOccurred())
-			pubAndIndexer = eth2.NewIPLDPublisher(db)
+			}
-			err = pubAndIndexer.Publish(test_helpers.MockConvertedPayload)
+
+			err = tx.Submit(err)
 			Expect(err).ToNot(HaveOccurred())
 			fetcher = eth.NewIPLDFetcher(db)
+
 		})
 		AfterEach(func() {
-			eth.TearDownDB(db)
+			shared.TearDownDB(db)
 		})
 
 		It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {

pkg/eth/ipld_retriever.go

@@ -19,91 +19,164 @@ package eth
 import (
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/statediff/trie_helpers"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/jmoiron/sqlx"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/lib/pq"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 )
 
 const (
-	RetrieveHeadersByHashesPgStr = `SELECT cid, data FROM eth.header_cids 
+	// node type removed value.
+	// https://github.com/vulcanize/go-ethereum/blob/271f4d01e7e2767ffd8e0cd469bf545be96f2a84/statediff/indexer/helpers.go#L34
+	removedNode = 3
+
+	RetrieveHeadersByHashesPgStr = `SELECT cid, data
+								FROM eth.header_cids
 									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
 								WHERE block_hash = ANY($1::VARCHAR(66)[])`
-	RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data FROM eth.header_cids 
+	RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data
+								FROM eth.header_cids
 									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
 								WHERE block_number = $1`
-	RetrieveHeaderByHashPgStr = `SELECT cid, data FROM eth.header_cids 
+	RetrieveHeaderByHashPgStr = `SELECT cid, data
+								FROM eth.header_cids
 									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
 								WHERE block_hash = $1`
-	RetrieveUnclesByHashesPgStr = `SELECT cid, data FROM eth.uncle_cids
+	RetrieveUnclesByHashesPgStr = `SELECT cid, data
+								FROM eth.uncle_cids
 									INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
 								WHERE block_hash = ANY($1::VARCHAR(66)[])`
-	RetrieveUnclesByBlockHashPgStr = `SELECT cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks
+	RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data
-										WHERE uncle_cids.header_id = header_cids.id
+										FROM eth.uncle_cids
-										AND uncle_cids.mh_key = blocks.key
+											INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.block_hash)
-										AND block_hash = $1`
-	RetrieveUnclesByBlockNumberPgStr = `SELECT cid, data FROM eth.uncle_cids, eth.header_cids, public.blocks
-										WHERE uncle_cids.header_id = header_cids.id
-										AND uncle_cids.mh_key = blocks.key
-										AND block_number = $1`
-	RetrieveUncleByHashPgStr = `SELECT cid, data FROM eth.uncle_cids
 											INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
 										WHERE block_hash = $1`
-	RetrieveTransactionsByHashesPgStr = `SELECT cid, data FROM eth.transaction_cids
+	RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data
+										FROM eth.uncle_cids
+											INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.block_hash)
+											INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
+										WHERE block_number = $1`
+	RetrieveUncleByHashPgStr = `SELECT cid, data
+								FROM eth.uncle_cids
+									INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
+								WHERE block_hash = $1`
+	RetrieveTransactionsByHashesPgStr = `SELECT cid, data
+									FROM eth.transaction_cids
 										INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
 									WHERE tx_hash = ANY($1::VARCHAR(66)[])`
-	RetrieveTransactionsByBlockHashPgStr = `SELECT cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks
+	RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data
-											WHERE transaction_cids.header_id = header_cids.id
+											FROM eth.transaction_cids
-											AND transaction_cids.mh_key = blocks.key
+												INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash)
-											AND block_hash = $1`
+												INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
-	RetrieveTransactionsByBlockNumberPgStr = `SELECT cid, data FROM eth.transaction_cids, eth.header_cids, public.blocks
+											WHERE block_hash = $1
-											WHERE transaction_cids.header_id = header_cids.id
+											ORDER BY eth.transaction_cids.index ASC`
-											AND transaction_cids.mh_key = blocks.key
+	RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data
-											AND block_number = $1`
+											FROM eth.transaction_cids
-	RetrieveTransactionByHashPgStr = `SELECT cid, data FROM eth.transaction_cids
+												INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash)
+												INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
+											WHERE block_number = $1
+											ORDER BY eth.transaction_cids.index ASC`
+	RetrieveTransactionByHashPgStr = `SELECT cid, data
+									FROM eth.transaction_cids
 										INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
 									WHERE tx_hash = $1`
-	RetrieveReceiptsByTxHashesPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, public.blocks
+	RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.leaf_cid, data
-									WHERE receipt_cids.mh_key = blocks.key
+									FROM eth.receipt_cids
-									AND receipt_cids.tx_id = transaction_cids.id
+										INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash)
-									AND tx_hash = ANY($1::VARCHAR(66)[])`
+										INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
-	RetrieveReceiptsByBlockHashPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks
+									WHERE tx_hash = ANY($1::VARCHAR(66)[])`
-										WHERE receipt_cids.tx_id = transaction_cids.id
+	RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.leaf_cid, data, eth.transaction_cids.tx_hash
-										AND transaction_cids.header_id = header_cids.id
+										FROM eth.receipt_cids
-										AND receipt_cids.mh_key = blocks.key
+											INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash)
-										AND block_hash = $1`
+											INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash)
-	RetrieveReceiptsByBlockNumberPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids, public.blocks
+											INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
-										WHERE receipt_cids.tx_id = transaction_cids.id
+										WHERE block_hash = $1
-										AND transaction_cids.header_id = header_cids.id
+										ORDER BY eth.transaction_cids.index ASC`
-										AND receipt_cids.mh_key = blocks.key
+	RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.leaf_cid, data
-										AND block_number = $1`
+										FROM eth.receipt_cids
-	RetrieveReceiptByTxHashPgStr = `SELECT cid, data FROM eth.receipt_cids, eth.transaction_cids, eth.receipt_cids
+											INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash)
-									WHERE receipt_cids.mh_key = blocks.key
+											INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.block_hash)
-									AND receipt_cids.tx_id = transaction_cids.id
+											INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
-									AND tx_hash = $1`
+										WHERE block_number = $1
-	RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT cid, data FROM eth.state_cids, eth.header_cids, public.blocks
+										ORDER BY eth.transaction_cids.index ASC`
-												WHERE state_cids.header_id = header_cids.id
+	RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.leaf_cid, data
-												AND state_cids.mh_key = blocks.key
+									FROM eth.receipt_cids
-												AND state_leaf_key = $1
+										INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.tx_hash)
-												AND block_hash = $2`
+										INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = blocks.key)
-	RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT cid, data FROM eth.state_cids, eth.header_cids, public.blocks
+									WHERE tx_hash = $1`
-												WHERE state_cids.header_id = header_cids.id
+	RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid, data, state_cids.node_type
-												AND state_cids.mh_key = blocks.key
+												FROM eth.state_cids
-												AND state_leaf_key = $1
+													INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
-												AND block_number = $2`
+													INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key)
+												WHERE state_leaf_key = $1
+												AND block_number <= (SELECT block_number
+																	FROM eth.header_cids
+																	WHERE block_hash = $2)
+												AND header_cids.block_hash = (SELECT canonical_header_hash(block_number))
+												ORDER BY block_number DESC
+												LIMIT 1`
+	RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid, data, state_cids.node_type
+													FROM eth.state_cids
+														INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
+														INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key)
+													WHERE state_leaf_key = $1
+													AND block_number <= $2
+													ORDER BY block_number DESC
+													LIMIT 1`
+	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed($1, $3) AS state_leaf_removed
+																	FROM eth.storage_cids
+																		INNER JOIN eth.state_cids ON (
+																			storage_cids.header_id = state_cids.header_id
+																			AND storage_cids.state_path = state_cids.state_path
+																		)
+																		INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
+																		INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key)
+																	WHERE state_leaf_key = $1
+																	AND storage_leaf_key = $2
+																	AND block_number <= $3
+																	ORDER BY block_number DESC
+																	LIMIT 1`
+	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid, data, storage_cids.node_type, was_state_leaf_removed($1, $3) AS state_leaf_removed
+																	FROM eth.storage_cids
+																		INNER JOIN eth.state_cids ON (
+																			storage_cids.header_id = state_cids.header_id
+																			AND storage_cids.state_path = state_cids.state_path
+																		)
+																		INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
+																		INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key)
+																	WHERE state_leaf_key = $1
+																	AND storage_leaf_key = $2
+																	AND block_number <= (SELECT block_number
+																						FROM eth.header_cids
+																						WHERE block_hash = $3)
+																	AND header_cids.block_hash = (SELECT canonical_header_hash(block_number))
+																	ORDER BY block_number DESC
+																	LIMIT 1`
 )
 
-type ipldResult struct {
+var EmptyNodeValue = make([]byte, common.HashLength)
-	cid  string `db:"cid"`
+
-	data []byte `db:"data"`
+type rctIpldResult struct {
-}
+	LeafCID string `db:"leaf_cid"`
-type IPLDRetriever struct {
+	Data    []byte `db:"data"`
-	db *postgres.DB
+	TxHash  string `db:"tx_hash"`
 }
 
-func NewIPLDRetriever(db *postgres.DB) *IPLDRetriever {
+type ipldResult struct {
+	CID    string `db:"cid"`
+	Data   []byte `db:"data"`
+	TxHash string `db:"tx_hash"`
+}
+
+type IPLDRetriever struct {
+	db *sqlx.DB
+}
+
+func NewIPLDRetriever(db *sqlx.DB) *IPLDRetriever {
 	return &IPLDRetriever{
 		db: db,
 	}
@@ -122,8 +195,8 @@ func (r *IPLDRetriever) RetrieveHeadersByHashes(hashes []common.Hash) ([]string,
 	cids := make([]string, len(headerResults))
 	headers := make([][]byte, len(headerResults))
 	for i, res := range headerResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		headers[i] = res.data
+		headers[i] = res.Data
 	}
 	return cids, headers, nil
 }
@@ -138,8 +211,8 @@ func (r *IPLDRetriever) RetrieveHeadersByBlockNumber(number uint64) ([]string, [
 	cids := make([]string, len(headerResults))
 	headers := make([][]byte, len(headerResults))
 	for i, res := range headerResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		headers[i] = res.data
+		headers[i] = res.Data
 	}
 	return cids, headers, nil
 }
@@ -147,7 +220,7 @@ func (r *IPLDRetriever) RetrieveHeadersByBlockNumber(number uint64) ([]string, [
 // RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash
 func (r *IPLDRetriever) RetrieveHeaderByHash(hash common.Hash) (string, []byte, error) {
 	headerResult := new(ipldResult)
-	return headerResult.cid, headerResult.data, r.db.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
+	return headerResult.CID, headerResult.Data, r.db.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
 }
 
 // RetrieveUnclesByHashes returns the cids and rlp bytes for the uncles corresponding to the provided uncle hashes
@@ -163,8 +236,8 @@ func (r *IPLDRetriever) RetrieveUnclesByHashes(hashes []common.Hash) ([]string,
 	cids := make([]string, len(uncleResults))
 	uncles := make([][]byte, len(uncleResults))
 	for i, res := range uncleResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		uncles[i] = res.data
+		uncles[i] = res.Data
 	}
 	return cids, uncles, nil
 }
@@ -178,8 +251,8 @@ func (r *IPLDRetriever) RetrieveUnclesByBlockHash(hash common.Hash) ([]string, [
 	cids := make([]string, len(uncleResults))
 	uncles := make([][]byte, len(uncleResults))
 	for i, res := range uncleResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		uncles[i] = res.data
+		uncles[i] = res.Data
 	}
 	return cids, uncles, nil
 }
@@ -193,8 +266,8 @@ func (r *IPLDRetriever) RetrieveUnclesByBlockNumber(number uint64) ([]string, []
 	cids := make([]string, len(uncleResults))
 	uncles := make([][]byte, len(uncleResults))
 	for i, res := range uncleResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		uncles[i] = res.data
+		uncles[i] = res.Data
 	}
 	return cids, uncles, nil
 }
@@ -202,7 +275,7 @@ func (r *IPLDRetriever) RetrieveUnclesByBlockNumber(number uint64) ([]string, []
 // RetrieveUncleByHash returns the cid and rlp bytes for the uncle corresponding to the provided uncle hash
 func (r *IPLDRetriever) RetrieveUncleByHash(hash common.Hash) (string, []byte, error) {
 	uncleResult := new(ipldResult)
-	return uncleResult.cid, uncleResult.data, r.db.Get(uncleResult, RetrieveUncleByHashPgStr, hash.Hex())
+	return uncleResult.CID, uncleResult.Data, r.db.Get(uncleResult, RetrieveUncleByHashPgStr, hash.Hex())
 }
 
 // RetrieveTransactionsByHashes returns the cids and rlp bytes for the transactions corresponding to the provided tx hashes
@@ -218,8 +291,8 @@ func (r *IPLDRetriever) RetrieveTransactionsByHashes(hashes []common.Hash) ([]st
 	cids := make([]string, len(txResults))
 	txs := make([][]byte, len(txResults))
 	for i, res := range txResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		txs[i] = res.data
+		txs[i] = res.Data
 	}
 	return cids, txs, nil
 }
@@ -233,8 +306,8 @@ func (r *IPLDRetriever) RetrieveTransactionsByBlockHash(hash common.Hash) ([]str
 	cids := make([]string, len(txResults))
 	txs := make([][]byte, len(txResults))
 	for i, res := range txResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		txs[i] = res.data
+		txs[i] = res.Data
 	}
 	return cids, txs, nil
 }
@@ -248,8 +321,8 @@ func (r *IPLDRetriever) RetrieveTransactionsByBlockNumber(number uint64) ([]stri
 	cids := make([]string, len(txResults))
 	txs := make([][]byte, len(txResults))
 	for i, res := range txResults {
-		cids[i] = res.cid
+		cids[i] = res.CID
-		txs[i] = res.data
+		txs[i] = res.Data
 	}
 	return cids, txs, nil
 }
@@ -257,12 +330,29 @@ func (r *IPLDRetriever) RetrieveTransactionsByBlockNumber(number uint64) ([]stri
 // RetrieveTransactionByTxHash returns the cid and rlp bytes for the transaction corresponding to the provided tx hash
 func (r *IPLDRetriever) RetrieveTransactionByTxHash(hash common.Hash) (string, []byte, error) {
 	txResult := new(ipldResult)
-	return txResult.cid, txResult.data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex())
+	return txResult.CID, txResult.Data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex())
+}
+
+// DecodeLeafNode decodes the leaf node data
+func DecodeLeafNode(node []byte) ([]byte, error) {
+	var nodeElements []interface{}
+	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return nil, err
+	}
+	ty, err := trie_helpers.CheckKeyType(nodeElements)
+	if err != nil {
+		return nil, err
+	}
+
+	if ty != sdtypes.Leaf {
+		return nil, fmt.Errorf("expected leaf node but found %s", ty)
+	}
+	return nodeElements[1].([]byte), nil
 }
 
 // RetrieveReceiptsByTxHashes returns the cids and rlp bytes for the receipts corresponding to the provided tx hashes
 func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]string, [][]byte, error) {
-	rctResults := make([]ipldResult, 0)
+	rctResults := make([]rctIpldResult, 0)
 	hashStrs := make([]string, len(hashes))
 	for i, hash := range hashes {
 		hashStrs[i] = hash.Hex()
@@ -273,85 +363,168 @@ func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]stri
 	cids := make([]string, len(rctResults))
 	rcts := make([][]byte, len(rctResults))
 	for i, res := range rctResults {
-		cids[i] = res.cid
+		cids[i] = res.LeafCID
-		rcts[i] = res.data
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, err
+		}
+		rcts[i] = nodeVal
 	}
 	return cids, rcts, nil
 }
 
-// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash
+// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
-func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, [][]byte, error) {
+// cid returned corresponds to the leaf node data which contains the receipt.
-	rctResults := make([]ipldResult, 0)
+func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, [][]byte, []common.Hash, error) {
+	rctResults := make([]rctIpldResult, 0)
 	if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	cids := make([]string, len(rctResults))
 	rcts := make([][]byte, len(rctResults))
+	txs := make([]common.Hash, len(rctResults))
+
 	for i, res := range rctResults {
-		cids[i] = res.cid
+		cids[i] = res.LeafCID
-		rcts[i] = res.data
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, nil, err
 		}
-	return cids, rcts, nil
+		rcts[i] = nodeVal
+		txs[i] = common.HexToHash(res.TxHash)
 	}
 
-// RetrieveReceiptsByBlockNumber returns the cids and rlp bytes for the receipts corresponding to the provided block hash
+	return cids, rcts, txs, nil
+}
+
+// RetrieveReceiptsByBlockNumber returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
+// cid returned corresponds to the leaf node data which contains the receipt.
 func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string, [][]byte, error) {
-	rctResults := make([]ipldResult, 0)
+	rctResults := make([]rctIpldResult, 0)
 	if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockNumberPgStr, number); err != nil {
 		return nil, nil, err
 	}
 	cids := make([]string, len(rctResults))
 	rcts := make([][]byte, len(rctResults))
 	for i, res := range rctResults {
-		cids[i] = res.cid
+		cids[i] = res.LeafCID
-		rcts[i] = res.data
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, err
+		}
+		rcts[i] = nodeVal
 	}
 	return cids, rcts, nil
 }
 
-// RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash
+// RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash.
+// cid returned corresponds to the leaf node data which contains the receipt.
 func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, error) {
-	rctResult := new(ipldResult)
+	rctResult := new(rctIpldResult)
-	return rctResult.cid, rctResult.data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex())
+	if err := r.db.Select(&rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex()); err != nil {
+		return "", nil, err
+	}
+
+	nodeVal, err := DecodeLeafNode(rctResult.Data)
+	if err != nil {
+		return "", nil, err
+	}
+	return rctResult.LeafCID, nodeVal, nil
+}
+
+type nodeInfo struct {
+	CID              string `db:"cid"`
+	Data             []byte `db:"data"`
+	NodeType         int    `db:"node_type"`
+	StateLeafRemoved bool   `db:"state_leaf_removed"`
 }
 
 // RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
+// TODO: ensure this handles deleted accounts appropriately
 func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) {
-	accountResult := new(ipldResult)
+	accountResult := new(nodeInfo)
 	leafKey := crypto.Keccak256Hash(address.Bytes())
 	if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil {
 		return "", nil, err
 	}
+
+	if accountResult.NodeType == removedNode {
+		return "", EmptyNodeValue, nil
+	}
+
 	var i []interface{}
-	if err := rlp.DecodeBytes(accountResult.data, &i); err != nil {
+	if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
 		return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
 	}
 	if len(i) != 2 {
 		return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements")
 	}
-	return accountResult.cid, i[1].([]byte), nil
+	return accountResult.CID, i[1].([]byte), nil
 }
 
 // RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number
-// This can return multiple results if we have two versions of state in the database as the provided height
+// This can return a non-canonical account
-func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) ([]string, [][]byte, error) {
+func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) (string, []byte, error) {
-	accountResults := make([]ipldResult, 0)
+	accountResult := new(nodeInfo)
 	leafKey := crypto.Keccak256Hash(address.Bytes())
-	if err := r.db.Get(&accountResults, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil {
+	if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil {
-		return nil, nil, err
+		return "", nil, err
 	}
-	cids := make([]string, len(accountResults))
+
-	accounts := make([][]byte, len(accountResults))
+	if accountResult.NodeType == removedNode {
-	for i, res := range accountResults {
+		return "", EmptyNodeValue, nil
-		cids[i] = res.cid
-		var iface []interface{}
-		if err := rlp.DecodeBytes(res.data, &iface); err != nil {
-			return nil, nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
 	}
-		if len(iface) != 2 {
+
-			return nil, nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements")
+	var i []interface{}
+	if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
+		return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
 	}
-		accounts[i] = iface[1].([]byte)
+	if len(i) != 2 {
+		return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements")
 	}
-	return cids, accounts, nil
+	return accountResult.CID, i[1].([]byte), nil
+}
+
+// RetrieveStorageAtByAddressAndStorageSlotAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage slot, and block hash
+func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address common.Address, key, hash common.Hash) (string, []byte, []byte, error) {
+	storageResult := new(nodeInfo)
+	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
+	storageHash := crypto.Keccak256Hash(key.Bytes())
+	if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil {
+		return "", nil, nil, err
+	}
+	if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode {
+		return "", EmptyNodeValue, EmptyNodeValue, nil
+	}
+	var i []interface{}
+	if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {
+		err = fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())
+		return "", nil, nil, err
+	}
+	if len(i) != 2 {
+		return "", nil, nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements")
+	}
+	return storageResult.CID, storageResult.Data, i[1].([]byte), nil
+}
+
+// RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block number
+// This can retrun a non-canonical value
+func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(address common.Address, storageLeafKey common.Hash, number uint64) (string, []byte, error) {
+	storageResult := new(nodeInfo)
+	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
+	if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), number); err != nil {
+		return "", nil, err
+	}
+
+	if storageResult.StateLeafRemoved || storageResult.NodeType == removedNode {
+		return "", EmptyNodeValue, nil
+	}
+	var i []interface{}
+	if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {
+		return "", nil, fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())
+	}
+	if len(i) != 2 {
+		return "", nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements")
+	}
+	return storageResult.CID, i[1].([]byte), nil
 }

pkg/eth/ipld_retriever_test.go

@@ -1,156 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-/*
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth/mocks"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-var _ = Describe("IPLD Retriever", func() {
-	var (
-		db   *postgres.DB
-		repo *eth2.IPLDPublisher
-		//retriever *eth.IPLDRetriever
-	)
-	BeforeEach(func() {
-		var err error
-		db, err = shared.SetupDB()
-		Expect(err).ToNot(HaveOccurred())
-		repo = eth2.NewIPLDPublisher(db)
-		//retriever = eth.NewIPLDRetriever(db)
-		err = repo.Publish(mocks.MockConvertedPayload)
-		Expect(err).ToNot(HaveOccurred())
-		err = repo.Publish(mocks.MockConvertedPayload2)
-		Expect(err).ToNot(HaveOccurred())
-	})
-	AfterEach(func() {
-		eth.TearDownDB(db)
-	})
-
-	Describe("RetrieveHeadersByHashes", func() {
-		It("Retrieves all of the headers that correspond to the provided hashes", func() {
-
-		})
-	})
-
-	Describe("RetrieveHeadersByBlockNumber", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveHeaderByHash", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveUnclesByHashes", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveUnclesByBlockHash", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveUnclesByBlockNumber", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveUncleByHash", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveTransactionsByHashes", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveTransactionsByBlockHash", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveTransactionsByBlockNumber", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveTransactionByTxHash", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveReceiptsByTxHashes", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveReceiptsByBlockHash", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveReceiptsByBlockNumber", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveReceiptByHash", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveAccountByAddressAndBlockHash", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-	Describe("RetrieveAccountByAddressAndBlockNumber", func() {
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-
-		})
-	})
-
-})
-*/

pkg/eth/test_helpers.go

@@ -17,36 +17,11 @@
 package eth
 
 import (
-	. "github.com/onsi/gomega"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 )
 
-// TearDownDB is used to tear down the watcher dbs after tests
-func TearDownDB(db *postgres.DB) {
-	tx, err := db.Beginx()
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DELETE FROM eth.header_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.transaction_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.receipt_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.state_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.storage_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM blocks`)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = tx.Commit()
-	Expect(err).NotTo(HaveOccurred())
-}
-
 // TxModelsContainsCID used to check if a list of TxModels contains a specific cid string
-func TxModelsContainsCID(txs []eth.TxModel, cid string) bool {
+func TxModelsContainsCID(txs []models.TxModel, cid string) bool {
 	for _, tx := range txs {
 		if tx.CID == cid {
 			return true
@@ -55,10 +30,10 @@ func TxModelsContainsCID(txs []eth.TxModel, cid string) bool {
 	return false
 }
 
-// ListContainsBytes used to check if a list of byte arrays contains a particular byte array
+// ReceiptModelsContainsCID used to check if a list of ReceiptModel contains a specific cid string
-func ReceiptModelsContainsCID(rcts []eth.ReceiptModel, cid string) bool {
+func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool {
 	for _, rct := range rcts {
-		if rct.CID == cid {
+		if rct.LeafCID == cid {
 			return true
 		}
 	}

pkg/eth/test_helpers/chain_maker.go

@@ -41,8 +41,15 @@ var (
 	Account2Key, _       = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
 	Account1Addr         = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
 	Account2Addr         = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e
-	ContractCode   = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032")
+	DeploymentTxData     = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032")
+	ContractCode         = common.Hex2Bytes("608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032")
+	CodeHash             = crypto.Keccak256Hash(ContractCode)
 	ContractAddr         common.Address
+	IndexZero            = "0000000000000000000000000000000000000000000000000000000000000000"
+	IndexOne             = "0000000000000000000000000000000000000000000000000000000000000001"
+	ContractSlotPosition = common.FromHex(IndexOne)
+	ContractSlotKeyHash  = crypto.Keccak256Hash(ContractSlotPosition)
+	MiningReward         = big.NewInt(2000000000000000000)
 )
 
 /* test function signatures
@@ -55,8 +62,9 @@ data function sig: 73d4a13a
 // the returned hash chain is ordered head->parent.
 func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, []types.Receipts, *core.BlockChain) {
 	config := params.TestChainConfig
+	config.LondonBlock = big.NewInt(100)
 	blocks, receipts := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
-	chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil)
+	chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
 	return append([]*types.Block{parent}, blocks...), receipts, chain
 }
 
@@ -75,7 +83,7 @@ func TestChainGen(i int, block *core.BlockGen) {
 		nonce := block.TxNonce(Account1Addr)
 		tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key)
 		nonce++
-		tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, Account1Key)
+		tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), DeploymentTxData), signer, Account1Key)
 		ContractAddr = crypto.CreateAddress(Account1Addr, nonce)
 		block.AddTx(tx1)
 		block.AddTx(tx2)
@@ -88,8 +96,8 @@ func TestChainGen(i int, block *core.BlockGen) {
 	case 3:
 		block.SetCoinbase(Account2Addr)
 		data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000009")
-		tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
-		block.AddTx(tx1)
+		block.AddTx(tx)
 	case 4:
 		block.SetCoinbase(Account1Addr)
 		data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000000")

pkg/eth/test_helpers/test_data.go

@@ -17,30 +17,28 @@
 package test_helpers
 
 import (
+	"bytes"
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
 	"math/big"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
-
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
-	"github.com/ethereum/go-ethereum/statediff/testhelpers"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
-	"github.com/ipfs/go-block-format"
+	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
+	testhelpers "github.com/ethereum/go-ethereum/statediff/test_helpers"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/ethereum/go-ethereum/trie"
+	blocks "github.com/ipfs/go-block-format"
 	"github.com/multiformats/go-multihash"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
-
-	eth2 "github.com/vulcanize/ipld-eth-server/pkg/eth"
 )
 
 // Test variables
@@ -56,12 +54,46 @@ var (
 		Difficulty:  big.NewInt(5000000),
 		Extra:       []byte{},
 	}
-	MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts()
+	MockTransactions, MockReceipts, SenderAddr = createLegacyTransactionsAndReceipts()
+	MockUncles                                 = []*types.Header{
+		{
+			Time:        1,
+			Number:      new(big.Int).Add(BlockNumber, big.NewInt(1)),
+			Root:        common.HexToHash("0x1"),
+			TxHash:      common.HexToHash("0x1"),
+			ReceiptHash: common.HexToHash("0x1"),
+			Difficulty:  big.NewInt(500001),
+			Extra:       []byte{},
+		},
+		{
+			Time:        2,
+			Number:      new(big.Int).Add(BlockNumber, big.NewInt(2)),
+			Root:        common.HexToHash("0x2"),
+			TxHash:      common.HexToHash("0x2"),
+			ReceiptHash: common.HexToHash("0x2"),
+			Difficulty:  big.NewInt(500002),
+			Extra:       []byte{},
+		},
+	}
 	ReceiptsRlp, _   = rlp.EncodeToBytes(MockReceipts)
-	MockBlock                                  = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts)
+	MockBlock        = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie))
 	MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header())
+	MockChildHeader  = types.Header{
+		Time:        0,
+		Number:      new(big.Int).Add(BlockNumber, common.Big1),
+		Root:        common.HexToHash("0x0"),
+		TxHash:      common.HexToHash("0x0"),
+		ReceiptHash: common.HexToHash("0x0"),
+		Difficulty:  big.NewInt(5000001),
+		Extra:       []byte{},
+		ParentHash:  MockBlock.Header().Hash(),
+	}
+	MockChild            = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie))
+	MockChildRlp, _      = rlp.EncodeToBytes(MockChild.Header())
 	Address              = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
 	AnotherAddress       = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	AnotherAddress1      = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476594")
+	AnotherAddress2      = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476596")
 	ContractAddress      = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
 	ContractHash         = crypto.Keccak256Hash(ContractAddress.Bytes()).String()
 	MockContractByteCode = []byte{0, 1, 2, 3, 4, 5}
@@ -69,37 +101,97 @@ var (
 	mockTopic12          = common.HexToHash("0x06")
 	mockTopic21          = common.HexToHash("0x05")
 	mockTopic22          = common.HexToHash("0x07")
+	mockTopic31          = common.HexToHash("0x08")
+	mockTopic41          = common.HexToHash("0x09")
+	mockTopic42          = common.HexToHash("0x0a")
+	mockTopic43          = common.HexToHash("0x0b")
+	mockTopic51          = common.HexToHash("0x0c")
+	mockTopic61          = common.HexToHash("0x0d")
 	MockLog1             = &types.Log{
 		Address:     Address,
 		Topics:      []common.Hash{mockTopic11, mockTopic12},
 		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     0,
+		Index:       0,
 	}
 	MockLog2 = &types.Log{
 		Address:     AnotherAddress,
 		Topics:      []common.Hash{mockTopic21, mockTopic22},
 		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     1,
+		Index:       1,
 	}
+	MockLog3 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic31},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     2,
+		Index:       2,
+	}
+
+	MockLog4 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic41, mockTopic42, mockTopic43},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     2,
+		Index:       3,
+	}
+	MockLog5 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic51},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     2,
+		Index:       4,
+	}
+	MockLog6 = &types.Log{
+		Address:     AnotherAddress2,
+		Topics:      []common.Hash{mockTopic61},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     3,
+		Index:       5,
+	}
+
+	Tx1 = GetTxnRlp(0, MockTransactions)
+	Tx2 = GetTxnRlp(1, MockTransactions)
+	Tx3 = GetTxnRlp(2, MockTransactions)
+	Tx4 = GetTxnRlp(3, MockTransactions)
+
+	rctCIDs, rctIPLDData, _ = eth.GetRctLeafNodeData(MockReceipts)
 	HeaderCID, _            = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
 	HeaderMhKey             = shared.MultihashKeyFromCID(HeaderCID)
-	Trx1CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(0), multihash.KECCAK_256)
+	Trx1CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx1, multihash.KECCAK_256)
 	Trx1MhKey               = shared.MultihashKeyFromCID(Trx1CID)
-	Trx2CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
+	Trx2CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx2, multihash.KECCAK_256)
 	Trx2MhKey               = shared.MultihashKeyFromCID(Trx2CID)
-	Trx3CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(2), multihash.KECCAK_256)
+	Trx3CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx3, multihash.KECCAK_256)
 	Trx3MhKey               = shared.MultihashKeyFromCID(Trx3CID)
-	Rct1CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
+	Trx4CID, _              = ipld.RawdataToCid(ipld.MEthTx, Tx4, multihash.KECCAK_256)
+	Trx4MhKey               = shared.MultihashKeyFromCID(Trx4CID)
+	Rct1CID                 = rctCIDs[0]
 	Rct1MhKey               = shared.MultihashKeyFromCID(Rct1CID)
-	Rct2CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
+	Rct2CID                 = rctCIDs[1]
 	Rct2MhKey               = shared.MultihashKeyFromCID(Rct2CID)
-	Rct3CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(2), multihash.KECCAK_256)
+	Rct3CID                 = rctCIDs[2]
 	Rct3MhKey               = shared.MultihashKeyFromCID(Rct3CID)
+	Rct4CID                 = rctCIDs[3]
+	Rct4MhKey               = shared.MultihashKeyFromCID(Rct4CID)
 	State1CID, _            = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
 	State1MhKey             = shared.MultihashKeyFromCID(State1CID)
 	State2CID, _            = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
 	State2MhKey             = shared.MultihashKeyFromCID(State2CID)
 	StorageCID, _           = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
 	StorageMhKey            = shared.MultihashKeyFromCID(StorageCID)
-	MockTrxMeta   = []eth.TxModel{
+	Rct1IPLD                = rctIPLDData[0]
+	Rct2IPLD                = rctIPLDData[1]
+	Rct3IPLD                = rctIPLDData[2]
+	Rct4IPLD                = rctIPLDData[3]
+	MockTrxMeta             = []models.TxModel{
 		{
 			CID:    "", // This is empty until we go to publish to ipfs
 			MhKey:  "",
@@ -127,8 +219,17 @@ var (
 			TxHash: MockTransactions[2].Hash().String(),
 			Data:   MockContractByteCode,
 		},
+		{
+			CID:    "",
+			MhKey:  "",
+			Src:    SenderAddr.Hex(),
+			Dst:    "",
+			Index:  3,
+			TxHash: MockTransactions[3].Hash().String(),
+			Data:   []byte{},
+		},
 	}
-	MockTrxMetaPostPublsh = []eth.TxModel{
+	MockTrxMetaPostPublsh = []models.TxModel{
 		{
 			CID:    Trx1CID.String(), // This is empty until we go to publish to ipfs
 			MhKey:  Trx1MhKey,
@@ -156,146 +257,139 @@ var (
 			TxHash: MockTransactions[2].Hash().String(),
 			Data:   MockContractByteCode,
 		},
-	}
-	MockRctMeta = []eth.ReceiptModel{
 		{
-			CID:   "",
+			CID:    Trx4CID.String(),
-			MhKey: "",
+			MhKey:  Trx4MhKey,
-			Topic0s: []string{
+			Src:    SenderAddr.Hex(),
-				mockTopic11.String(),
+			Dst:    AnotherAddress1.String(),
-			},
+			Index:  3,
-			Topic1s: []string{
+			TxHash: MockTransactions[3].Hash().String(),
-				mockTopic12.String(),
+			Data:   []byte{},
-			},
-			Contract:     "",
-			ContractHash: "",
-			LogContracts: []string{
-				Address.String(),
-			},
-		},
-		{
-			CID:   "",
-			MhKey: "",
-			Topic0s: []string{
-				mockTopic21.String(),
-			},
-			Topic1s: []string{
-				mockTopic22.String(),
-			},
-			Contract:     "",
-			ContractHash: "",
-			LogContracts: []string{
-				AnotherAddress.String(),
-			},
-		},
-		{
-			CID:          "",
-			MhKey:        "",
-			Contract:     ContractAddress.String(),
-			ContractHash: ContractHash,
-			LogContracts: []string{},
 		},
 	}
-	MockRctMetaPostPublish = []eth.ReceiptModel{
+	MockRctMeta = []models.ReceiptModel{
 		{
-			CID:   Rct1CID.String(),
+			LeafCID:      "",
-			MhKey: Rct1MhKey,
+			LeafMhKey:    "",
-			Topic0s: []string{
-				mockTopic11.String(),
-			},
-			Topic1s: []string{
-				mockTopic12.String(),
-			},
 			Contract:     "",
 			ContractHash: "",
-			LogContracts: []string{
-				Address.String(),
-			},
 		},
 		{
-			CID:   Rct2CID.String(),
+			LeafCID:      "",
-			MhKey: Rct2MhKey,
+			LeafMhKey:    "",
-			Topic0s: []string{
-				mockTopic21.String(),
-			},
-			Topic1s: []string{
-				mockTopic22.String(),
-			},
 			Contract:     "",
 			ContractHash: "",
-			LogContracts: []string{
-				AnotherAddress.String(),
-			},
 		},
 		{
-			CID:          Rct3CID.String(),
+			LeafCID:      "",
-			MhKey:        Rct3MhKey,
+			LeafMhKey:    "",
 			Contract:     ContractAddress.String(),
 			ContractHash: ContractHash,
-			LogContracts: []string{},
+		},
+		{
+			LeafCID:      "",
+			LeafMhKey:    "",
+			Contract:     "",
+			ContractHash: "",
+		},
+	}
+
+	MockRctMetaPostPublish = []models.ReceiptModel{
+		{
+			LeafCID:      Rct1CID.String(),
+			LeafMhKey:    Rct1MhKey,
+			Contract:     "",
+			ContractHash: "",
+		},
+		{
+			LeafCID:      Rct2CID.String(),
+			LeafMhKey:    Rct2MhKey,
+			Contract:     "",
+			ContractHash: "",
+		},
+		{
+			LeafCID:      Rct3CID.String(),
+			LeafMhKey:    Rct3MhKey,
+			Contract:     ContractAddress.String(),
+			ContractHash: ContractHash,
+		},
+		{
+			LeafCID:      Rct4CID.String(),
+			LeafMhKey:    Rct4MhKey,
+			Contract:     "",
+			ContractHash: "",
 		},
 	}
 
 	// statediff data
 	storageLocation    = common.HexToHash("0")
 	StorageLeafKey     = crypto.Keccak256Hash(storageLocation[:]).Bytes()
-	StorageValue       = common.Hex2Bytes("01")
+	StorageValue       = crypto.Keccak256([]byte{1, 2, 3, 4, 5})
 	StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
-	StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		StoragePartialPath,
 		StorageValue,
 	})
 
 	nonce1             = uint64(1)
 	ContractRoot       = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
-	ContractCodeHash   = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea")
+	ContractCodeHash   = crypto.Keccak256Hash(MockContractByteCode)
 	contractPath       = common.Bytes2Hex([]byte{'\x06'})
 	ContractLeafKey    = testhelpers.AddressToLeafKey(ContractAddress)
-	ContractAccount, _ = rlp.EncodeToBytes(state.Account{
+	ContractAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    nonce1,
 		Balance:  big.NewInt(0),
 		CodeHash: ContractCodeHash.Bytes(),
 		Root:     common.HexToHash(ContractRoot),
 	})
 	ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
-	ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	ContractLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		ContractPartialPath,
 		ContractAccount,
 	})
 
 	nonce0          = uint64(0)
+	AccountBalance  = big.NewInt(1000)
 	AccountRoot     = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
 	AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
-	accountPath     = common.Bytes2Hex([]byte{'\x0c'})
 	AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
 	AccountLeafKey  = testhelpers.Account2LeafKey
-	Account, _      = rlp.EncodeToBytes(state.Account{
+	Account, _      = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    nonce0,
-		Balance:  big.NewInt(1000),
+		Balance:  AccountBalance,
 		CodeHash: AccountCodeHash.Bytes(),
 		Root:     common.HexToHash(AccountRoot),
 	})
 	AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
-	AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	AccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		AccountPartialPath,
 		Account,
 	})
 
-	MockStateNodes = []eth.TrieNode{
+	MockStateNodes = []sdtypes.StateNode{
 		{
-			LeafKey: common.BytesToHash(ContractLeafKey),
+			LeafKey:   ContractLeafKey,
 			Path:      []byte{'\x06'},
-			Value:   ContractLeafNode,
+			NodeValue: ContractLeafNode,
-			Type:    statediff.Leaf,
+			NodeType:  sdtypes.Leaf,
+			StorageNodes: []sdtypes.StorageNode{
+				{
+					Path:      []byte{},
+					NodeType:  sdtypes.Leaf,
+					LeafKey:   StorageLeafKey,
+					NodeValue: StorageLeafNode,
+				},
+			},
 		},
 		{
-			LeafKey: common.BytesToHash(AccountLeafKey),
+			LeafKey:      AccountLeafKey,
 			Path:         []byte{'\x0c'},
-			Value:   AccountLeafNode,
+			NodeValue:    AccountLeafNode,
-			Type:    statediff.Leaf,
+			NodeType:     sdtypes.Leaf,
+			StorageNodes: []sdtypes.StorageNode{},
 		},
 	}
-	MockStateMetaPostPublish = []eth.StateNodeModel{
+	MockStateMetaPostPublish = []models.StateNodeModel{
 		{
 			CID:      State1CID.String(),
 			MhKey:    State1MhKey,
@@ -311,12 +405,12 @@ var (
 			StateKey: common.BytesToHash(AccountLeafKey).Hex(),
 		},
 	}
-	MockStorageNodes = map[string][]eth.TrieNode{
+	MockStorageNodes = map[string][]sdtypes.StorageNode{
 		contractPath: {
 			{
-				LeafKey: common.BytesToHash(StorageLeafKey),
+				LeafKey:   StorageLeafKey,
-				Value:   StorageLeafNode,
+				NodeValue: StorageLeafNode,
-				Type:    statediff.Leaf,
+				NodeType:  sdtypes.Leaf,
 				Path:      []byte{},
 			},
 		},
@@ -331,17 +425,27 @@ var (
 		StorageNodes:    MockStorageNodes,
 		StateNodes:      MockStateNodes,
 	}
+	MockConvertedPayloadForChild = eth.ConvertedPayload{
+		TotalDifficulty: MockChild.Difficulty(),
+		Block:           MockChild,
+		Receipts:        MockReceipts,
+		TxMetaData:      MockTrxMeta,
+		ReceiptMetaData: MockRctMeta,
+		StorageNodes:    MockStorageNodes,
+		StateNodes:      MockStateNodes,
+	}
 
-	MockCIDWrapper = &eth2.CIDWrapper{
+	Reward         = shared.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts)
+	MockCIDWrapper = &eth.CIDWrapper{
 		BlockNumber: new(big.Int).Set(BlockNumber),
-		Header: eth.HeaderModel{
+		Header: models.HeaderModel{
 			BlockNumber:     "1",
 			BlockHash:       MockBlock.Hash().String(),
 			ParentHash:      "0x0000000000000000000000000000000000000000000000000000000000000000",
 			CID:             HeaderCID.String(),
 			MhKey:           HeaderMhKey,
 			TotalDifficulty: MockBlock.Difficulty().String(),
-			Reward:          "5000000000000000000",
+			Reward:          Reward.String(),
 			StateRoot:       MockBlock.Root().String(),
 			RctRoot:         MockBlock.ReceiptHash().String(),
 			TxRoot:          MockBlock.TxHash().String(),
@@ -349,12 +453,13 @@ var (
 			Bloom:           MockBlock.Bloom().Bytes(),
 			Timestamp:       MockBlock.Time(),
 			TimesValidated:  1,
+			Coinbase:        "0x0000000000000000000000000000000000000000",
 		},
 		Transactions: MockTrxMetaPostPublsh,
 		Receipts:     MockRctMetaPostPublish,
-		Uncles:       []eth.UncleModel{},
+		Uncles:       []models.UncleModel{},
 		StateNodes:   MockStateMetaPostPublish,
-		StorageNodes: []eth.StorageNodeWithStateKeyModel{
+		StorageNodes: []models.StorageNodeWithStateKeyModel{
 			{
 				Path:       []byte{},
 				CID:        StorageCID.String(),
@@ -367,91 +472,169 @@ var (
 	}
 
 	HeaderIPLD, _  = blocks.NewBlockWithCid(MockHeaderRlp, HeaderCID)
-	Trx1IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(0), Trx1CID)
+	Trx1IPLD, _    = blocks.NewBlockWithCid(Tx1, Trx1CID)
-	Trx2IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), Trx2CID)
+	Trx2IPLD, _    = blocks.NewBlockWithCid(Tx2, Trx2CID)
-	Trx3IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(2), Trx3CID)
+	Trx3IPLD, _    = blocks.NewBlockWithCid(Tx3, Trx3CID)
-	Rct1IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), Rct1CID)
+	Trx4IPLD, _    = blocks.NewBlockWithCid(Tx4, Trx4CID)
-	Rct2IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), Rct2CID)
-	Rct3IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(2), Rct3CID)
 	State1IPLD, _  = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
 	State2IPLD, _  = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
 	StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
 
-	MockIPLDs = eth2.IPLDs{
+	MockIPLDs = eth.IPLDs{
 		BlockNumber: new(big.Int).Set(BlockNumber),
-		Header: ipfs.BlockModel{
+		Header: models.IPLDModel{
 			Data: HeaderIPLD.RawData(),
-			CID:  HeaderIPLD.Cid().String(),
+			Key:  HeaderIPLD.Cid().String(),
 		},
-		Transactions: []ipfs.BlockModel{
+		Transactions: []models.IPLDModel{
 			{
 				Data: Trx1IPLD.RawData(),
-				CID:  Trx1IPLD.Cid().String(),
+				Key:  Trx1IPLD.Cid().String(),
 			},
 			{
 				Data: Trx2IPLD.RawData(),
-				CID:  Trx2IPLD.Cid().String(),
+				Key:  Trx2IPLD.Cid().String(),
 			},
 			{
 				Data: Trx3IPLD.RawData(),
-				CID:  Trx3IPLD.Cid().String(),
+				Key:  Trx3IPLD.Cid().String(),
-			},
-		},
-		Receipts: []ipfs.BlockModel{
-			{
-				Data: Rct1IPLD.RawData(),
-				CID:  Rct1IPLD.Cid().String(),
 			},
 			{
-				Data: Rct2IPLD.RawData(),
+				Data: Trx4IPLD.RawData(),
-				CID:  Rct2IPLD.Cid().String(),
+				Key:  Trx4IPLD.Cid().String(),
+			},
+		},
+		Receipts: []models.IPLDModel{
+			{
+				Data: Rct1IPLD,
+				Key:  Rct1CID.String(),
 			},
 			{
-				Data: Rct3IPLD.RawData(),
+				Data: Rct2IPLD,
-				CID:  Rct3IPLD.Cid().String(),
+				Key:  Rct2CID.String(),
+			},
+			{
+				Data: Rct3IPLD,
+				Key:  Rct3CID.String(),
+			},
+			{
+				Data: Rct4IPLD,
+				Key:  Rct4CID.String(),
 			},
 		},
-		StateNodes: []eth2.StateNode{
+		StateNodes: []eth.StateNode{
 			{
 				StateLeafKey: common.BytesToHash(ContractLeafKey),
-				Type:         statediff.Leaf,
+				Type:         sdtypes.Leaf,
-				IPLD: ipfs.BlockModel{
+				IPLD: models.IPLDModel{
 					Data: State1IPLD.RawData(),
-					CID:  State1IPLD.Cid().String(),
+					Key:  State1IPLD.Cid().String(),
 				},
 				Path: []byte{'\x06'},
 			},
 			{
 				StateLeafKey: common.BytesToHash(AccountLeafKey),
-				Type:         statediff.Leaf,
+				Type:         sdtypes.Leaf,
-				IPLD: ipfs.BlockModel{
+				IPLD: models.IPLDModel{
 					Data: State2IPLD.RawData(),
-					CID:  State2IPLD.Cid().String(),
+					Key:  State2IPLD.Cid().String(),
 				},
 				Path: []byte{'\x0c'},
 			},
 		},
-		StorageNodes: []eth2.StorageNode{
+		StorageNodes: []eth.StorageNode{
 			{
 				StateLeafKey:   common.BytesToHash(ContractLeafKey),
 				StorageLeafKey: common.BytesToHash(StorageLeafKey),
-				Type:           statediff.Leaf,
+				Type:           sdtypes.Leaf,
-				IPLD: ipfs.BlockModel{
+				IPLD: models.IPLDModel{
 					Data: StorageIPLD.RawData(),
-					CID:  StorageIPLD.Cid().String(),
+					Key:  StorageIPLD.Cid().String(),
 				},
 				Path: []byte{},
 			},
 		},
 	}
+
+	LondonBlockNum   = new(big.Int).Add(BlockNumber, big.NewInt(2))
+	MockLondonHeader = types.Header{
+		Time:       0,
+		Number:     LondonBlockNum,
+		Root:       common.HexToHash("0x00"),
+		Difficulty: big.NewInt(5000000),
+		Extra:      []byte{},
+		BaseFee:    big.NewInt(params.InitialBaseFee),
+	}
+
+	MockLondonTransactions, MockLondonReceipts, _ = createDynamicTransactionsAndReceipts(LondonBlockNum)
+	MockLondonBlock                               = createNewBlock(&MockLondonHeader, MockLondonTransactions, nil, MockLondonReceipts, new(trie.Trie))
 )
 
-// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createNewBlock(header *types.Header, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, hasher types.TrieHasher) *types.Block {
-func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
+	block := types.NewBlock(header, txs, uncles, receipts, hasher)
+	bHash := block.Hash()
+	for _, r := range receipts {
+		for _, l := range r.Logs {
+			l.BlockHash = bHash
+		}
+	}
+	return block
+}
+
+// createDynamicTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createDynamicTransactionsAndReceipts(blockNumber *big.Int) (types.Transactions, types.Receipts, common.Address) {
+	// make transactions
+	config := params.TestChainConfig
+	config.LondonBlock = blockNumber
+	trx1 := types.NewTx(&types.DynamicFeeTx{
+		ChainID:   config.ChainID,
+		Nonce:     1,
+		GasTipCap: big.NewInt(50),
+		GasFeeCap: big.NewInt(100),
+		Gas:       50,
+		To:        &Address,
+		Value:     big.NewInt(1000),
+		Data:      []byte{},
+	})
+
+	transactionSigner := types.MakeSigner(config, blockNumber)
+	mockCurve := elliptic.P256()
+	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+	signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	senderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	// make receipts
+	// TODO: Change the receipt type to DynamicFeeTxType once this PR is merged.
+	// https://github.com/ethereum/go-ethereum/pull/22806
+	mockReceipt1 := &types.Receipt{
+		Type:              types.DynamicFeeTxType,
+		PostState:         common.HexToHash("0x0").Bytes(),
+		Status:            types.ReceiptStatusSuccessful,
+		CumulativeGasUsed: 50,
+		Logs:              []*types.Log{},
+		TxHash:            signedTrx1.Hash(),
+	}
+
+	return types.Transactions{signedTrx1}, types.Receipts{mockReceipt1}, senderAddr
+}
+
+// createLegacyTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
 	// make transactions
 	trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
 	trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
 	trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode)
+	trx4 := types.NewTransaction(3, AnotherAddress1, big.NewInt(2000), 100, big.NewInt(200), []byte{})
 	transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber))
 	mockCurve := elliptic.P256()
 	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
@@ -470,19 +653,60 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common
 	if err != nil {
 		log.Fatal(err)
 	}
+	signedTrx4, err := types.SignTx(trx4, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
 	SenderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
 	if err != nil {
 		log.Fatal(err)
 	}
 	// make receipts
-	mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
+	mockReceipt1 := types.NewReceipt(nil, false, 50)
+
+	hash1 := signedTrx1.Hash()
+	MockLog1.TxHash = hash1
+
 	mockReceipt1.Logs = []*types.Log{MockLog1}
-	mockReceipt1.TxHash = signedTrx1.Hash()
+	mockReceipt1.TxHash = hash1
+	mockReceipt1.GasUsed = mockReceipt1.CumulativeGasUsed
+
 	mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
+	hash2 := signedTrx2.Hash()
+	MockLog2.TxHash = hash2
+
 	mockReceipt2.Logs = []*types.Log{MockLog2}
-	mockReceipt2.TxHash = signedTrx2.Hash()
+	mockReceipt2.TxHash = hash2
-	mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75)
+	mockReceipt2.GasUsed = mockReceipt2.CumulativeGasUsed - mockReceipt1.CumulativeGasUsed
-	mockReceipt3.Logs = []*types.Log{}
+
+	mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 175)
+	mockReceipt3.Logs = []*types.Log{MockLog3, MockLog4, MockLog5}
 	mockReceipt3.TxHash = signedTrx3.Hash()
-	return types.Transactions{signedTrx1, signedTrx2, signedTrx3}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3}, SenderAddr
+	mockReceipt3.GasUsed = mockReceipt3.CumulativeGasUsed - mockReceipt2.CumulativeGasUsed
+
+	// Receipt with failed status.
+	mockReceipt4 := types.NewReceipt(nil, true, 250)
+	mockReceipt4.Logs = []*types.Log{MockLog6}
+	mockReceipt4.TxHash = signedTrx4.Hash()
+	mockReceipt4.GasUsed = mockReceipt4.CumulativeGasUsed - mockReceipt3.CumulativeGasUsed
+
+	return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4}, SenderAddr
+}
+
+func GetTxnRlp(num int, txs types.Transactions) []byte {
+	buf := new(bytes.Buffer)
+	txs.EncodeIndex(num, buf)
+	tx := make([]byte, buf.Len())
+	copy(tx, buf.Bytes())
+	buf.Reset()
+	return tx
+}
+
+func GetRctRlp(num int, rcts types.Receipts) []byte {
+	buf := new(bytes.Buffer)
+	rcts.EncodeIndex(num, buf)
+	rct := make([]byte, buf.Len())
+	copy(rct, buf.Bytes())
+	buf.Reset()
+	return rct
 }

pkg/eth/types.go

@@ -17,40 +17,206 @@
 package eth
 
 import (
+	"errors"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/ethereum/go-ethereum/common/math"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/sirupsen/logrus"
 )
 
+// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
+type RPCTransaction struct {
+	BlockHash        *common.Hash      `json:"blockHash"`
+	BlockNumber      *hexutil.Big      `json:"blockNumber"`
+	From             common.Address    `json:"from"`
+	Gas              hexutil.Uint64    `json:"gas"`
+	GasPrice         *hexutil.Big      `json:"gasPrice"`
+	GasFeeCap        *hexutil.Big      `json:"maxFeePerGas,omitempty"`
+	GasTipCap        *hexutil.Big      `json:"maxPriorityFeePerGas,omitempty"`
+	Hash             common.Hash       `json:"hash"`
+	Input            hexutil.Bytes     `json:"input"`
+	Nonce            hexutil.Uint64    `json:"nonce"`
+	To               *common.Address   `json:"to"`
+	TransactionIndex *hexutil.Uint64   `json:"transactionIndex"`
+	Value            *hexutil.Big      `json:"value"`
+	Type             hexutil.Uint64    `json:"type"`
+	Accesses         *types.AccessList `json:"accessList,omitempty"`
+	ChainID          *hexutil.Big      `json:"chainId,omitempty"`
+	V                *hexutil.Big      `json:"v"`
+	R                *hexutil.Big      `json:"r"`
+	S                *hexutil.Big      `json:"s"`
+}
+
+// RPCReceipt represents a receipt that will serialize to the RPC representation of a receipt
+type RPCReceipt struct {
+	BlockHash        *common.Hash    `json:"blockHash"`
+	BlockNumber      *hexutil.Big    `json:"blockNumber"`
+	TransactionHash  *common.Hash    `json:"transactionHash"`
+	TransactionIndex *hexutil.Uint64 `json:"transactionIndex"`
+	From             common.Address  `json:"from"`
+	To               *common.Address `json:"to"`
+	GasUsed          hexutil.Uint64  `json:"gasUsed"`
+	CumulativeGsUsed hexutil.Uint64  `json:"cumulativeGasUsed"`
+	ContractAddress  *common.Address `json:"contractAddress"`
+	Logs             []*types.Log    `json:"logs"`
+	Bloom            types.Bloom     `json:"logsBloom"`
+	Root             []byte          `json:"root"`
+	Status           uint64          `json:"status"`
+}
+
+// AccountResult struct for GetProof
+type AccountResult struct {
+	Address      common.Address  `json:"address"`
+	AccountProof []string        `json:"accountProof"`
+	Balance      *hexutil.Big    `json:"balance"`
+	CodeHash     common.Hash     `json:"codeHash"`
+	Nonce        hexutil.Uint64  `json:"nonce"`
+	StorageHash  common.Hash     `json:"storageHash"`
+	StorageProof []StorageResult `json:"storageProof"`
+}
+
+// StorageResult for GetProof
+type StorageResult struct {
+	Key   string       `json:"key"`
+	Value *hexutil.Big `json:"value"`
+	Proof []string     `json:"proof"`
+}
+
+// CallArgs represents the arguments for a call.
+type CallArgs struct {
+	From                 *common.Address   `json:"from"`
+	To                   *common.Address   `json:"to"`
+	Gas                  *hexutil.Uint64   `json:"gas"`
+	GasPrice             *hexutil.Big      `json:"gasPrice"`
+	MaxFeePerGas         *hexutil.Big      `json:"maxFeePerGas"`
+	MaxPriorityFeePerGas *hexutil.Big      `json:"maxPriorityFeePerGas"`
+	Value                *hexutil.Big      `json:"value"`
+	Data                 *hexutil.Bytes    `json:"data"`
+	AccessList           *types.AccessList `json:"accessList,omitempty"`
+	Input                *hexutil.Bytes    `json:"input"`
+}
+
+// from retrieves the transaction sender address.
+func (arg *CallArgs) from() common.Address {
+	if arg.From == nil {
+		return common.Address{}
+	}
+	return *arg.From
+}
+
+// data retrieves the transaction calldata. Input field is preferred.
+func (arg *CallArgs) data() []byte {
+	if arg.Input != nil {
+		return *arg.Input
+	}
+	if arg.Data != nil {
+		return *arg.Data
+	}
+	return nil
+}
+
+// ToMessage converts the transaction arguments to the Message type used by the
+// core evm. This method is used in calls and traces that do not require a real
+// live transaction.
+func (arg *CallArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (types.Message, error) {
+	// Reject invalid combinations of pre- and post-1559 fee styles
+	if arg.GasPrice != nil && (arg.MaxFeePerGas != nil || arg.MaxPriorityFeePerGas != nil) {
+		return types.Message{}, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified")
+	}
+	// Set sender address or use zero address if none specified.
+	addr := arg.from()
+
+	// Set default gas & gas price if none were set
+	gas := globalGasCap
+	if gas == 0 {
+		gas = uint64(math.MaxUint64 / 2)
+	}
+	if arg.Gas != nil {
+		gas = uint64(*arg.Gas)
+	}
+	if globalGasCap != 0 && globalGasCap < gas {
+		logrus.Warn("Caller gas above allowance, capping", "requested", gas, "cap", globalGasCap)
+		gas = globalGasCap
+	}
+	var (
+		gasPrice  *big.Int
+		gasFeeCap *big.Int
+		gasTipCap *big.Int
+	)
+	if baseFee == nil {
+		// If there's no basefee, then it must be a non-1559 execution
+		gasPrice = new(big.Int)
+		if arg.GasPrice != nil {
+			gasPrice = arg.GasPrice.ToInt()
+		}
+		gasFeeCap, gasTipCap = gasPrice, gasPrice
+	} else {
+		// A basefee is provided, necessitating 1559-type execution
+		if arg.GasPrice != nil {
+			// User specified the legacy gas field, convert to 1559 gas typing
+			gasPrice = arg.GasPrice.ToInt()
+			gasFeeCap, gasTipCap = gasPrice, gasPrice
+		} else {
+			// User specified 1559 gas feilds (or none), use those
+			gasFeeCap = new(big.Int)
+			if arg.MaxFeePerGas != nil {
+				gasFeeCap = arg.MaxFeePerGas.ToInt()
+			}
+			gasTipCap = new(big.Int)
+			if arg.MaxPriorityFeePerGas != nil {
+				gasTipCap = arg.MaxPriorityFeePerGas.ToInt()
+			}
+			// Backfill the legacy gasPrice for EVM execution, unless we're all zeroes
+			gasPrice = new(big.Int)
+			if gasFeeCap.BitLen() > 0 || gasTipCap.BitLen() > 0 {
+				gasPrice = math.BigMin(new(big.Int).Add(gasTipCap, baseFee), gasFeeCap)
+			}
+		}
+	}
+	value := new(big.Int)
+	if arg.Value != nil {
+		value = arg.Value.ToInt()
+	}
+	data := arg.data()
+	var accessList types.AccessList
+	if arg.AccessList != nil {
+		accessList = *arg.AccessList
+	}
+	msg := types.NewMessage(addr, arg.To, 0, value, gas, gasPrice, gasFeeCap, gasTipCap, data, accessList, true)
+	return msg, nil
+}
+
 // IPLDs is used to package raw IPLD block data fetched from IPFS and returned by the server
 // Returned by IPLDFetcher and ResponseFilterer
 type IPLDs struct {
 	BlockNumber     *big.Int
 	TotalDifficulty *big.Int
-	Header          ipfs.BlockModel
+	Header          models.IPLDModel
-	Uncles          []ipfs.BlockModel
+	Uncles          []models.IPLDModel
-	Transactions    []ipfs.BlockModel
+	Transactions    []models.IPLDModel
-	Receipts        []ipfs.BlockModel
+	Receipts        []models.IPLDModel
 	StateNodes      []StateNode
 	StorageNodes    []StorageNode
 }
 
 type StateNode struct {
-	Type         statediff.NodeType
+	Type         sdtypes.NodeType
 	StateLeafKey common.Hash
 	Path         []byte
-	IPLD         ipfs.BlockModel
+	IPLD         models.IPLDModel
 }
 
 type StorageNode struct {
-	Type           statediff.NodeType
+	Type           sdtypes.NodeType
 	StateLeafKey   common.Hash
 	StorageLeafKey common.Hash
 	Path           []byte
-	IPLD           ipfs.BlockModel
+	IPLD           models.IPLDModel
 }
 
 // CIDWrapper is used to direct fetching of IPLDs from IPFS
@@ -58,10 +224,43 @@ type StorageNode struct {
 // Passed to IPLDFetcher
 type CIDWrapper struct {
 	BlockNumber  *big.Int
-	Header       eth.HeaderModel
+	Header       models.HeaderModel
-	Uncles       []eth.UncleModel
+	Uncles       []models.UncleModel
-	Transactions []eth.TxModel
+	Transactions []models.TxModel
-	Receipts     []eth.ReceiptModel
+	Receipts     []models.ReceiptModel
-	StateNodes   []eth.StateNodeModel
+	StateNodes   []models.StateNodeModel
-	StorageNodes []eth.StorageNodeWithStateKeyModel
+	StorageNodes []models.StorageNodeWithStateKeyModel
+}
+
+// ConvertedPayload is a custom type which packages raw ETH data for publishing to IPFS and filtering to subscribers
+// Returned by PayloadConverter
+// Passed to IPLDPublisher and ResponseFilterer
+type ConvertedPayload struct {
+	TotalDifficulty *big.Int
+	Block           *types.Block
+	TxMetaData      []models.TxModel
+	Receipts        types.Receipts
+	ReceiptMetaData []models.ReceiptModel
+	StateNodes      []sdtypes.StateNode
+	StorageNodes    map[string][]sdtypes.StorageNode
+}
+
+// LogResult represent a log.
+type LogResult struct {
+	LeafCID     string `db:"leaf_cid"`
+	ReceiptID   string `db:"rct_id"`
+	Address     string `db:"address"`
+	Index       int64  `db:"index"`
+	Data        []byte `db:"log_data"`
+	Topic0      string `db:"topic0"`
+	Topic1      string `db:"topic1"`
+	Topic2      string `db:"topic2"`
+	Topic3      string `db:"topic3"`
+	LogLeafData []byte `db:"data"`
+	RctCID      string `db:"cid"`
+	RctStatus   uint64 `db:"post_status"`
+	BlockNumber string `db:"block_number"`
+	BlockHash   string `db:"block_hash"`
+	TxnIndex    int64  `db:"txn_index"`
+	TxHash      string `db:"tx_hash"`
 }

pkg/graphql/client.go

@@ -0,0 +1,267 @@
+package graphql
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	gqlclient "github.com/machinebox/graphql"
+)
+
+type StorageResponse struct {
+	CID       string        `json:"cid"`
+	Value     common.Hash   `json:"value"`
+	IpldBlock hexutil.Bytes `json:"ipldBlock"`
+}
+
+type GetStorageAt struct {
+	Response StorageResponse `json:"getStorageAt"`
+}
+
+type LogResponse struct {
+	Topics      []common.Hash       `json:"topics"`
+	Data        hexutil.Bytes       `json:"data"`
+	Transaction TransactionResponse `json:"transaction"`
+	ReceiptCID  string              `json:"receiptCID"`
+	Status      int32               `json:"status"`
+}
+
+type TransactionResponse struct {
+	Hash common.Hash `json:"hash"`
+}
+
+type GetLogs struct {
+	Responses []LogResponse `json:"getLogs"`
+}
+
+type IPFSBlockResponse struct {
+	Key  string `json:"key"`
+	Data string `json:"data"`
+}
+
+type EthTransactionCIDResponse struct {
+	CID          string            `json:"cid"`
+	TxHash       string            `json:"txHash"`
+	Index        int32             `json:"index"`
+	Src          string            `json:"src"`
+	Dst          string            `json:"dst"`
+	BlockByMhKey IPFSBlockResponse `json:"blockByMhKey"`
+}
+
+type EthTransactionCIDByTxHash struct {
+	Response EthTransactionCIDResponse `json:"ethTransactionCidByTxHash"`
+}
+
+type EthTransactionCIDsByHeaderIdResponse struct {
+	Nodes []EthTransactionCIDResponse `json:"nodes"`
+}
+
+type EthHeaderCIDResponse struct {
+	CID                          string                               `json:"cid"`
+	BlockNumber                  BigInt                               `json:"blockNumber"`
+	BlockHash                    string                               `json:"blockHash"`
+	ParentHash                   string                               `json:"parentHash"`
+	Timestamp                    BigInt                               `json:"timestamp"`
+	StateRoot                    string                               `json:"stateRoot"`
+	Td                           BigInt                               `json:"td"`
+	TxRoot                       string                               `json:"txRoot"`
+	ReceiptRoot                  string                               `json:"receiptRoot"`
+	UncleRoot                    string                               `json:"uncleRoot"`
+	Bloom                        string                               `json:"bloom"`
+	EthTransactionCIDsByHeaderId EthTransactionCIDsByHeaderIdResponse `json:"ethTransactionCidsByHeaderId"`
+	BlockByMhKey                 IPFSBlockResponse                    `json:"blockByMhKey"`
+}
+
+type AllEthHeaderCIDsResponse struct {
+	Nodes []EthHeaderCIDResponse `json:"nodes"`
+}
+
+type AllEthHeaderCIDs struct {
+	Response AllEthHeaderCIDsResponse `json:"allEthHeaderCids"`
+}
+
+type Client struct {
+	client *gqlclient.Client
+}
+
+func NewClient(endpoint string) *Client {
+	client := gqlclient.NewClient(endpoint)
+	return &Client{client: client}
+}
+
+func (c *Client) GetLogs(ctx context.Context, hash common.Hash, address *common.Address) ([]LogResponse, error) {
+	params := fmt.Sprintf(`blockHash: "%s"`, hash.String())
+	if address != nil {
+		params += fmt.Sprintf(`, contract: "%s"`, address.String())
+	}
+
+	getLogsQuery := fmt.Sprintf(`query{
+			getLogs(%s) {
+				data
+				topics
+				transaction {
+					hash
+				}
+				status
+				receiptCID
+			}
+		}`, params)
+
+	req := gqlclient.NewRequest(getLogsQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var logs GetLogs
+	err = json.Unmarshal(jsonStr, &logs)
+	if err != nil {
+		return nil, err
+	}
+	return logs.Responses, nil
+}
+
+func (c *Client) GetStorageAt(ctx context.Context, hash common.Hash, address common.Address, slot string) (*StorageResponse, error) {
+	getLogsQuery := fmt.Sprintf(`
+		query{
+			getStorageAt(blockHash: "%s", contract: "%s",slot: "%s") {
+				cid
+				value
+				ipldBlock
+			}
+		}
+	`, hash.String(), address.String(), common.HexToHash(slot))
+
+	req := gqlclient.NewRequest(getLogsQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var storageAt GetStorageAt
+	err = json.Unmarshal(jsonStr, &storageAt)
+	if err != nil {
+		return nil, err
+	}
+	return &storageAt.Response, nil
+}
+
+func (c *Client) AllEthHeaderCIDs(ctx context.Context, condition EthHeaderCIDCondition) (*AllEthHeaderCIDsResponse, error) {
+	var params string
+	if condition.BlockHash != nil {
+		params = fmt.Sprintf(`blockHash: "%s"`, *condition.BlockHash)
+	}
+	if condition.BlockNumber != nil {
+		params += fmt.Sprintf(`blockNumber: "%s"`, condition.BlockNumber.String())
+	}
+
+	getHeadersQuery := fmt.Sprintf(`
+		query{
+			allEthHeaderCids(condition: { %s }) {
+				nodes {
+					cid
+					blockNumber
+					blockHash
+					parentHash
+					timestamp
+					stateRoot
+					td
+					txRoot
+					receiptRoot
+					uncleRoot
+					bloom
+					blockByMhKey {
+						key
+						data
+					}
+					ethTransactionCidsByHeaderId {
+						nodes {
+							cid
+							txHash
+							index
+							src
+							dst
+						}
+					}
+				}
+			}
+		}
+	`, params)
+
+	req := gqlclient.NewRequest(getHeadersQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var allEthHeaderCIDs AllEthHeaderCIDs
+	err = json.Unmarshal(jsonStr, &allEthHeaderCIDs)
+	if err != nil {
+		return nil, err
+	}
+	return &allEthHeaderCIDs.Response, nil
+}
+
+func (c *Client) EthTransactionCIDByTxHash(ctx context.Context, txHash string) (*EthTransactionCIDResponse, error) {
+	getTxQuery := fmt.Sprintf(`
+		query{
+			ethTransactionCidByTxHash(txHash: "%s") {
+				cid
+				txHash
+				index
+				src
+				dst
+				blockByMhKey {
+					data
+				}
+			}
+		}
+	`, txHash)
+
+	req := gqlclient.NewRequest(getTxQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var ethTxCID EthTransactionCIDByTxHash
+	err = json.Unmarshal(jsonStr, &ethTxCID)
+	if err != nil {
+		return nil, err
+	}
+	return &ethTxCID.Response, nil
+}

pkg/graphql/graphiql.go

@@ -0,0 +1,114 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+)
+
+// GraphiQL is an in-browser IDE for exploring GraphiQL APIs.
+// This handler returns GraphiQL when requested.
+//
+// For more information, see https://github.com/graphql/graphiql.
+type GraphiQL struct{}
+
+func respond(w http.ResponseWriter, body []byte, code int) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.WriteHeader(code)
+	_, _ = w.Write(body)
+}
+
+func errorJSON(msg string) []byte {
+	buf := bytes.Buffer{}
+	fmt.Fprintf(&buf, `{"error": "%s"}`, msg)
+	return buf.Bytes()
+}
+
+func (h GraphiQL) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		respond(w, errorJSON("only GET requests are supported"), http.StatusMethodNotAllowed)
+		return
+	}
+	w.Header().Set("Content-Type", "text/html")
+	w.Write(graphiql)
+}
+
+var graphiql = []byte(`
+<!DOCTYPE html>
+<html>
+	<head>
+		<link
+                rel="icon"
+                type="image/png"
+                href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAAlwSFlzAAALEwAACxMBAJqcGAAAActpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx4bXA6Q3JlYXRvclRvb2w+QWRvYmUgSW1hZ2VSZWFkeTwveG1wOkNyZWF0b3JUb29sPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaWVudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KKS7NPQAAB5FJREFUWAm1FmtsnEdxdr/vfC8/mpgEfHYa6gaUJqAihfhVO7UprSokHsn5jKgKiKLGIIEEbSlpJdQLIJw+UFFQUSuBWir1z9nnpEmgCUnkcxPSmDRCgkKpGoJpfXdxHSc4ftzr2x1m9rvPPQdDDSgrfd/O7szOe2YX4H8cGEtY3tFK2Nu7pjMCChbgzVfD11h4XLKAibahL6dbBv+SaRl6LUsw78XBxTG80mEsWSkxu1oM9qmJlkR7UPhPWSDJCzSISw5zXZGxvpMezUp5GmtWQszunpiAKiPPZ20KyCqY1/ncgs4v+IUPwLJvYhzTVIbmvXgvqwAxkImKJHt1yzM+AQLXvdKXy3QevB4R+3O6wIYHSUCIlABEtTO9bf86pmFa7B6xPeHMi3l668p5SQjInbRGQQw0E3FMH4FHaFPoP8USVaveEo9aaH3LsdRh2vsYKqwhMhRBKw82vGbNQbcC9ePL1+PDmwf7iix0N+xmPoafq4TgDDaRYxmLCrBwD5HpSK4vKRVeP9b3ZyaaaE18UaL4KYE5x5afsWxoBgefFfX+jX6pMH9RvSnX2v1YxPP4D3UAHG2hgm80vRp7ns9nWxOb8kIt3HD6C+O8rpRVoYCxHDOtQwOg4QHS1kIb9oHGVQJlN0h8qPF07FFmkG4byouAjEdSO/bwOntr8kGt8EeNJ3uN27O37fse5PT3lVIjUsrL6MB2IVCThMcbx3ofIt7sZeMFExeTubSR3Zq4tVoEdhHSJs30WqjbIS1Zk6/VqzzhmdbBpyn5p1g4W8LMGkajj9GUSfcM/4IVaji+/QdOa7hehKz69xEPsllLkFZY+HdlWhOdLNxrXm5iTK1xPSHEeo4KxTFPzEsFLHH8D914rG+GGWe2Dd9UJav6ZbW1k9ep7rgF3SnTEUXA3hko2fdkowc2M27dk3deomgfLBIPYlJytC4QLzKLZdAoy3QzNTVqksT2y6Oz+YVL1TK4Oo9FYAVIkRFzgH8F/bOiD0cjv4m+hEA9IdXn8HaC4Mjxzx7OdCZH8R14mra6eB9sfUKTj4SCQLUvCHMqN235rKMGV5ZpPCAoSzGOcs2JaFZYVuc8FF5XQl8uCHV75FT0ZT6Q6Ry+02fZ3b7agLF+MGbYmF/Mg+vE14NY1Xnhjv2fZkTkWO+R2VXqc1BrLczp/OtULV0fOLXjHS5LlvkuhzL05oZf+xnMbtv3BLXZIwyPQNx4iRLvrXRXci/vcV/guXJ4dZ/elnwqfctQlnFxoGyhkY2+eCbTlnyCYU8GwzzcHHBhmKl7261X1CEBaIT0QNxJdyQfpLRdHblt4wNMeuhsVpWPvDulqAXQKH5i9f0Ut7pMT/LhOEWc96hfkBEYYnhDU3DJ2SUKMAEPIagRoTSJObF9uF5oHAC/uF/ENxeRrPcai0vt/k1mE+6GeE9eVIlvQwF+yGfL/KiNuMpUnmF4WQUYwX3AEEzjXmqi5yOp6DO8hrM7TeIZ+Orf2X6DY1oU+FeY1D8xJLh8G2bcsgpQ3vqoAU1P3nWouQaDd8mQdS8Tj1B/Z0sZXm6QyxbvAFlj3Us95e7Jbx6/EYScpnP/kjfMwy3DMre6mXVGIVTqiqi1mtVk8blZR78UOdGbQqDLheLMjWc54Yt7KSAaUvRwTyrdMXREvFF6VtRZfgrALNOcm8ixZxe9uOgBLsMPnftUIdM+tBFKcLtwxCeJ7GbdHDJlJ6DHYetX8gHfSTTEB4P9WNBb5JRq0VrfwbxZRuVN61pMt56ICz3elWxAB18OS//Nep4MKeowTOU/zMwo8RaV5fVKhs4WN1DzCjkzJV1jBT9K1TB6oWN4bR89arDMz7iTa1ikepxsy+CXqmXol1fUfJ4qwUfeptsXL1JNTFNWXkfmO5ydi8KXBIMWvCYnmbOWmKXr5zpZhHotSbQGp9YO+qkb3h05E3vBk+nmwJopw5SSdVxRsOjiCGhEXSMCMFdTrAdbPikul35PvWAN1adPgqAGz8Kk1FLTX2hlCyF9pHSIQlwnp+x6/yb1t9zu8LgFszJHt5v0K+TakuPmbFnmog2cXBzfbFtyj1b6O4SQ4BP76Zr1k1Etwoe7Ir+N/dwcfo8f3QnbsYR7yAO/kxICdAH1En+km/WxhtPRXZ4sZrOoQBk2npjcmmwu2ipMz6s/MlG6JflVqrC9pN8VqLK+1nhix4u8/3Z7YjXPRHeJ52z3vm7Mq6eISa0UeF/DK7FB3r/w8eGP0Htg4f1noud5TXgy1g1lpQIGQelGyLjbQk3J7TZr8yT7uxzwSfu+oiwdIL//gTKc+4MUltxL/lpPFn+ebvqByFhswAjid+VgTLNnXcGcyHGuY7PmvWUHZ2hlqXgXDRNfbD/YSE+2MeeWYzjZMmw+p+MYpnuSJy/FjtZ5DCvPuI9SFv5/DI4buZxfwZBuH7pnpu0QprcOztM3N9v2K8x2DH+FcZktB/nSWeJZ3v93Y8VasRubmqBoGKF4g6oBwjIQoi/MMDrqHOMamnMFmv6ziw0T97diTb0zHB7OEe4ZlCjf5X2U8vGm09HnKrPbo78mMwu6mjFn9tV713TtvWpZSCX83wr9J1EKd8CrhC26AAAAAElFTkSuQmCC"
+        />
+        <link
+                rel="stylesheet"
+                href="https://cdnjs.cloudflare.com/ajax/libs/graphiql/0.13.0/graphiql.css"
+                integrity="sha384-Qua2xoKBxcHOg1ivsKWo98zSI5KD/UuBpzMIg8coBd4/jGYoxeozCYFI9fesatT0"
+                crossorigin="anonymous"
+        />
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/fetch/3.0.0/fetch.min.js"
+                integrity="sha384-5B8/4F9AQqp/HCHReGLSOWbyAOwnJsPrvx6C0+VPUr44Olzi99zYT1xbVh+ZanQJ"
+                crossorigin="anonymous"
+        ></script>
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/react/16.8.5/umd/react.production.min.js"
+                integrity="sha384-dOCiLz3nZfHiJj//EWxjwSKSC6Z1IJtyIEK/b/xlHVNdVLXDYSesoxiZb94bbuGE"
+                crossorigin="anonymous"
+        ></script>
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/react-dom/16.8.5/umd/react-dom.production.min.js"
+                integrity="sha384-QI+ql5f+khgo3mMdCktQ3E7wUKbIpuQo8S5rA/3i1jg2rMsloCNyiZclI7sFQUGN"
+                crossorigin="anonymous"
+        ></script>
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/graphiql/0.13.0/graphiql.min.js"
+                integrity="sha384-roSmzNmO4zJK9X4lwggDi4/oVy+9V4nlS1+MN8Taj7tftJy1GvMWyAhTNXdC/fFR"
+                crossorigin="anonymous"
+        ></script>
+	</head>
+	<body style="width: 100%; height: 100%; margin: 0; overflow: hidden;">
+		<div id="graphiql" style="height: 100vh;">Loading...</div>
+		<script>
+			function fetchGQL(params) {
+				return fetch("/graphql", {
+					method: "post",
+					body: JSON.stringify(params),
+					credentials: "include",
+				}).then(function (resp) {
+					return resp.text();
+				}).then(function (body) {
+					try {
+						return JSON.parse(body);
+					} catch (error) {
+						return body;
+					}
+				});
+			}
+			ReactDOM.render(
+				React.createElement(GraphiQL, {fetcher: fetchGQL}),
+				document.getElementById("graphiql")
+			)
+		</script>
+	</body>
+</html>
+`)

pkg/graphql/graphql_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
+)
+
+func TestGraphQL(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "graphql test suite")
+}
+
+var _ = BeforeSuite(func() {
+	logrus.SetOutput(ioutil.Discard)
+})

pkg/graphql/graphql_test.go

@@ -0,0 +1,334 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql_test
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"strconv"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth/test_helpers"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/graphql"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+	ethServerShared "github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
+)
+
+var _ = Describe("GraphQL", func() {
+	const (
+		gqlEndPoint = "127.0.0.1:8083"
+	)
+	var (
+		randomAddr      = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
+		randomHash      = crypto.Keccak256Hash(randomAddr.Bytes())
+		blocks          []*types.Block
+		receipts        []types.Receipts
+		chain           *core.BlockChain
+		db              *sqlx.DB
+		blockHashes     []common.Hash
+		backend         *eth.Backend
+		graphQLServer   *graphql.Service
+		chainConfig     = params.TestChainConfig
+		mockTD          = big.NewInt(1337)
+		client          = graphql.NewClient(fmt.Sprintf("http://%s/graphql", gqlEndPoint))
+		ctx             = context.Background()
+		blockHash       common.Hash
+		contractAddress common.Address
+	)
+
+	It("test init", func() {
+		var err error
+		db = shared.SetupDB()
+		transformer := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
+
+		backend, err = eth.NewEthBackend(db, &eth.Config{
+			ChainConfig: chainConfig,
+			VMConfig:    vm.Config{},
+			RPCGasCap:   big.NewInt(10000000000),
+			GroupCacheConfig: &ethServerShared.GroupCacheConfig{
+				StateDB: ethServerShared.GroupConfig{
+					Name:                   "graphql_test",
+					CacheSizeInMB:          8,
+					CacheExpiryInMins:      60,
+					LogStatsIntervalInSecs: 0,
+				},
+			},
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		// make the test blockchain (and state)
+		blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen)
+		params := statediff.Params{
+			IntermediateStateNodes:   true,
+			IntermediateStorageNodes: true,
+		}
+
+		// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
+		builder := statediff.NewBuilder(chain.StateCache())
+		for i, block := range blocks {
+			blockHashes = append(blockHashes, block.Hash())
+			var args statediff.Args
+			var rcts types.Receipts
+			if i == 0 {
+				args = statediff.Args{
+					OldStateRoot: common.Hash{},
+					NewStateRoot: block.Root(),
+					BlockNumber:  block.Number(),
+					BlockHash:    block.Hash(),
+				}
+			} else {
+				args = statediff.Args{
+					OldStateRoot: blocks[i-1].Root(),
+					NewStateRoot: block.Root(),
+					BlockNumber:  block.Number(),
+					BlockHash:    block.Hash(),
+				}
+				rcts = receipts[i-1]
+			}
+
+			var diff sdtypes.StateObject
+			diff, err = builder.BuildStateDiffObject(args, params)
+			Expect(err).ToNot(HaveOccurred())
+
+			tx, err := transformer.PushBlock(block, rcts, mockTD)
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, node := range diff.Nodes {
+				err = transformer.PushStateNode(tx, node, block.Hash().String())
+				Expect(err).ToNot(HaveOccurred())
+			}
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		// Insert some non-canonical data into the database so that we test our ability to discern canonicity
+		indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
+
+		blockHash = test_helpers.MockBlock.Hash()
+		contractAddress = test_helpers.ContractAddr
+
+		tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+		Expect(err).ToNot(HaveOccurred())
+
+		err = tx.Submit(err)
+		Expect(err).ToNot(HaveOccurred())
+
+		// The non-canonical header has a child
+		tx, err = indexAndPublisher.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
+		Expect(err).ToNot(HaveOccurred())
+
+		ccHash := sdtypes.CodeAndCodeHash{
+			Hash: test_helpers.CodeHash,
+			Code: test_helpers.ContractCode,
+		}
+
+		err = indexAndPublisher.PushCodeAndCodeHash(tx, ccHash)
+		Expect(err).ToNot(HaveOccurred())
+
+		err = tx.Submit(err)
+		Expect(err).ToNot(HaveOccurred())
+
+		graphQLServer, err = graphql.New(backend, gqlEndPoint, nil, []string{"*"}, rpc.HTTPTimeouts{})
+		Expect(err).ToNot(HaveOccurred())
+
+		err = graphQLServer.Start(nil)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	defer It("test teardown", func() {
+		err := graphQLServer.Stop()
+		Expect(err).ToNot(HaveOccurred())
+		shared.TearDownDB(db)
+		chain.Stop()
+	})
+
+	Describe("eth_getLogs", func() {
+		It("Retrieves logs that matches the provided blockHash and contract address", func() {
+			logs, err := client.GetLogs(ctx, blockHash, &contractAddress)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedLogs := []graphql.LogResponse{
+				{
+					Topics:      test_helpers.MockLog1.Topics,
+					Data:        hexutil.Bytes(test_helpers.MockLog1.Data),
+					Transaction: graphql.TransactionResponse{Hash: test_helpers.MockTransactions[0].Hash()},
+					ReceiptCID:  test_helpers.Rct1CID.String(),
+					Status:      int32(test_helpers.MockReceipts[0].Status),
+				},
+			}
+
+			Expect(logs).To(Equal(expectedLogs))
+		})
+
+		It("Retrieves logs for the failed receipt status that matches the provided blockHash and another contract address", func() {
+			logs, err := client.GetLogs(ctx, blockHash, &test_helpers.AnotherAddress2)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedLogs := []graphql.LogResponse{
+				{
+					Topics:      test_helpers.MockLog6.Topics,
+					Data:        hexutil.Bytes(test_helpers.MockLog6.Data),
+					Transaction: graphql.TransactionResponse{Hash: test_helpers.MockTransactions[3].Hash()},
+					ReceiptCID:  test_helpers.Rct4CID.String(),
+					Status:      int32(test_helpers.MockReceipts[3].Status),
+				},
+			}
+
+			Expect(logs).To(Equal(expectedLogs))
+		})
+
+		It("Retrieves all the logs for the receipt that matches the provided blockHash and nil contract address", func() {
+			logs, err := client.GetLogs(ctx, blockHash, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(6))
+		})
+
+		It("Retrieves logs with random hash", func() {
+			logs, err := client.GetLogs(ctx, randomHash, &contractAddress)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(0))
+		})
+	})
+
+	Describe("eth_getStorageAt", func() {
+		It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash", func() {
+			storageRes, err := client.GetStorageAt(ctx, blockHashes[2], contractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.HexToHash("01")))
+
+			storageRes, err = client.GetStorageAt(ctx, blockHashes[3], contractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.HexToHash("03")))
+
+			storageRes, err = client.GetStorageAt(ctx, blockHashes[4], contractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.HexToHash("09")))
+		})
+
+		It("Retrieves empty data if it tries to access a contract at a blockHash which does not exist", func() {
+			storageRes, err := client.GetStorageAt(ctx, blockHashes[0], contractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.Hash{}))
+
+			storageRes, err = client.GetStorageAt(ctx, blockHashes[1], contractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.Hash{}))
+		})
+
+		It("Retrieves empty data if it tries to access a contract slot which does not exist", func() {
+			storageRes, err := client.GetStorageAt(ctx, blockHashes[3], contractAddress, randomHash.Hex())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.Hash{}))
+		})
+	})
+
+	Describe("allEthHeaderCids", func() {
+		It("Retrieves header_cids that matches the provided blockNumber", func() {
+			allEthHeaderCIDsResp, err := client.AllEthHeaderCIDs(ctx, graphql.EthHeaderCIDCondition{BlockNumber: new(graphql.BigInt).SetUint64(2)})
+			Expect(err).ToNot(HaveOccurred())
+
+			headerCIDs, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockNumber(2)
+			Expect(err).ToNot(HaveOccurred())
+
+			for idx, headerCID := range headerCIDs {
+				ethHeaderCID := allEthHeaderCIDsResp.Nodes[idx]
+
+				compareEthHeaderCID(ethHeaderCID, headerCID)
+			}
+		})
+
+		It("Retrieves header_cids that matches the provided blockHash", func() {
+			blockHash := blocks[1].Hash().String()
+			allEthHeaderCIDsResp, err := client.AllEthHeaderCIDs(ctx, graphql.EthHeaderCIDCondition{BlockHash: &blockHash})
+			Expect(err).ToNot(HaveOccurred())
+
+			headerCID, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(blocks[1].Hash())
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(allEthHeaderCIDsResp.Nodes)).To(Equal(1))
+			ethHeaderCID := allEthHeaderCIDsResp.Nodes[0]
+			compareEthHeaderCID(ethHeaderCID, headerCID)
+		})
+	})
+
+	Describe("ethTransactionCidByTxHash", func() {
+		It("Retrieves tx_cid that matches the provided txHash", func() {
+			txHash := blocks[2].Transactions()[0].Hash().String()
+			ethTransactionCIDResp, err := client.EthTransactionCIDByTxHash(ctx, txHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			txCID, err := backend.Retriever.RetrieveTxCIDByHash(txHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			compareEthTxCID(*ethTransactionCIDResp, txCID)
+
+			Expect(ethTransactionCIDResp.BlockByMhKey.Data).To(Equal(graphql.Bytes(txCID.IPLD.Data).String()))
+		})
+	})
+})
+
+func compareEthHeaderCID(ethHeaderCID graphql.EthHeaderCIDResponse, headerCID eth.HeaderCIDRecord) {
+	blockNumber, err := strconv.ParseInt(headerCID.BlockNumber, 10, 64)
+	Expect(err).ToNot(HaveOccurred())
+
+	td, err := strconv.ParseInt(headerCID.TotalDifficulty, 10, 64)
+	Expect(err).ToNot(HaveOccurred())
+
+	Expect(ethHeaderCID.CID).To(Equal(headerCID.CID))
+	Expect(ethHeaderCID.BlockNumber).To(Equal(*new(graphql.BigInt).SetUint64(uint64(blockNumber))))
+	Expect(ethHeaderCID.BlockHash).To(Equal(headerCID.BlockHash))
+	Expect(ethHeaderCID.ParentHash).To(Equal(headerCID.ParentHash))
+	Expect(ethHeaderCID.Timestamp).To(Equal(*new(graphql.BigInt).SetUint64(headerCID.Timestamp)))
+	Expect(ethHeaderCID.StateRoot).To(Equal(headerCID.StateRoot))
+	Expect(ethHeaderCID.Td).To(Equal(*new(graphql.BigInt).SetUint64(uint64(td))))
+	Expect(ethHeaderCID.TxRoot).To(Equal(headerCID.TxRoot))
+	Expect(ethHeaderCID.ReceiptRoot).To(Equal(headerCID.RctRoot))
+	Expect(ethHeaderCID.UncleRoot).To(Equal(headerCID.UncleRoot))
+	Expect(ethHeaderCID.Bloom).To(Equal(graphql.Bytes(headerCID.Bloom).String()))
+
+	for tIdx, txCID := range headerCID.TransactionCIDs {
+		ethTxCID := ethHeaderCID.EthTransactionCIDsByHeaderId.Nodes[tIdx]
+		compareEthTxCID(ethTxCID, txCID)
+	}
+
+	Expect(ethHeaderCID.BlockByMhKey.Data).To(Equal(graphql.Bytes(headerCID.IPLD.Data).String()))
+	Expect(ethHeaderCID.BlockByMhKey.Key).To(Equal(headerCID.IPLD.Key))
+}
+
+func compareEthTxCID(ethTxCID graphql.EthTransactionCIDResponse, txCID eth.TransactionCIDRecord) {
+	Expect(ethTxCID.CID).To(Equal(txCID.CID))
+	Expect(ethTxCID.TxHash).To(Equal(txCID.TxHash))
+	Expect(ethTxCID.Index).To(Equal(int32(txCID.Index)))
+	Expect(ethTxCID.Src).To(Equal(txCID.Src))
+	Expect(ethTxCID.Dst).To(Equal(txCID.Dst))
+}

pkg/graphql/schema.go

@@ -0,0 +1,354 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+const schema string = `
+    # Bytes32 is a 32 byte binary string, represented as 0x-prefixed hexadecimal.
+    scalar Bytes32
+    # Address is a 20 byte Ethereum address, represented as 0x-prefixed hexadecimal.
+    scalar Address
+    # Bytes is an arbitrary length binary string, represented as 0x-prefixed hexadecimal.
+    # An empty byte string is represented as '0x'. Byte strings must have an even number of hexadecimal nybbles.
+    scalar Bytes
+    # BigInt is a large integer. Input is accepted as either a JSON number or as a string.
+    # Input and output strings may be either decimal or 0x-prefixed hexadecimal depending upon the resolver implementation.
+    scalar BigInt
+    # Long is a 64 bit unsigned integer.
+    scalar Long
+
+    schema {
+        query: Query
+    }
+
+    # Account is an Ethereum account at a particular block.
+    type Account {
+        # Address is the address owning the account.
+        address: Address!
+        # Balance is the balance of the account, in wei.
+        balance: BigInt!
+        # TransactionCount is the number of transactions sent from this account,
+        # or in the case of a contract, the number of contracts created. Otherwise
+        # known as the nonce.
+        transactionCount: Long!
+        # Code contains the smart contract code for this account, if the account
+        # is a (non-self-destructed) contract.
+        code: Bytes!
+        # Storage provides access to the storage of a contract account, indexed
+        # by its 32 byte slot identifier.
+        storage(slot: Bytes32!): Bytes32!
+    }
+
+    # Log is an Ethereum event log.
+    type Log {
+        # Index is the index of this log in the block.
+        index: Int!
+        # Account is the account which generated this log - this will always
+        # be a contract account.
+        account(block: Long): Account!
+        # Topics is a list of 0-4 indexed topics for the log.
+        topics: [Bytes32!]!
+        # Data is unindexed data for this log.
+        data: Bytes!
+        # Transaction is the transaction that generated this log entry.
+        transaction: Transaction
+
+        # CID for the leaf node IPLD block of the log.
+        cid: String!
+
+        # ReceiptCID for the Receipt IPLD block this Log exists in.
+        receiptCID: String!
+
+        # IPLD block data for the Log Leaf node.
+        ipldBlock: Bytes!
+
+        # Status of the Receipt IPLD block this Log exists in.
+        status: Int!
+    }
+
+    # Transaction is an Ethereum transaction.
+    type Transaction {
+        # Hash is the hash of this transaction.
+        hash: Bytes32!
+        # Nonce is the nonce of the account this transaction was generated with.
+        nonce: Long!
+        # Index is the index of this transaction in the parent block. This will
+        # be null if the transaction has not yet been mined.
+        index: Int
+        # From is the account that sent this transaction - this will always be
+        # an externally owned account.
+        from(block: Long): Account!
+        # To is the account the transaction was sent to. This is null for
+        # contract-creating transactions.
+        to(block: Long): Account
+        # Value is the value, in wei, sent along with this transaction.
+        value: BigInt!
+        # GasPrice is the price offered to miners for gas, in wei per unit.
+        gasPrice: BigInt!
+        # Gas is the maximum amount of gas this transaction can consume.
+        gas: Long!
+        # InputData is the data supplied to the target of the transaction.
+        inputData: Bytes!
+        # Block is the block this transaction was mined in. This will be null if
+        # the transaction has not yet been mined.
+        block: Block
+
+        # Status is the return status of the transaction. This will be 1 if the
+        # transaction succeeded, or 0 if it failed (due to a revert, or due to
+        # running out of gas). If the transaction has not yet been mined, this
+        # field will be null.
+        status: Long
+        # GasUsed is the amount of gas that was used processing this transaction.
+        # If the transaction has not yet been mined, this field will be null.
+        gasUsed: Long
+        # CumulativeGasUsed is the total gas used in the block up to and including
+        # this transaction. If the transaction has not yet been mined, this field
+        # will be null.
+        cumulativeGasUsed: Long
+        # CreatedContract is the account that was created by a contract creation
+        # transaction. If the transaction was not a contract creation transaction,
+        # or it has not yet been mined, this field will be null.
+        createdContract(block: Long): Account
+        # Logs is a list of log entries emitted by this transaction. If the
+        # transaction has not yet been mined, this field will be null.
+        logs: [Log!]
+        r: BigInt!
+        s: BigInt!
+        v: BigInt!
+    }
+
+    # BlockFilterCriteria encapsulates log filter criteria for a filter applied
+    # to a single block.
+    input BlockFilterCriteria {
+        # Addresses is list of addresses that are of interest. If this list is
+        # empty, results will not be filtered by address.
+        addresses: [Address!]
+        # Topics list restricts matches to particular event topics. Each event has a list
+        # of topics. Topics matches a prefix of that list. An empty element array matches any
+        # topic. Non-empty elements represent an alternative that matches any of the
+        # contained topics.
+        #
+        # Examples:
+        #  - [] or nil          matches any topic list
+        #  - [[A]]              matches topic A in first position
+        #  - [[], [B]]          matches any topic in first position, B in second position
+        #  - [[A], [B]]         matches topic A in first position, B in second position
+        #  - [[A, B]], [C, D]]  matches topic (A OR B) in first position, (C OR D) in second position
+        topics: [[Bytes32!]!]
+    }
+
+    # Block is an Ethereum block.
+    type Block {
+        # Number is the number of this block, starting at 0 for the genesis block.
+        number: Long!
+        # Hash is the block hash of this block.
+        hash: Bytes32!
+        # Parent is the parent block of this block.
+        parent: Block
+        # Nonce is the block nonce, an 8 byte sequence determined by the miner.
+        nonce: Bytes!
+        # TransactionsRoot is the keccak256 hash of the root of the trie of transactions in this block.
+        transactionsRoot: Bytes32!
+        # TransactionCount is the number of transactions in this block. if
+        # transactions are not available for this block, this field will be null.
+        transactionCount: Int
+        # StateRoot is the keccak256 hash of the state trie after this block was processed.
+        stateRoot: Bytes32!
+        # ReceiptsRoot is the keccak256 hash of the trie of transaction receipts in this block.
+        receiptsRoot: Bytes32!
+        # Miner is the account that mined this block.
+        miner(block: Long): Account!
+        # ExtraData is an arbitrary data field supplied by the miner.
+        extraData: Bytes!
+        # GasLimit is the maximum amount of gas that was available to transactions in this block.
+        gasLimit: Long!
+        # GasUsed is the amount of gas that was used executing transactions in this block.
+        gasUsed: Long!
+        # Timestamp is the unix timestamp at which this block was mined.
+        timestamp: Long!
+        # LogsBloom is a bloom filter that can be used to check if a block may
+        # contain log entries matching a filter.
+        logsBloom: Bytes!
+        # MixHash is the hash that was used as an input to the PoW process.
+        mixHash: Bytes32!
+        # Difficulty is a measure of the difficulty of mining this block.
+        difficulty: BigInt!
+        # TotalDifficulty is the sum of all difficulty values up to and including
+        # this block.
+        totalDifficulty: BigInt!
+        # OmmerCount is the number of ommers (AKA uncles) associated with this
+        # block. If ommers are unavailable, this field will be null.
+        ommerCount: Int
+        # Ommers is a list of ommer (AKA uncle) blocks associated with this block.
+        # If ommers are unavailable, this field will be null. Depending on your
+        # node, the transactions, transactionAt, transactionCount, ommers,
+        # ommerCount and ommerAt fields may not be available on any ommer blocks.
+        ommers: [Block]
+        # OmmerAt returns the ommer (AKA uncle) at the specified index. If ommers
+        # are unavailable, or the index is out of bounds, this field will be null.
+        ommerAt(index: Int!): Block
+        # OmmerHash is the keccak256 hash of all the ommers (AKA uncles)
+        # associated with this block.
+        ommerHash: Bytes32!
+        # Transactions is a list of transactions associated with this block. If
+        # transactions are unavailable for this block, this field will be null.
+        transactions: [Transaction!]
+        # TransactionAt returns the transaction at the specified index. If
+        # transactions are unavailable for this block, or if the index is out of
+        # bounds, this field will be null.
+        transactionAt(index: Int!): Transaction
+        # Logs returns a filtered set of logs from this block.
+        logs(filter: BlockFilterCriteria!): [Log!]!
+        # Account fetches an Ethereum account at the current block's state.
+        account(address: Address!): Account!
+        # Call executes a local call operation at the current block's state.
+        call(data: CallData!): CallResult
+    }
+
+    # CallData represents the data associated with a local contract call.
+    # All fields are optional.
+    input CallData {
+        # From is the address making the call.
+        from: Address
+        # To is the address the call is sent to.
+        to: Address
+        # Gas is the amount of gas sent with the call.
+        gas: Long
+        # GasPrice is the price, in wei, offered for each unit of gas.
+        gasPrice: BigInt
+        # Value is the value, in wei, sent along with the call.
+        value: BigInt
+        # Data is the data sent to the callee.
+        data: Bytes
+    }
+
+    # CallResult is the result of a local call operation.
+    type CallResult {
+        # Data is the return data of the called contract.
+        data: Bytes!
+        # GasUsed is the amount of gas used by the call, after any refunds.
+        gasUsed: Long!
+        # Status is the result of the call - 1 for success or 0 for failure.
+        status: Long!
+    }
+
+    # FilterCriteria encapsulates log filter criteria for searching log entries.
+    input FilterCriteria {
+        # FromBlock is the block at which to start searching, inclusive. Defaults
+        # to the latest block if not supplied.
+        fromBlock: Long
+        # ToBlock is the block at which to stop searching, inclusive. Defaults
+        # to the latest block if not supplied.
+        toBlock: Long
+        # Addresses is a list of addresses that are of interest. If this list is
+        # empty, results will not be filtered by address.
+        addresses: [Address!]
+        # Topics list restricts matches to particular event topics. Each event has a list
+        # of topics. Topics matches a prefix of that list. An empty element array matches any
+        # topic. Non-empty elements represent an alternative that matches any of the
+        # contained topics.
+        #
+        # Examples:
+        #  - [] or nil          matches any topic list
+        #  - [[A]]              matches topic A in first position
+        #  - [[], [B]]          matches any topic in first position, B in second position
+        #  - [[A], [B]]         matches topic A in first position, B in second position
+        #  - [[A, B]], [C, D]]  matches topic (A OR B) in first position, (C OR D) in second position
+        topics: [[Bytes32!]!]
+    }
+
+    # Storage trie value with IPLD data.
+    type StorageResult {
+        value: Bytes32!
+
+        # CID for the storage trie IPLD block.
+        cid: String!
+
+        # Storage trie IPLD block.
+        ipldBlock: Bytes!
+    }
+
+    input EthHeaderCidCondition {
+        blockNumber: BigInt
+        blockHash: String
+    }
+
+    type EthTransactionCid {
+        cid: String!
+        txHash: String!
+        index: Int!
+        src: String!
+        dst: String!
+        blockByMhKey: IPFSBlock!
+    }
+
+    type EthTransactionCidsConnection {
+        nodes: [EthTransactionCid]!
+    }
+
+    type IPFSBlock {
+        key: String!
+        data: String!
+    }
+
+    type EthHeaderCid {
+        cid: String!
+        blockNumber: BigInt!
+        blockHash: String!
+        parentHash: String!
+        timestamp: BigInt!
+        stateRoot: String!
+        td: BigInt!
+        txRoot: String!
+        receiptRoot: String!
+        uncleRoot: String!
+        bloom: String!
+        ethTransactionCidsByHeaderId: EthTransactionCidsConnection!
+        blockByMhKey: IPFSBlock!
+    }
+
+    type EthHeaderCidsConnection {
+        nodes: [EthHeaderCid]!
+    }
+
+    type Query {
+        # Block fetches an Ethereum block by number or by hash. If neither is
+        # supplied, the most recent known block is returned.
+        block(number: Long, hash: Bytes32): Block
+
+        # Blocks returns all the blocks between two numbers, inclusive. If
+        # to is not supplied, it defaults to the most recent known block.
+        blocks(from: Long!, to: Long): [Block!]!
+
+        # Transaction returns a transaction specified by its hash.
+        transaction(hash: Bytes32!): Transaction
+
+        # Logs returns log entries matching the provided filter.
+        logs(filter: FilterCriteria!): [Log!]!
+
+        # Get storage slot by block hash and contract address.
+        getStorageAt(blockHash: Bytes32!, contract: Address!, slot: Bytes32!): StorageResult
+
+        # Get contract logs by block hash and contract address.
+        getLogs(blockHash: Bytes32!, contract: Address): [Log!]
+
+        # PostGraphile alternative to get headers with transactions using block number or block hash.
+        allEthHeaderCids(condition: EthHeaderCidCondition): EthHeaderCidsConnection
+
+        # PostGraphile alternative to get transactions using transaction hash.
+        ethTransactionCidByTxHash(txHash: String!): EthTransactionCid
+    }
+`

pkg/graphql/service.go

@@ -0,0 +1,111 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/graph-gophers/graphql-go"
+	"github.com/graph-gophers/graphql-go/relay"
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
+)
+
+// Service encapsulates a GraphQL service.
+type Service struct {
+	endpoint string           // The host:port endpoint for this service.
+	cors     []string         // Allowed CORS domains
+	vhosts   []string         // Recognised vhosts
+	timeouts rpc.HTTPTimeouts // Timeout settings for HTTP requests.
+	backend  *eth.Backend     // The backend that queries will operate onn.
+	handler  http.Handler     // The `http.Handler` used to answer queries.
+	listener net.Listener     // The listening socket.
+}
+
+// New constructs a new GraphQL service instance.
+func New(backend *eth.Backend, endpoint string, cors, vhosts []string, timeouts rpc.HTTPTimeouts) (*Service, error) {
+	return &Service{
+		endpoint: endpoint,
+		cors:     cors,
+		vhosts:   vhosts,
+		timeouts: timeouts,
+		backend:  backend,
+	}, nil
+}
+
+// Protocols returns the list of protocols exported by this service.
+func (s *Service) Protocols() []p2p.Protocol { return nil }
+
+// APIs returns the list of APIs exported by this service.
+func (s *Service) APIs() []rpc.API { return nil }
+
+// Start is called after all services have been constructed and the networking
+// layer was also initialized to spawn any goroutines required by the service.
+func (s *Service) Start(server *p2p.Server) error {
+	var err error
+	s.handler, err = NewHandler(s.backend)
+	if err != nil {
+		return err
+	}
+
+	handler := node.NewHTTPHandlerStack(s.handler, s.cors, s.vhosts, nil)
+
+	// start http server
+	_, addr, err := node.StartHTTPEndpoint(s.endpoint, rpc.DefaultHTTPTimeouts, handler)
+	if err != nil {
+		utils.Fatalf("Could not start RPC api: %v", err)
+	}
+	extapiURL := fmt.Sprintf("http://%v/", addr)
+	logrus.Infof("graphQL endpoint opened for url %s", extapiURL)
+	return nil
+}
+
+// newHandler returns a new `http.Handler` that will answer GraphQL queries.
+// It additionally exports an interactive query browser on the / endpoint.
+func NewHandler(backend *eth.Backend) (http.Handler, error) {
+	q := Resolver{backend}
+
+	s, err := graphql.ParseSchema(schema, &q)
+	if err != nil {
+		return nil, err
+	}
+	h := &relay.Handler{Schema: s}
+
+	mux := http.NewServeMux()
+	mux.Handle("/", GraphiQL{})
+	mux.Handle("/graphql", h)
+	mux.Handle("/graphql/", h)
+	return mux, nil
+}
+
+// Stop terminates all goroutines belonging to the service, blocking until they
+// are all terminated.
+func (s *Service) Stop() error {
+	if s.listener != nil {
+		s.listener.Close()
+		s.listener = nil
+		logrus.Debugf("graphQL endpoint closed for url %s", fmt.Sprintf("http://%s", s.endpoint))
+	}
+	return nil
+}

pkg/graphql/types.go

@@ -0,0 +1,122 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+import (
+	"encoding/hex"
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+// Bytes marshals as a JSON string with \x prefix.
+// The empty slice marshals as "\x".
+type Bytes []byte
+
+// MarshalText implements encoding.TextMarshaler
+func (b Bytes) MarshalText() ([]byte, error) {
+	result := make([]byte, len(b)*2+2)
+	copy(result, `\x`)
+	hex.Encode(result[2:], b)
+	return result, nil
+}
+
+// String returns the hex encoding of b.
+func (b Bytes) String() string {
+	return b.encode()
+}
+
+// Encode encodes b as a hex string with "\x" prefix.
+// This is to make the output to be the same as given by postgraphile.
+// graphql-go prepends another "\" to the output resulting in prefix "\\x".
+func (b Bytes) encode() string {
+	result := make([]byte, len(b)*2+2)
+	copy(result, `\x`)
+	hex.Encode(result[2:], b)
+	return string(result)
+}
+
+type BigInt big.Int
+
+// ToInt converts b to a big.Int.
+func (b *BigInt) ToInt() *big.Int {
+	return (*big.Int)(b)
+}
+
+// String returns value of b as a decimal string.
+func (b *BigInt) String() string {
+	return b.ToInt().String()
+}
+
+// SetUint64 sets b to x and returns x.
+func (b *BigInt) SetUint64(x uint64) *BigInt {
+	var val big.Int
+	val.SetUint64(x)
+	*b = (BigInt)(val)
+	return b
+}
+
+// MarshalText implements encoding.TextMarshaler
+func (b BigInt) MarshalText() ([]byte, error) {
+	return []byte(b.String()), nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler
+func (b *BigInt) UnmarshalText(input []byte) error {
+	raw, err := checkNumberText(input)
+	if err != nil {
+		return err
+	}
+	if len(raw) > 64 {
+		return hexutil.ErrBig256Range
+	}
+
+	var val big.Int
+	val.SetString(string(input[:]), 10)
+	*b = (BigInt)(val)
+	return nil
+}
+
+// ImplementsGraphQLType returns true if BigInt implements the provided GraphQL type.
+func (b BigInt) ImplementsGraphQLType(name string) bool { return name == "BigInt" }
+
+// UnmarshalGraphQL unmarshals the provided GraphQL query data.
+func (b *BigInt) UnmarshalGraphQL(input interface{}) error {
+	var err error
+	switch input := input.(type) {
+	case string:
+		return b.UnmarshalText([]byte(input))
+	case int32:
+		var num big.Int
+		num.SetInt64(int64(input))
+		*b = BigInt(num)
+	default:
+		err = fmt.Errorf("unexpected type %T for BigInt", input)
+	}
+	return err
+}
+
+func checkNumberText(input []byte) (raw []byte, err error) {
+	if len(input) == 0 {
+		return nil, nil // empty strings are allowed
+	}
+	if len(input) > 1 && input[0] == '0' {
+		return nil, hexutil.ErrLeadingZero
+	}
+	return input, nil
+}

pkg/net/api.go

@@ -0,0 +1,90 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// APIName is the namespace for the watcher's eth api
+const APIName = "net"
+
+// APIVersion is the version of the watcher's eth api
+const APIVersion = "0.0.1"
+
+// PublicNetAPI is the net nampespace API
+type PublicNetAPI struct {
+	// Proxy node for forwarding cache misses
+	networkVersion uint64
+	rpc            *rpc.Client
+	ethClient      *ethclient.Client
+}
+
+// NewPublicNetAPI creates a new PublicNetAPI with the provided underlying Backend
+func NewPublicNetAPI(networkID uint64, client *rpc.Client) *PublicNetAPI {
+	var ethClient *ethclient.Client
+	if client != nil {
+		ethClient = ethclient.NewClient(client)
+	}
+	return &PublicNetAPI{
+		networkVersion: networkID,
+		rpc:            client,
+		ethClient:      ethClient,
+	}
+}
+
+// Listening returns an indication if the node is listening for network connections.
+func (pna *PublicNetAPI) Listening() bool {
+	// in this case it is actually whether or not the proxied node is listening
+	if pna.rpc != nil {
+		var listening bool
+		if err := pna.rpc.Call(&listening, "net_listening"); err == nil {
+			return listening
+		}
+	}
+	return false
+}
+
+// PeerCount returns the number of connected peers
+func (pna *PublicNetAPI) PeerCount() hexutil.Uint {
+	// in this case it is actually the peer count of the proxied node
+	if pna.rpc != nil {
+		var num hexutil.Uint
+		if err := pna.rpc.Call(&num, "net_peerCount"); err == nil {
+			return num
+		}
+	}
+	return hexutil.Uint(0)
+}
+
+// Version returns the current ethereum protocol version.
+func (pna *PublicNetAPI) Version() string {
+	if pna.networkVersion != 0 {
+		return fmt.Sprintf("%d", pna.networkVersion)
+	}
+	if pna.rpc != nil {
+		var version string
+		if err := pna.rpc.Call(&version, "net_version"); err == nil {
+			return version
+		}
+	}
+	return ""
+}

pkg/net/api_test.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/net"
+)
+
+var _ = Describe("API", func() {
+	var (
+		api *net.PublicNetAPI
+	)
+	BeforeEach(func() {
+		api = net.NewPublicNetAPI(1, nil)
+	})
+	Describe("net_listening", func() {
+		It("Retrieves whether or not the node is listening to the p2p network", func() {
+			listening := api.Listening()
+			Expect(listening).To(BeFalse())
+		})
+	})
+
+	Describe("net_version", func() {
+		It("Retrieves the network id", func() {
+			version := api.Version()
+			Expect(version).To(Equal("1"))
+		})
+	})
+	// TODO: test PeerCount with mock proxy node
+})

pkg/net/net_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
+)
+
+func TestNetSuite(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "eth ipld server net suite test")
+}
+
+var _ = BeforeSuite(func() {
+	logrus.SetOutput(ioutil.Discard)
+})

pkg/prom/db_stats_collector.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package prom
 
 import (

pkg/prom/middleware.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package prom
 
 import (

pkg/prom/prom.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package prom
 
 import (

pkg/prom/serve.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package prom
 
 import (

pkg/rpc/check.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package rpc
 
 import "github.com/ethereum/go-ethereum/rpc"

pkg/rpc/http.go

@@ -1,41 +1,47 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package rpc
 
 import (
-	"net"
+	"fmt"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-eth-server/pkg/prom"
 )
 
 // StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules.
-func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (net.Listener, *rpc.Server, error) {
+func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (*rpc.Server, error) {
-	if bad, available := checkModuleAvailability(modules, apis); len(bad) > 0 {
+
-		log.Error("Unavailable modules in HTTP API list", "unavailable", bad, "available", available)
+	srv := rpc.NewServer()
+	err := node.RegisterApis(apis, modules, srv, false)
+	if err != nil {
+		utils.Fatalf("Could not register HTTP API: %w", err)
 	}
-	// Generate the whitelist based on the allowed modules
+	handler := node.NewHTTPHandlerStack(srv, cors, vhosts, nil)
-	whitelist := make(map[string]bool)
+
-	for _, module := range modules {
+	// start http server
-		whitelist[module] = true
+	_, addr, err := node.StartHTTPEndpoint(endpoint, rpc.DefaultHTTPTimeouts, handler)
+	if err != nil {
+		utils.Fatalf("Could not start RPC api: %v", err)
 	}
-	// Register all the APIs exposed by the services
+	extapiURL := fmt.Sprintf("http://%v/", addr)
-	handler := rpc.NewServer()
+	log.Infof("HTTP endpoint opened %s", extapiURL)
-	for _, api := range apis {
+
-		if whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) {
+	return srv, err
-			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
-				return nil, nil, err
-			}
-			log.Debug("HTTP registered", "namespace", api.Namespace)
-		}
-	}
-	// All APIs registered, start the HTTP listener
-	var (
-		listener net.Listener
-		err      error
-	)
-	if listener, err = net.Listen("tcp", endpoint); err != nil {
-		return nil, nil, err
-	}
-	go rpc.NewHTTPServer(cors, vhosts, timeouts, prom.HTTPMiddleware(handler)).Serve(listener)
-	return listener, handler, err
 }

pkg/rpc/ipc.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package rpc
 
 import (
@@ -9,7 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-eth-server/pkg/prom"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/prom"
 )
 
 var (

pkg/rpc/ws.go

@@ -1,46 +1,62 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package rpc
 
 import (
 	"net"
+	"net/http"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
-	log "github.com/sirupsen/logrus"
+
-	"github.com/vulcanize/ipld-eth-server/pkg/prom"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/prom"
 )
 
 // StartWSEndpoint starts a websocket endpoint.
 func StartWSEndpoint(endpoint string, apis []rpc.API, modules []string, wsOrigins []string, exposeAll bool) (net.Listener, *rpc.Server, error) {
-	if bad, available := checkModuleAvailability(modules, apis); len(bad) > 0 {
-		log.Error("Unavailable modules in WS API list", "unavailable", bad, "available", available)
-	}
-	// Generate the whitelist based on the allowed modules
-	whitelist := make(map[string]bool)
-	for _, module := range modules {
-		whitelist[module] = true
-	}
-	// Register all the APIs exposed by the services
-	handler := rpc.NewServer()
-	for _, api := range apis {
-		if exposeAll || whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) {
-			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
-				return nil, nil, err
-			}
-			log.Debug("WebSocket registered", "service", api.Service, "namespace", api.Namespace)
-		}
-	}
 	// All APIs registered, start the HTTP listener
 	var (
 		listener net.Listener
 		err      error
 	)
+
+	// Register all the APIs exposed by the services
+	handler := rpc.NewServer()
+	err = node.RegisterApis(apis, modules, handler, exposeAll)
+	if err != nil {
+		utils.Fatalf("Could not register WS API: %w", err)
+	}
+
 	if listener, err = net.Listen("tcp", endpoint); err != nil {
 		return nil, nil, err
 	}
 
-	wsServer := rpc.NewWSServer(wsOrigins, handler)
+	wsServer := NewWSServer(wsOrigins, handler)
 	wsServer.Handler = prom.WSMiddleware(wsServer.Handler)
 	go wsServer.Serve(listener)
 
 	return listener, handler, err
 
 }
+
+// NewWSServer creates a new websocket RPC server around an API provider.
+//
+// Deprecated: use prc.Server.WebsocketHandler
+func NewWSServer(allowedOrigins []string, srv *rpc.Server) *http.Server {
+	return &http.Server{Handler: srv.WebsocketHandler(allowedOrigins)}
+}

pkg/serve/api.go

@@ -19,14 +19,11 @@ package serve
 import (
 	"context"
 
-	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff/types"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	v "github.com/vulcanize/ipld-eth-server/version"
 )
 
 // APIName is the namespace used for the state diffing service API
@@ -38,12 +35,14 @@ const APIVersion = "0.0.1"
 // PublicServerAPI is the public api for the watcher
 type PublicServerAPI struct {
 	w   Server
+	rpc *rpc.Client
 }
 
 // NewPublicServerAPI creates a new PublicServerAPI with the provided underlying Server process
-func NewPublicServerAPI(w Server) *PublicServerAPI {
+func NewPublicServerAPI(w Server, client *rpc.Client) *PublicServerAPI {
 	return &PublicServerAPI{
 		w:   w,
+		rpc: client,
 	}
 }
 
@@ -86,36 +85,12 @@ func (api *PublicServerAPI) Stream(ctx context.Context, params eth.SubscriptionS
 	return rpcSub, nil
 }
 
-// Chain returns the chain type that this watcher instance supports
+// WatchAddress makes a geth WatchAddress API call with the given operation and args
-func (api *PublicServerAPI) Chain() shared.ChainType {
+func (api *PublicServerAPI) WatchAddress(operation types.OperationType, args []types.WatchAddressArg) error {
-	return shared.Ethereum
+	err := api.rpc.Call(nil, "statediff_watchAddress", operation, args)
+	if err != nil {
+		return err
 	}
 
-// Struct for holding watcher meta data
+	return nil
-type InfoAPI struct{}
-
-// NewInfoAPI creates a new InfoAPI
-func NewInfoAPI() *InfoAPI {
-	return &InfoAPI{}
-}
-
-// Modules returns modules supported by this api
-func (iapi *InfoAPI) Modules() map[string]string {
-	return map[string]string{
-		"vdb": "Stream",
-	}
-}
-
-// NodeInfo gathers and returns a collection of metadata for the watcher
-func (iapi *InfoAPI) NodeInfo() *p2p.NodeInfo {
-	return &p2p.NodeInfo{
-		// TODO: formalize this
-		ID:   "vulcanizeDB",
-		Name: "ipld-eth-server",
-	}
-}
-
-// Version returns the version of the watcher
-func (iapi *InfoAPI) Version() string {
-	return v.VersionWithMeta
 }

pkg/serve/config.go

@@ -17,20 +17,23 @@
 package serve
 
 import (
+	"errors"
+	"fmt"
 	"math/big"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/jmoiron/sqlx"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/node"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-	"github.com/vulcanize/ipld-eth-indexer/utils"
-	"github.com/vulcanize/ipld-eth-server/pkg/prom"
 
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/prom"
+	ethServerShared "github.com/vulcanize/ipld-eth-server/v3/pkg/shared"
 )
 
 // Env variables
@@ -43,23 +46,55 @@ const (
 	SERVER_MAX_OPEN_CONNECTIONS = "SERVER_MAX_OPEN_CONNECTIONS"
 	SERVER_MAX_CONN_LIFETIME    = "SERVER_MAX_CONN_LIFETIME"
 
-	ETH_CHAIN_ID = "ETH_CHAIN_ID"
-
 	ETH_DEFAULT_SENDER_ADDR = "ETH_DEFAULT_SENDER_ADDR"
-
 	ETH_RPC_GAS_CAP         = "ETH_RPC_GAS_CAP"
+	ETH_CHAIN_CONFIG        = "ETH_CHAIN_CONFIG"
+	ETH_SUPPORTS_STATEDIFF  = "ETH_SUPPORTS_STATEDIFF"
+	ETH_FORWARD_ETH_CALLS   = "ETH_FORWARD_ETH_CALLS"
+	ETH_PROXY_ON_ERROR      = "ETH_PROXY_ON_ERROR"
+
+	VALIDATOR_ENABLED         = "VALIDATOR_ENABLED"
+	VALIDATOR_EVERY_NTH_BLOCK = "VALIDATOR_EVERY_NTH_BLOCK"
 )
 
 // Config struct
 type Config struct {
-	DB            *postgres.DB
+	DB       *sqlx.DB
 	DBConfig postgres.Config
+
+	WSEnabled  bool
 	WSEndpoint string
+
+	HTTPEnabled  bool
 	HTTPEndpoint string
+
+	IPCEnabled  bool
 	IPCEndpoint string
+
+	EthGraphqlEnabled  bool
+	EthGraphqlEndpoint string
+
+	IpldGraphqlEnabled          bool
+	IpldGraphqlEndpoint         string
+	IpldPostgraphileEndpoint    string
+	TracingHttpEndpoint         string
+	TracingPostgraphileEndpoint string
+
 	ChainConfig      *params.ChainConfig
 	DefaultSender    *common.Address
 	RPCGasCap        *big.Int
+	EthHttpEndpoint  string
+	Client           *rpc.Client
+	SupportStateDiff bool
+	ForwardEthCalls  bool
+	ProxyOnError     bool
+	NodeNetworkID    string
+
+	// Cache configuration.
+	GroupCache *ethServerShared.GroupCacheConfig
+
+	StateValidationEnabled       bool
+	StateValidationEveryNthBlock uint64
 }
 
 // NewConfig is used to initialize a watcher config from a .toml file
@@ -67,21 +102,43 @@ type Config struct {
 func NewConfig() (*Config, error) {
 	c := new(Config)
 
-	viper.BindEnv("server.wsPath", SERVER_WS_PATH)
+	viper.BindEnv("ethereum.httpPath", ETH_HTTP_PATH)
-	viper.BindEnv("server.ipcPath", SERVER_IPC_PATH)
-	viper.BindEnv("server.httpPath", SERVER_HTTP_PATH)
-	viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
 	viper.BindEnv("ethereum.defaultSender", ETH_DEFAULT_SENDER_ADDR)
 	viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP)
+	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
+	viper.BindEnv("ethereum.supportsStateDiff", ETH_SUPPORTS_STATEDIFF)
+	viper.BindEnv("ethereum.forwardEthCalls", ETH_FORWARD_ETH_CALLS)
+	viper.BindEnv("ethereum.proxyOnError", ETH_PROXY_ON_ERROR)
 
-	c.DBConfig.Init()
+	c.dbInit()
+	ethHTTP := viper.GetString("ethereum.httpPath")
+	ethHTTPEndpoint := fmt.Sprintf("http://%s", ethHTTP)
+	nodeInfo, cli, err := getEthNodeAndClient(ethHTTPEndpoint)
+	c.NodeNetworkID = nodeInfo.NetworkID
+	if err != nil {
+		return nil, err
+	}
+	c.Client = cli
+	c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff")
+	c.ForwardEthCalls = viper.GetBool("ethereum.forwardEthCalls")
+	c.ProxyOnError = viper.GetBool("ethereum.proxyOnError")
+	c.EthHttpEndpoint = ethHTTPEndpoint
 
-	wsPath := viper.GetString("server.wsPath")
+	// websocket server
+	wsEnabled := viper.GetBool("eth.server.ws")
+	if wsEnabled {
+		wsPath := viper.GetString("eth.server.wsPath")
 		if wsPath == "" {
 			wsPath = "127.0.0.1:8080"
 		}
 		c.WSEndpoint = wsPath
-	ipcPath := viper.GetString("server.ipcPath")
+	}
+	c.WSEnabled = wsEnabled
+
+	// ipc server
+	ipcEnabled := viper.GetBool("eth.server.ipc")
+	if ipcEnabled {
+		ipcPath := viper.GetString("eth.server.ipcPath")
 		if ipcPath == "" {
 			home, err := os.UserHomeDir()
 			if err != nil {
@@ -90,15 +147,67 @@ func NewConfig() (*Config, error) {
 			ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
 		}
 		c.IPCEndpoint = ipcPath
-	httpPath := viper.GetString("server.httpPath")
+	}
+	c.IPCEnabled = ipcEnabled
+
+	// http server
+	httpEnabled := viper.GetBool("eth.server.http")
+	if httpEnabled {
+		httpPath := viper.GetString("eth.server.httpPath")
 		if httpPath == "" {
 			httpPath = "127.0.0.1:8081"
 		}
 		c.HTTPEndpoint = httpPath
+	}
+	c.HTTPEnabled = httpEnabled
+
+	// eth graphql endpoint
+	ethGraphqlEnabled := viper.GetBool("eth.server.graphql")
+	if ethGraphqlEnabled {
+		ethGraphqlPath := viper.GetString("eth.server.graphqlPath")
+		if ethGraphqlPath == "" {
+			ethGraphqlPath = "127.0.0.1:8082"
+		}
+		c.EthGraphqlEndpoint = ethGraphqlPath
+	}
+	c.EthGraphqlEnabled = ethGraphqlEnabled
+
+	// ipld graphql endpoint
+	ipldGraphqlEnabled := viper.GetBool("ipld.server.graphql")
+	if ipldGraphqlEnabled {
+		ipldGraphqlPath := viper.GetString("ipld.server.graphqlPath")
+		if ipldGraphqlPath == "" {
+			ipldGraphqlPath = "127.0.0.1:8083"
+		}
+		c.IpldGraphqlEndpoint = ipldGraphqlPath
+
+		ipldPostgraphilePath := viper.GetString("ipld.postgraphilePath")
+		if ipldPostgraphilePath == "" {
+			return nil, errors.New("ipld-postgraphile-path parameter is empty")
+		}
+		c.IpldPostgraphileEndpoint = ipldPostgraphilePath
+
+		tracingHttpEndpoint := viper.GetString("tracing.httpPath")
+		tracingPostgraphilePath := viper.GetString("tracing.postgraphilePath")
+
+		// these two parameters either can be both empty or both set
+		if (tracingHttpEndpoint == "" && tracingPostgraphilePath != "") || (tracingHttpEndpoint != "" && tracingPostgraphilePath == "") {
+			return nil, errors.New("tracing.httpPath and tracing.postgraphilePath parameters either can be both empty or both set")
+		}
+
+		c.TracingHttpEndpoint = tracingHttpEndpoint
+		c.TracingPostgraphileEndpoint = tracingPostgraphilePath
+	}
+	c.IpldGraphqlEnabled = ipldGraphqlEnabled
+
 	overrideDBConnConfig(&c.DBConfig)
-	serveDB := utils.LoadPostgres(c.DBConfig, node.Info{})
+	serveDB, err := ethServerShared.NewDB(c.DBConfig.DbConnectionString(), c.DBConfig)
-	prom.RegisterDBCollector(c.DBConfig.Name, serveDB.DB)
+	if err != nil {
-	c.DB = &serveDB
+		return nil, err
+	}
+
+	prom.RegisterDBCollector(c.DBConfig.DatabaseName, serveDB)
+	c.DB = serveDB
 
 	defaultSenderStr := viper.GetString("ethereum.defaultSender")
 	if defaultSenderStr != "" {
@@ -111,9 +220,17 @@ func NewConfig() (*Config, error) {
 			c.RPCGasCap = rpcGasCap
 		}
 	}
-	chainID := viper.GetUint64("ethereum.chainID")
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
-	var err error
+	if chainConfigPath != "" {
-	c.ChainConfig, err = eth.ChainConfig(chainID)
+		c.ChainConfig, err = statediff.LoadConfig(chainConfigPath)
+	} else {
+		c.ChainConfig, err = statediff.ChainConfig(nodeInfo.ChainID)
+	}
+
+	c.loadGroupCacheConfig()
+
+	c.loadValidatorConfig()
+
 	return c, err
 }
 
@@ -122,6 +239,57 @@ func overrideDBConnConfig(con *postgres.Config) {
 	viper.BindEnv("database.server.maxOpen", SERVER_MAX_OPEN_CONNECTIONS)
 	viper.BindEnv("database.server.maxLifetime", SERVER_MAX_CONN_LIFETIME)
 	con.MaxIdle = viper.GetInt("database.server.maxIdle")
-	con.MaxOpen = viper.GetInt("database.server.maxOpen")
+	con.MaxConns = viper.GetInt("database.server.maxOpen")
-	con.MaxLifetime = viper.GetInt("database.server.maxLifetime")
+	con.MaxConnLifetime = time.Duration(viper.GetInt("database.server.maxLifetime"))
+}
+
+func (c *Config) dbInit() {
+	viper.BindEnv("database.name", DATABASE_NAME)
+	viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
+	viper.BindEnv("database.port", DATABASE_PORT)
+	viper.BindEnv("database.user", DATABASE_USER)
+	viper.BindEnv("database.password", DATABASE_PASSWORD)
+	viper.BindEnv("database.maxIdle", DATABASE_MAX_IDLE_CONNECTIONS)
+	viper.BindEnv("database.maxOpen", DATABASE_MAX_OPEN_CONNECTIONS)
+	viper.BindEnv("database.maxLifetime", DATABASE_MAX_CONN_LIFETIME)
+
+	c.DBConfig.DatabaseName = viper.GetString("database.name")
+	c.DBConfig.Hostname = viper.GetString("database.hostname")
+	c.DBConfig.Port = viper.GetInt("database.port")
+	c.DBConfig.Username = viper.GetString("database.user")
+	c.DBConfig.Password = viper.GetString("database.password")
+	c.DBConfig.MaxIdle = viper.GetInt("database.maxIdle")
+	c.DBConfig.MaxConns = viper.GetInt("database.maxOpen")
+	c.DBConfig.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime"))
+}
+
+func (c *Config) loadGroupCacheConfig() {
+	viper.BindEnv("groupcache.pool.enabled", ethServerShared.GcachePoolEnabled)
+	viper.BindEnv("groupcache.pool.httpEndpoint", ethServerShared.GcachePoolHttpPath)
+	viper.BindEnv("groupcache.pool.peerHttpEndpoints", ethServerShared.GcachePoolHttpPeers)
+	viper.BindEnv("groupcache.statedb.cacheSizeInMB", ethServerShared.GcacheStatedbCacheSize)
+	viper.BindEnv("groupcache.statedb.cacheExpiryInMins", ethServerShared.GcacheStatedbCacheExpiry)
+	viper.BindEnv("groupcache.statedb.logStatsIntervalInSecs", ethServerShared.GcacheStatedbLogStatsInterval)
+
+	gcc := ethServerShared.GroupCacheConfig{}
+	gcc.Pool.Enabled = viper.GetBool("groupcache.pool.enabled")
+	if gcc.Pool.Enabled {
+		gcc.Pool.HttpEndpoint = viper.GetString("groupcache.pool.httpEndpoint")
+		gcc.Pool.PeerHttpEndpoints = viper.GetStringSlice("groupcache.pool.peerHttpEndpoints")
+	}
+
+	// Irrespective of whether the pool is enabled, we always use the hot/local cache.
+	gcc.StateDB.CacheSizeInMB = viper.GetInt("groupcache.statedb.cacheSizeInMB")
+	gcc.StateDB.CacheExpiryInMins = viper.GetInt("groupcache.statedb.cacheExpiryInMins")
+	gcc.StateDB.LogStatsIntervalInSecs = viper.GetInt("groupcache.statedb.logStatsIntervalInSecs")
+
+	c.GroupCache = &gcc
+}
+
+func (c *Config) loadValidatorConfig() {
+	viper.BindEnv("validator.enabled", VALIDATOR_ENABLED)
+	viper.BindEnv("validator.everyNthBlock", VALIDATOR_EVERY_NTH_BLOCK)
+
+	c.StateValidationEnabled = viper.GetBool("validator.enabled")
+	c.StateValidationEveryNthBlock = viper.GetUint64("validator.everyNthBlock")
 }

pkg/serve/env.go

@@ -0,0 +1,50 @@
+package serve
+
+import (
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/spf13/viper"
+)
+
+// Env variables
+const (
+	HTTP_TIMEOUT = "HTTP_TIMEOUT"
+
+	ETH_WS_PATH       = "ETH_WS_PATH"
+	ETH_HTTP_PATH     = "ETH_HTTP_PATH"
+	ETH_NODE_ID       = "ETH_NODE_ID"
+	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
+	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
+	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
+	ETH_CHAIN_ID      = "ETH_CHAIN_ID"
+
+	DATABASE_NAME                 = "DATABASE_NAME"
+	DATABASE_HOSTNAME             = "DATABASE_HOSTNAME"
+	DATABASE_PORT                 = "DATABASE_PORT"
+	DATABASE_USER                 = "DATABASE_USER"
+	DATABASE_PASSWORD             = "DATABASE_PASSWORD"
+	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
+	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
+	DATABASE_MAX_CONN_LIFETIME    = "DATABASE_MAX_CONN_LIFETIME"
+)
+
+// GetEthNodeAndClient returns eth node info and client from path url
+func getEthNodeAndClient(path string) (node.Info, *rpc.Client, error) {
+	viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
+	viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
+	viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
+	viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
+	viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
+
+	rpcClient, err := rpc.Dial(path)
+	if err != nil {
+		return node.Info{}, nil, err
+	}
+	return node.Info{
+		ID:           viper.GetString("ethereum.nodeID"),
+		ClientName:   viper.GetString("ethereum.clientName"),
+		GenesisBlock: viper.GetString("ethereum.genesisBlock"),
+		NetworkID:    viper.GetString("ethereum.networkID"),
+		ChainID:      viper.GetUint64("ethereum.chainID"),
+	}, rpcClient, nil
+}

pkg/serve/service.go

@@ -18,22 +18,21 @@ package serve
 
 import (
 	"fmt"
+	"strconv"
 	"sync"
 
-	"github.com/ethereum/go-ethereum/core/vm"
-
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	ethnode "github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/jmoiron/sqlx"
 	log "github.com/sirupsen/logrus"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
+	"github.com/vulcanize/ipld-eth-server/v3/pkg/net"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 )
 
 const (
@@ -44,14 +43,18 @@ const (
 // and indexing all chain data; screening this data; and serving it up to subscribed clients
 // This service is compatible with the Ethereum service interface (node.Service)
 type Server interface {
-	// APIs(), Protocols(), Start() and Stop()
+	// Start() and Stop()
-	ethnode.Service
+	ethnode.Lifecycle
+	APIs() []rpc.API
+	Protocols() []p2p.Protocol
 	// Pub-Sub handling event loop
-	Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth2.ConvertedPayload)
+	Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth.ConvertedPayload)
 	// Method to subscribe to the service
 	Subscribe(id rpc.ID, sub chan<- SubscriptionPayload, quitChan chan<- bool, params eth.SubscriptionSettings)
 	// Method to unsubscribe from the service
 	Unsubscribe(id rpc.ID)
+	// Backend exposes the server's backend
+	Backend() *eth.Backend
 }
 
 // Service is the underlying struct for the watcher
@@ -71,30 +74,47 @@ type Service struct {
 	// A mapping of subscription params hash to the corresponding subscription params
 	SubscriptionTypes map[common.Hash]eth.SubscriptionSettings
 	// Underlying db
-	db *postgres.DB
+	db *sqlx.DB
 	// wg for syncing serve processes
 	serveWg *sync.WaitGroup
-	// config for backend
+	// rpc client for forwarding cache misses
-	config *eth.Config
+	client *rpc.Client
+	// whether the proxied client supports state diffing
+	supportsStateDiffing bool
+	// backend for the server
+	backend *eth.Backend
+	// whether to forward eth_calls directly to proxy node
+	forwardEthCalls bool
+	// whether to forward all calls to proxy node if they throw an error locally
+	proxyOnError bool
+	// eth node network id
+	nodeNetworkId string
 }
 
 // NewServer creates a new Server using an underlying Service struct
 func NewServer(settings *Config) (Server, error) {
-	sn := new(Service)
+	sap := new(Service)
-	sn.Retriever = eth.NewCIDRetriever(settings.DB)
+	sap.Retriever = eth.NewCIDRetriever(settings.DB)
-	sn.IPLDFetcher = eth.NewIPLDFetcher(settings.DB)
+	sap.IPLDFetcher = eth.NewIPLDFetcher(settings.DB)
-	sn.Filterer = eth.NewResponseFilterer()
+	sap.Filterer = eth.NewResponseFilterer()
-	sn.db = settings.DB
+	sap.db = settings.DB
-	sn.QuitChan = make(chan bool)
+	sap.QuitChan = make(chan bool)
-	sn.Subscriptions = make(map[common.Hash]map[rpc.ID]Subscription)
+	sap.Subscriptions = make(map[common.Hash]map[rpc.ID]Subscription)
-	sn.SubscriptionTypes = make(map[common.Hash]eth.SubscriptionSettings)
+	sap.SubscriptionTypes = make(map[common.Hash]eth.SubscriptionSettings)
-	sn.config = &eth.Config{
+	sap.client = settings.Client
+	sap.supportsStateDiffing = settings.SupportStateDiff
+	sap.forwardEthCalls = settings.ForwardEthCalls
+	sap.proxyOnError = settings.ProxyOnError
+	sap.nodeNetworkId = settings.NodeNetworkID
+	var err error
+	sap.backend, err = eth.NewEthBackend(sap.db, &eth.Config{
 		ChainConfig:      settings.ChainConfig,
-		VmConfig:      vm.Config{},
+		VMConfig:         vm.Config{NoBaseFee: true},
 		DefaultSender:    settings.DefaultSender,
 		RPCGasCap:        settings.RPCGasCap,
-	}
+		GroupCacheConfig: settings.GroupCache,
-	return sn, nil
+	})
+	return sap, err
 }
 
 // Protocols exports the services p2p protocols, this service has none
@@ -104,42 +124,29 @@ func (sap *Service) Protocols() []p2p.Protocol {
 
 // APIs returns the RPC descriptors the watcher service offers
 func (sap *Service) APIs() []rpc.API {
-	infoAPI := NewInfoAPI()
+	networkID, _ := strconv.ParseUint(sap.nodeNetworkId, 10, 64)
 	apis := []rpc.API{
 		{
 			Namespace: APIName,
 			Version:   APIVersion,
-			Service:   NewPublicServerAPI(sap),
+			Service:   NewPublicServerAPI(sap, sap.client),
 			Public:    true,
 		},
 		{
-			Namespace: "rpc",
+			Namespace: net.APIName,
-			Version:   APIVersion,
+			Version:   net.APIVersion,
-			Service:   infoAPI,
+			Service:   net.NewPublicNetAPI(networkID, sap.client),
-			Public:    true,
-		},
-		{
-			Namespace: "net",
-			Version:   APIVersion,
-			Service:   infoAPI,
-			Public:    true,
-		},
-		{
-			Namespace: "admin",
-			Version:   APIVersion,
-			Service:   infoAPI,
 			Public:    true,
 		},
 	}
-	backend, err := eth.NewEthBackend(sap.db, sap.config)
+	ethAPI, err := eth.NewPublicEthAPI(sap.backend, sap.client, sap.supportsStateDiffing, sap.forwardEthCalls, sap.proxyOnError)
 	if err != nil {
-		log.Error(err)
+		log.Fatalf("unable to create public eth api: %v", err)
-		return nil
 	}
 	return append(apis, rpc.API{
 		Namespace: eth.APIName,
 		Version:   eth.APIVersion,
-		Service:   eth.NewPublicEthAPI(backend),
+		Service:   ethAPI,
 		Public:    true,
 	})
 }
@@ -148,7 +155,7 @@ func (sap *Service) APIs() []rpc.API {
 // It filters and sends this data to any subscribers to the service
 // This process can also be stood up alone, without an screenAndServePayload attached to a Sync process
 // and it will hang on the WaitGroup indefinitely, allowing the Service to serve historical data requests only
-func (sap *Service) Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth2.ConvertedPayload) {
+func (sap *Service) Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth.ConvertedPayload) {
 	sap.serveWg = wg
 	go func() {
 		wg.Add(1)
@@ -167,7 +174,7 @@ func (sap *Service) Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth2.
 }
 
 // filterAndServe filters the payload according to each subscription type and sends to the subscriptions
-func (sap *Service) filterAndServe(payload eth2.ConvertedPayload) {
+func (sap *Service) filterAndServe(payload eth.ConvertedPayload) {
 	log.Debug("sending eth ipld payload to subscriptions")
 	sap.Lock()
 	sap.serveWg.Add(1)
@@ -337,10 +344,10 @@ func (sap *Service) Unsubscribe(id rpc.ID) {
 
 // Start is used to begin the service
 // This is mostly just to satisfy the node.Service interface
-func (sap *Service) Start(*p2p.Server) error {
+func (sap *Service) Start() error {
 	log.Info("starting eth ipld server")
 	wg := new(sync.WaitGroup)
-	payloadChan := make(chan eth2.ConvertedPayload, PayloadChanBufferSize)
+	payloadChan := make(chan eth.ConvertedPayload, PayloadChanBufferSize)
 	sap.Serve(wg, payloadChan)
 	return nil
 }
@@ -356,6 +363,11 @@ func (sap *Service) Stop() error {
 	return nil
 }
 
+// Backend exposes the server's backend
+func (sap *Service) Backend() *eth.Backend {
+	return sap.backend
+}
+
 // close is used to close all listening subscriptions
 // close needs to be called with subscription access locked
 func (sap *Service) close() {

pkg/shared/constants.go

@@ -19,4 +19,11 @@ package shared
 const (
 	DefaultMaxBatchSize   uint64 = 100
 	DefaultMaxBatchNumber int64  = 50
+
+	GcachePoolEnabled             = "GCACHE_POOL_ENABLED"
+	GcachePoolHttpPath            = "GCACHE_POOL_HTTP_PATH"
+	GcachePoolHttpPeers           = "GCACHE_POOL_HTTP_PEERS"
+	GcacheStatedbCacheSize        = "GCACHE_STATEDB_CACHE_SIZE"
+	GcacheStatedbCacheExpiry      = "GCACHE_STATEDB_CACHE_EXPIRY"
+	GcacheStatedbLogStatsInterval = "GCACHE_STATEDB_LOG_STATS_INTERVAL"
 )

pkg/shared/database.go

@@ -0,0 +1,41 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+import (
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/jmoiron/sqlx"
+)
+
+// NewDB creates a new db connection and initializes the connection pool
+func NewDB(connectString string, config postgres.Config) (*sqlx.DB, error) {
+	db, connectErr := sqlx.Connect("postgres", connectString)
+	if connectErr != nil {
+		return nil, postgres.ErrDBConnectionFailed(connectErr)
+	}
+	if config.MaxConns > 0 {
+		db.SetMaxOpenConns(config.MaxConns)
+	}
+	if config.MaxIdle > 0 {
+		db.SetMaxIdleConns(config.MaxIdle)
+	}
+	if config.MaxConnLifetime > 0 {
+		db.SetConnMaxLifetime(config.MaxConnLifetime)
+	}
+
+	return db, nil
+}

pkg/shared/functions.go

@@ -18,13 +18,13 @@ package shared
 
 import (
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-ipfs-blockstore"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	"github.com/ipfs/go-ipfs-ds-help"
+	dshelp "github.com/ipfs/go-ipfs-ds-help"
 	node "github.com/ipfs/go-ipld-format"
 	"github.com/jmoiron/sqlx"
 	"github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
 )
 
 // HandleZeroAddrPointer will return an emtpy string for a nil address pointer

pkg/shared/test_helpers.go

@@ -18,38 +18,24 @@ package shared
 
 import (
 	"bytes"
+	"context"
+	"os"
+	"strconv"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	. "github.com/onsi/gomega"
 
-	"github.com/ipfs/go-cid"
+	"github.com/ethereum/go-ethereum/common"
-	"github.com/multiformats/go-multihash"
+	"github.com/ethereum/go-ethereum/params"
-
+	"github.com/ethereum/go-ethereum/statediff/indexer"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/node"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/jmoiron/sqlx"
 )
 
-// SetupDB is use to setup a db for watcher tests
-func SetupDB() (*postgres.DB, error) {
-	return postgres.NewDB(postgres.Config{
-		Hostname: "localhost",
-		Name:     "vulcanize_testing",
-		Port:     5432,
-	}, node.Info{})
-}
-
-// ListContainsString used to check if a list of strings contains a particular string
-func ListContainsString(sss []string, s string) bool {
-	for _, str := range sss {
-		if s == str {
-			return true
-		}
-	}
-	return false
-}
-
 // IPLDsContainBytes used to check if a list of strings contains a particular string
-func IPLDsContainBytes(iplds []ipfs.BlockModel, b []byte) bool {
+func IPLDsContainBytes(iplds []models.IPLDModel, b []byte) bool {
 	for _, ipld := range iplds {
 		if bytes.Equal(ipld.Data, b) {
 			return true
@@ -58,30 +44,64 @@ func IPLDsContainBytes(iplds []ipfs.BlockModel, b []byte) bool {
 	return false
 }
 
-// ListContainsGap used to check if a list of Gaps contains a particular Gap
+// SetupDB is use to setup a db for watcher tests
-func ListContainsGap(gapList []eth.DBGap, gap eth.DBGap) bool {
+func SetupDB() *sqlx.DB {
-	for _, listGap := range gapList {
+	config := getTestDBConfig()
-		if listGap == gap {
+
-			return true
+	db, err := NewDB(config.DbConnectionString(), config)
-		}
+	Expect(err).NotTo(HaveOccurred())
-	}
+
-	return false
+	return db
 }
 
-// TestCID creates a basic CID for testing purposes
+// TearDownDB is used to tear down the watcher dbs after tests
-func TestCID(b []byte) cid.Cid {
+func TearDownDB(db *sqlx.DB) {
-	pref := cid.Prefix{
+	tx, err := db.Beginx()
-		Version:  1,
+	Expect(err).NotTo(HaveOccurred())
-		Codec:    cid.Raw,
+	_, err = tx.Exec(`DELETE FROM eth.header_cids`)
-		MhType:   multihash.KECCAK_256,
+	Expect(err).NotTo(HaveOccurred())
-		MhLength: -1,
+	_, err = tx.Exec(`DELETE FROM eth.transaction_cids`)
-	}
+	Expect(err).NotTo(HaveOccurred())
-	c, _ := pref.Sum(b)
+	_, err = tx.Exec(`DELETE FROM eth.receipt_cids`)
-	return c
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM eth.state_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM eth.storage_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM blocks`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM eth.log_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM eth_meta.watched_addresses`)
+	Expect(err).NotTo(HaveOccurred())
+
+	err = tx.Commit()
+	Expect(err).NotTo(HaveOccurred())
 }
 
-// PublishMockIPLD writes a mhkey-data pair to the public.blocks table so that test data can FK reference the mhkey
+func SetupTestStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, genHash common.Hash) interfaces.StateDiffIndexer {
-func PublishMockIPLD(db *postgres.DB, mhKey string, mockData []byte) error {
+	testInfo := node.Info{
-	_, err := db.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, mhKey, mockData)
+		GenesisBlock: genHash.String(),
-	return err
+		NetworkID:    "1",
+		ID:           "1",
+		ClientName:   "geth",
+		ChainID:      params.TestChainConfig.ChainID.Uint64(),
+	}
+
+	_, stateDiffIndexer, err := indexer.NewStateDiffIndexer(ctx, chainConfig, testInfo, getTestDBConfig())
+	Expect(err).NotTo(HaveOccurred())
+
+	return stateDiffIndexer
+}
+
+func getTestDBConfig() postgres.Config {
+	port, _ := strconv.Atoi(os.Getenv("DATABASE_PORT"))
+	return postgres.Config{
+		Hostname:     os.Getenv("DATABASE_HOSTNAME"),
+		DatabaseName: os.Getenv("DATABASE_NAME"),
+		Username:     os.Getenv("DATABASE_USER"),
+		Password:     os.Getenv("DATABASE_PASSWORD"),
+		Port:         port,
+		Driver:       postgres.SQLX,
+	}
 }

pkg/shared/types.go

@@ -0,0 +1,38 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+type PoolConfig struct {
+	Enabled           bool
+	HttpEndpoint      string
+	PeerHttpEndpoints []string
+}
+
+type GroupConfig struct {
+	CacheSizeInMB          int
+	CacheExpiryInMins      int
+	LogStatsIntervalInSecs int
+
+	// Used in tests to override the cache name, to work around
+	// the "duplicate registration of group" error from groupcache
+	Name string
+}
+
+type GroupCacheConfig struct {
+	Pool    PoolConfig
+	StateDB GroupConfig
+}

scripts/run_integration_test.sh

@@ -0,0 +1,17 @@
+set -e
+set -o xtrace
+
+export ETH_FORWARD_ETH_CALLS=false
+export DB_WRITE=true
+export ETH_PROXY_ON_ERROR=false
+
+export PGPASSWORD=password
+export DATABASE_USER=vdbm
+export DATABASE_PORT=8077
+export DATABASE_PASSWORD=password
+export DATABASE_HOSTNAME=127.0.0.1
+
+# Wait for containers to be up and execute the integration test.
+while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8081)" != "200" ]; do echo "waiting for ipld-eth-server..." && sleep 5; done && \
+          while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
+          make integrationtest

scripts/run_integration_test_forward_eth_calls.sh

@@ -0,0 +1,17 @@
+set -e
+set -o xtrace
+
+export ETH_FORWARD_ETH_CALLS=true
+export DB_WRITE=false
+export ETH_PROXY_ON_ERROR=false
+
+export PGPASSWORD=password
+export DATABASE_USER=vdbm
+export DATABASE_PORT=8077
+export DATABASE_PASSWORD=password
+export DATABASE_HOSTNAME=127.0.0.1
+
+# Wait for containers to be up and execute the integration test.
+while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8081)" != "200" ]; do echo "waiting for ipld-eth-server..." && sleep 5; done && \
+          while [ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8545)" != "200" ]; do echo "waiting for geth-statediff..." && sleep 5; done && \
+          make integrationtest

scripts/run_unit_test.sh

@@ -0,0 +1,8 @@
+# Clear up existing docker images and volume.
+docker-compose down --remove-orphans --volumes
+
+docker-compose -f docker-compose.yml up -d ipld-eth-db
+sleep 10
+PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 DATABASE_NAME=vulcanize_testing make test
+
+docker-compose down --remove-orphans --volumes

test/README.md

@@ -0,0 +1,87 @@
+# Test Insructions
+
+## Setup
+
+- Clone [stack-orchestrator](https://github.com/vulcanize/stack-orchestrator) and [go-ethereum](https://github.com/vulcanize/go-ethereum) repositories.
+
+- Checkout [v3 release](https://github.com/vulcanize/go-ethereum/releases/tag/v1.10.17-statediff-3.2.1) in go-ethereum repo.
+  ```bash
+  # In go-ethereum repo.
+  git checkout v1.10.17-statediff-3.2.1
+  ```
+
+- Checkout working commit in stack-orchestrator repo.
+  ```bash
+  # In stack-orchestrator repo.
+  git checkout fcbc74451c5494664fe21f765e89c9c6565c07cb
+  ```
+
+## Run
+
+- Run unit tests:
+
+  ```bash
+  # In ipld-eth-server root directory.
+  ./scripts/run_unit_test.sh
+  ```
+
+- Run integration tests:
+
+  - Update (Replace existing content) config file [config.sh](https://github.com/vulcanize/stack-orchestrator/blob/main/config.sh) in stack-orchestrator repo:
+    ```bash
+    #!/bin/bash
+
+    # Path to go-ethereum repo.
+    vulcanize_go_ethereum=~/go-ethereum/
+
+    # Path to ipld-eth-server repo.
+    vulcanize_ipld_eth_server=~/ipld-eth-server/
+
+    db_write=true
+    eth_forward_eth_calls=false
+    eth_proxy_on_error=false
+    eth_http_path="go-ethereum:8545"
+    ```
+
+  - Run stack-orchestrator:
+    ```bash
+    # In stack-orchestrator root directory.
+    cd helper-scripts
+
+    ./wrapper.sh \
+    -e docker \
+    -d ../docker/latest/docker-compose-db.yml \
+    -d ../docker/local/docker-compose-go-ethereum.yml \
+    -d ../docker/local/docker-compose-ipld-eth-server.yml \
+    -v remove \
+    -p ../config.sh
+    ```
+
+  - Run test:
+    ```bash
+    # In ipld-eth-server root directory.
+    ./scripts/run_integration_test.sh
+    ```
+
+  - Update `config.sh` file:
+    ```bash
+    #!/bin/bash
+
+    # Path to go-ethereum repo.
+    vulcanize_go_ethereum=~/go-ethereum/
+
+    # Path to ipld-eth-server repo.
+    vulcanize_ipld_eth_server=~/ipld-eth-server/
+
+    db_write=false
+    eth_forward_eth_calls=true
+    eth_proxy_on_error=false
+    eth_http_path="go-ethereum:8545"
+    ```
+
+  - Stop the stack-orchestrator and start again using the same command
+
+  - Run integration tests for direct proxy fall-through of eth_calls:
+    ```bash
+    ./scripts/run_integration_test_forward_eth_calls.sh
+    ```

test/contract/.dockerignore

@@ -0,0 +1,3 @@
+node_modules
+artifacts
+cache

test/contract/.gitignore

@@ -0,0 +1,5 @@
+node_modules
+
+#Hardhat files
+cache
+artifacts

test/contract/Dockerfile

@@ -0,0 +1,14 @@
+FROM node:14
+
+ARG ETH_ADDR
+ENV ETH_ADDR $ETH_ADDR
+
+WORKDIR /usr/src/app
+COPY package*.json ./
+RUN npm ci
+COPY . .
+RUN npm run compile && ls -lah
+
+EXPOSE 3000
+
+ENTRYPOINT ["npm", "start"]