Fix typo in CI workflow (#254)

* Remove canonical_header_hash() uses and update SQL queries for latest v5 schema.

* Remove get_storage_at() uses.

.github/workflows/on-master.yaml

@@ -1,25 +0,0 @@
-name: Docker Compose Build
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Get the version
-        id: vars
-        run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-      - name: Run docker build
-        run: make docker-build
-      - name: Tag docker image
-        run: docker tag vulcanize/ipld-eth-server docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
-      - name: Docker Login
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Push
-        run: docker push docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
-

.github/workflows/on-pr-publish.yaml

@@ -0,0 +1,51 @@
+name: Test, Build, and/or Publish
+on:
+  release:
+    types: [published]
+  pull_request:
+
+jobs:
+  pre_job:
+    # continue-on-error: true # Uncomment once integration is finished
+    runs-on: ubuntu-latest
+    # Map a step output to a job output
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@v4
+        with:
+          # All of these options are optional, so you can remove them if you are happy with the defaults
+          concurrent_skipping: "never"
+          skip_after_successful_duplicate: "true"
+          do_not_skip: '["workflow_dispatch", "schedule"]'
+  run-tests:
+    uses: ./.github/workflows/tests.yaml
+    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+    needs: pre_job
+  build:
+    name: Run docker build
+    runs-on: ubuntu-latest
+    needs: run-tests
+    if: |
+      always() &&
+      (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') &&
+      github.event_name == 'release'
+    steps:
+      - uses: actions/checkout@v3
+      - id: vars
+        name: Output SHA and version tag
+        run: |
+          echo sha=$(echo ${GITHUB_SHA:0:7}) >> "$GITHUB_OUTPUT"
+          echo tag=$(echo ${GITHUB_REF#refs/tags/}) >> "$GITHUB_OUTPUT"
+      - name: Build and tag Docker image
+        run: |
+          docker build . \
+          -t cerc-io/ipld-eth-server \
+          -t git.vdb.to/cerc-io/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}} \
+          -t git.vdb.to/cerc-io/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.tag}}
+      - name: Push Docker tags
+        run: |
+          echo ${{ secrets.GITEA_PUBLISH_TOKEN }} | docker login https://git.vdb.to -u cerccicd --password-stdin
+          docker push git.vdb.to/cerc-io/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
+          docker push git.vdb.to/cerc-io/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.tag}}

.github/workflows/on-pr.yaml

@@ -1,12 +0,0 @@
-name: Docker Build
-
-on: [pull_request]
-
-jobs:
-  build:
-    name: Run docker build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run docker build
-        run: make docker-build

.github/workflows/publish.yaml

@@ -1,25 +0,0 @@
-name: Publish Docker image
-on:
-  release:
-    types: [published]
-jobs:
-  push_to_registries:
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    steps:
-      - name: Get the version
-        id: vars
-        run: |
-          echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
-          echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
-      - name: Docker Login to Github Registry
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
-      - name: Docker Pull
-        run: docker pull docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}}
-      - name: Docker Login to Docker Registry
-        run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
-      - name: Tag docker image
-        run: docker tag docker.pkg.github.com/vulcanize/ipld-eth-server/ipld-eth-server:${{steps.vars.outputs.sha}} vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}}
-      - name: Docker Push to Docker Hub
-        run: docker push vulcanize/ipld-eth-server:${{steps.vars.outputs.tag}}
-

.github/workflows/tests.yaml

@@ -0,0 +1,91 @@
+name: Test the stack.
+on:
+  workflow_call:
+
+jobs:
+  test:
+    name: "Run unit tests"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: "Run DB container"
+        run: docker compose -f test/compose-db.yml up --wait --quiet-pull
+      - name: "Build and run tests"
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo
+          ginkgo -v -r --skipPackage=./integration
+
+  integration-test:
+    name: "Run integration tests"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: "Install stack-orchestrator"
+        # FIXME: using my dev branch for v5 migration changes until a release has them
+        # run: |
+        #   curl -L -O https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+        uses: actions/checkout@v3
+        with:
+          repository: cerc-io/stack-orchestrator
+          ref: roy/ipld-eth-ci
+          path: ./stack-orchestrator
+      - run: pip install ./stack-orchestrator
+      - name: "Run testnet stack"
+        run: ./scripts/integration-setup.sh
+      - name: "Build and run server"
+        env:
+          ETH_FORWARD_ETH_CALLS: false
+          ETH_CHAIN_ID: ${{ env.ETH_CHAIN_ID }}
+          ETH_CHAIN_CONFIG: ${{ env.ETH_CHAIN_CONFIG }}
+          DEPLOYER_PRIVATE_KEY: ${{ env.DEPLOYER_PRIVATE_KEY }}
+        run: docker compose -f test/compose-server.yml up --wait --quiet-pull
+      - name: "Run tests"
+        env:
+          ETH_CHAIN_ID: ${{ env.ETH_CHAIN_ID }}
+          ETH_HTTP_PATH: ${{ env.ETH_HTTP_PATH }}
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo
+          ginkgo -v --label-filter '!proxy' -r ./integration
+
+  integration-test-eth-proxy:
+    name: "Run direct-proxy integration tests"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+          check-latest: true
+      - name: "Install stack-orchestrator"
+        uses: actions/checkout@v3
+        with:
+          repository: cerc-io/stack-orchestrator
+          ref: roy/ipld-eth-ci
+          path: ./stack-orchestrator
+      - run: pip install ./stack-orchestrator
+      - name: "Run testnet stack"
+        env:
+          CERC_RUN_STATEDIFF: false
+        run: ./scripts/integration-setup.sh
+      - name: "Build and run server"
+        env:
+          ETH_FORWARD_ETH_CALLS: true
+          ETH_CHAIN_ID: ${{ env.ETH_CHAIN_ID }}
+          ETH_CHAIN_CONFIG: ${{ env.ETH_CHAIN_CONFIG }}
+          DEPLOYER_PRIVATE_KEY: ${{ env.DEPLOYER_PRIVATE_KEY }}
+        run: docker compose -f test/compose-server.yml up --wait --quiet-pull
+      - name: "Run tests"
+        env:
+          ETH_CHAIN_ID: ${{ env.ETH_CHAIN_ID }}
+          ETH_HTTP_PATH: ${{ env.ETH_HTTP_PATH }}
+        run: |
+          go install github.com/onsi/ginkgo/v2/ginkgo
+          ginkgo -v --label-filter 'proxy' -r ./integration

Dockerfile

@@ -1,18 +1,30 @@
-FROM golang:1.13-alpine as builder
+FROM golang:1.19-alpine as builder
 
-RUN apk --update --no-cache add make git g++ linux-headers
+RUN apk --update --no-cache add gcc musl-dev
 # DEBUG
 RUN apk add busybox-extras
 
-# Build ipld-eth-server
-WORKDIR /go/src/github.com/vulcanize/ipld-eth-server
-ADD . .
-RUN GO111MODULE=on GCO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-eth-server .
+# Include dlv
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
 
-# Copy migration tool
+# Build ipld-eth-server
+WORKDIR /go/src/github.com/cerc-io/ipld-eth-server
+
+# Cache the modules
+ENV GO111MODULE=on
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+COPY . .
+
+# Build the binary
+RUN GOOS=linux go build -a -installsuffix cgo -ldflags '-extldflags "-static"' -o ipld-eth-server .
+
+# Get migration tool
 WORKDIR /
-ARG GOOSE_VER="v2.6.0"
-ADD https://github.com/pressly/goose/releases/download/${GOOSE_VER}/goose-linux64 ./goose
+ARG GOOSE_VER="v3.6.1"
+ADD https://github.com/pressly/goose/releases/download/${GOOSE_VER}/goose_linux_x86_64 ./goose
 RUN chmod +x ./goose
 
 # app container
@@ -28,14 +40,16 @@ USER $USER
 
 # chown first so dir is writable
 # note: using $USER is merged, but not in the stable release yet
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/$CONFIG_FILE config.toml
-COPY --chown=5000:5000 --from=builder /go/src/github.com/vulcanize/ipld-eth-server/entrypoint.sh .
+COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/ipld-eth-server/$CONFIG_FILE config.toml
+COPY --chown=5000:5000 --from=builder /go/src/github.com/cerc-io/ipld-eth-server/entrypoint.sh .
 
 
 # keep binaries immutable
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/ipld-eth-server ipld-eth-server
+COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-server/ipld-eth-server ipld-eth-server
 COPY --from=builder /goose goose
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/db/migrations migrations/vulcanizedb
-COPY --from=builder /go/src/github.com/vulcanize/ipld-eth-server/environments environments
+COPY --from=builder /go/src/github.com/cerc-io/ipld-eth-server/environments environments
+
+# Allow for debugging
+COPY --from=builder  /go/bin/dlv /usr/local/bin/
 
 ENTRYPOINT ["/app/entrypoint.sh"]

Makefile

@@ -3,16 +3,11 @@ BASE = $(GOPATH)/src/$(PACKAGE)
 PKGS = go list ./... | grep -v "^vendor/"
 
 # Tools
-## Testing library
-GINKGO = $(BIN)/ginkgo
-$(BIN)/ginkgo:
-	go get -u github.com/onsi/ginkgo/ginkgo
 
 ## Migration tool
 GOOSE = $(BIN)/goose
 $(BIN)/goose:
-	go get -u -d github.com/pressly/goose/cmd/goose
-	go build -tags='no_mysql no_sqlite' -o $(BIN)/goose github.com/pressly/goose/cmd/goose
+	go get -u github.com/pressly/goose/cmd/goose
 
 ## Source linter
 LINT = $(BIN)/golint
@@ -27,8 +22,9 @@ $(BIN)/gometalinter.v2:
 
 
 .PHONY: installtools
-installtools: | $(LINT) $(GOOSE) $(GINKGO)
+installtools: | $(LINT) $(GOOSE)
 	echo "Installing tools"
+	go mod download
 
 .PHONY: metalint
 metalint: | $(METALINT)
@@ -46,33 +42,31 @@ HOST_NAME = localhost
 PORT = 5432
 NAME =
 USER = postgres
-CONNECT_STRING=postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
+PASSWORD = password
+CONNECT_STRING=postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
 
 #Test
 TEST_DB = vulcanize_testing
-TEST_CONNECT_STRING = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
+TEST_CONNECT_STRING = postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(TEST_DB)?sslmode=disable
+TEST_CONNECT_STRING_LOCAL = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
 
 .PHONY: test
-test: | $(GINKGO) $(LINT)
+test:
 	go vet ./...
 	go fmt ./...
-	dropdb --if-exists $(TEST_DB)
-	createdb $(TEST_DB)
-	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
-	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" reset
-	make migrate NAME=$(TEST_DB)
-	$(GINKGO) -r --skipPackage=integration_tests,integration
+	go run github.com/onsi/ginkgo/ginkgo -r --skipPackage=test
 
 .PHONY: integrationtest
-integrationtest: | $(GINKGO) $(LINT)
+integrationtest:
 	go vet ./...
 	go fmt ./...
-	dropdb --if-exists $(TEST_DB)
-	createdb $(TEST_DB)
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" up
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" reset
-	make migrate NAME=$(TEST_DB)
-	$(GINKGO) -r integration_test/
+	go run github.com/onsi/ginkgo/ginkgo -r test/ -v
+
+.PHONY: test_local
+test_local:
+	go vet ./...
+	go fmt ./...
+	./scripts/run_unit_test.sh
 
 build:
 	go fmt ./...
@@ -141,4 +135,4 @@ import:
 ## Build docker image
 .PHONY: docker-build
 docker-build:
-	docker build -t vulcanize/ipld-eth-server .
+	docker build -t cerc-io/ipld-eth-server .

README.md

@@ -22,20 +22,20 @@ Additional, unique endpoints are exposed which utilize the new indexes and state
 
 ## Dependencies
 Minimal build dependencies
-* Go (1.13)
+* Go (1.19)
 * Git
 * GCC compiler
 * This repository
 
 External dependency
-* Postgres database populated by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer)
+* Postgres database populated by [ipld-eth-db](https://github.com/cerc-io/ipld-eth-db)
 
 ## Install
 Start by downloading ipld-eth-server and moving into the repo:
 
-`GO111MODULE=off go get -d github.com/vulcanize/ipld-eth-server`
+`GO111MODULE=off go get -d github.com/cerc-io/ipld-eth-server/v5`
 
-`cd $GOPATH/src/github.com/vulcanize/ipld-eth-server`
+`cd $GOPATH/src/github.com/cerc-io/ipld-eth-server/v5@v5.x.x`
 
 Then, build the binary:
 
@@ -67,11 +67,10 @@ The corresponding CLI flags can be found with the `./ipld-eth-server serve --hel
     wsPath = "127.0.0.1:8081" # $SERVER_WS_PATH
     httpPath = "127.0.0.1:8082" # $SERVER_HTTP_PATH
     graphql = true # $SERVER_GRAPHQL
-    graphqlEndpoint = "" # $SERVER_GRAPHQL_ENDPOINT
+    graphqlPath = "" # $SERVER_GRAPHQL_PATH
 
 [ethereum]
     chainID = "1" # $ETH_CHAIN_ID
-    defaultSender = "" # $ETH_DEFAULT_SENDER_ADDR
     rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
     httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
     nodeID = "arch1" # $ETH_NODE_ID
@@ -80,9 +79,9 @@ The corresponding CLI flags can be found with the `./ipld-eth-server serve --hel
     networkID = "1" # $ETH_NETWORK_ID
 ```
 
-The `database` fields are for connecting to a Postgres database that has been/is being populated by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer)  
-The `server` fields set the paths for exposing the ipld-eth-server endpoints  
-The `ethereum` fields set the chainID and default sender address to use for EVM simulation, and can optionally be used to configure a remote eth node to forward cache misses to  
+The `database` fields are for connecting to a Postgres database that has been/is being populated by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer)
+The `server` fields set the paths for exposing the ipld-eth-server endpoints
+The `ethereum` fields set the chainID and default sender address to use for EVM simulation, and can optionally be used to configure a remote eth node to forward cache misses to
 
 
 ### Endpoints
@@ -92,38 +91,38 @@ TODO: Port the IPLD RPC subscription endpoints after the decoupling
 #### Ethereum JSON-RPC
 ipld-eth-server currently recapitulates portions of the Ethereum JSON-RPC api standard.
 
-The currently supported standard endpoints are:  
-`eth_call`  
-`eth_getBalance`  
-`eth_getStorageAt`  
-`eth_getCode`  
-`eth_getProof`  
-`eth_blockNumber`  
-`eth_getHeaderByNumber`  
-`eth_getHeaderByHash`  
-`eth_getBlockByNumber`  
-`eth_getBlockByHash`  
-`eth_getTransactionCount`  
-`eth_getBlockTransactionCountByHash`  
-`eth_getBlockTransactionCountByNumber`  
-`eth_getTransactionByHash`  
-`eth_getRawTransactionByHash`  
-`eth_getTransactionByBlockHashAndIndex`  
-`eth_getTransactionByBlockNumberAndIndex`  
-`eth_getRawTransactionByBlockHashAndIndex`  
-`eth_getRawTransactionByBlockNumberAndIndex`  
-`eth_getTransactionReceipt`  
-`eth_getLogs`  
-`eth_getUncleCountByBlockHash`  
-`eth_getUncleCountByBlockNumber`  
-`eth_getUncleByBlockHashAndIndex`  
-`eth_getUncleByBlockNumberAndIndex`  
+The currently supported standard endpoints are:
+- `eth_call`
+- `eth_getBalance`
+- `eth_getStorageAt`
+- `eth_getCode`
+- `eth_getProof`
+- `eth_blockNumber`
+- `eth_getHeaderByNumber`
+- `eth_getHeaderByHash`
+- `eth_getBlockByNumber`
+- `eth_getBlockByHash`
+- `eth_getTransactionCount`
+- `eth_getBlockTransactionCountByHash`
+- `eth_getBlockTransactionCountByNumber`
+- `eth_getTransactionByHash`
+- `eth_getRawTransactionByHash`
+- `eth_getTransactionByBlockHashAndIndex`
+- `eth_getTransactionByBlockNumberAndIndex`
+- `eth_getRawTransactionByBlockHashAndIndex`
+- `eth_getRawTransactionByBlockNumberAndIndex`
+- `eth_getTransactionReceipt`
+- `eth_getLogs`
+- `eth_getUncleCountByBlockHash`
+- `eth_getUncleCountByBlockNumber`
+- `eth_getUncleByBlockHashAndIndex`
+- `eth_getUncleByBlockNumberAndIndex`
 
 TODO: Add the rest of the standard endpoints and unique endpoints (e.g. getSlice)
 
 ### Testing
-`make test` will run the unit tests  
-`make test` setups a clean `vulcanize_testing` db
+
+Follow steps in [test/README.md](./test/README.md)
 
 ## Monitoring
 

cmd/common.go

@@ -0,0 +1,37 @@
+// Copyright © 2021 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func addDatabaseFlags(command *cobra.Command) {
+	// database flags
+	command.PersistentFlags().String("database-name", "vulcanize_public", "database name")
+	command.PersistentFlags().Int("database-port", 5432, "database port")
+	command.PersistentFlags().String("database-hostname", "localhost", "database hostname")
+	command.PersistentFlags().String("database-user", "", "database user")
+	command.PersistentFlags().String("database-password", "", "database password")
+
+	// database flag bindings
+	viper.BindPFlag("database.name", command.PersistentFlags().Lookup("database-name"))
+	viper.BindPFlag("database.port", command.PersistentFlags().Lookup("database-port"))
+	viper.BindPFlag("database.hostname", command.PersistentFlags().Lookup("database-hostname"))
+	viper.BindPFlag("database.user", command.PersistentFlags().Lookup("database-user"))
+	viper.BindPFlag("database.password", command.PersistentFlags().Lookup("database-password"))
+}

cmd/root.go

@@ -18,17 +18,20 @@ package cmd
 
 import (
 	"fmt"
-	"os"
+	"path/filepath"
 	"strings"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/joho/godotenv"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/ipld-eth-server/pkg/prom"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/prom"
 )
 
 var (
 	cfgFile        string
+	envFile        string
 	subCommand     string
 	logWithCommand log.Entry
 )
@@ -46,24 +49,7 @@ func Execute() {
 }
 
 func initFuncs(cmd *cobra.Command, args []string) {
-	viper.BindEnv("log.file", "LOGRUS_FILE")
-	logfile := viper.GetString("log.file")
-	if logfile != "" {
-		file, err := os.OpenFile(logfile,
-			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
-		if err == nil {
-			log.Infof("Directing output to %s", logfile)
-			log.SetOutput(file)
-		} else {
-			log.SetOutput(os.Stdout)
-			log.Info("Failed to log to file, using default stdout")
-		}
-	} else {
-		log.SetOutput(os.Stdout)
-	}
-	if err := logLevel(); err != nil {
-		log.Fatal("Could not set log level: ", err)
-	}
+	log.Init()
 
 	if viper.GetBool("metrics") {
 		prom.Init()
@@ -79,31 +65,14 @@ func initFuncs(cmd *cobra.Command, args []string) {
 	}
 }
 
-func logLevel() error {
-	viper.BindEnv("log.level", "LOGRUS_LEVEL")
-	lvl, err := log.ParseLevel(viper.GetString("log.level"))
-	if err != nil {
-		return err
-	}
-	log.SetLevel(lvl)
-	if lvl > log.InfoLevel {
-		log.SetReportCaller(true)
-	}
-	log.Info("Log level set to ", lvl.String())
-	return nil
-}
-
 func init() {
 	cobra.OnInitialize(initConfig)
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	viper.AutomaticEnv()
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
-	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
-	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
-	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
-	rootCmd.PersistentFlags().String("database-user", "", "database user")
-	rootCmd.PersistentFlags().String("database-password", "", "database password")
+	rootCmd.PersistentFlags().StringVar(&envFile, "env", "", "environment file location")
+
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
 	rootCmd.PersistentFlags().String("log-file", "", "file path for logging")
@@ -114,11 +83,6 @@ func init() {
 	rootCmd.PersistentFlags().String("prom-http-addr", "127.0.0.1", "http host for prometheus")
 	rootCmd.PersistentFlags().String("prom-http-port", "8090", "http port for prometheus")
 
-	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
-	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
-	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
-	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
-	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
 
@@ -130,14 +94,32 @@ func init() {
 }
 
 func initConfig() {
+	if cfgFile == "" && envFile == "" {
+		log.Fatal("No configuration file specified, use --config , --env flag to provide configuration")
+	}
+
 	if cfgFile != "" {
-		viper.SetConfigFile(cfgFile)
-		if err := viper.ReadInConfig(); err == nil {
-			log.Printf("Using config file: %s", viper.ConfigFileUsed())
-		} else {
-			log.Fatal(fmt.Sprintf("Couldn't read config file: %s", err.Error()))
+		if filepath.Ext(cfgFile) != ".toml" {
+			log.Fatal("Provide .toml file for --config flag")
 		}
-	} else {
-		log.Warn("No config file passed with --config flag")
+
+		viper.SetConfigFile(cfgFile)
+		if err := viper.ReadInConfig(); err != nil {
+			log.Fatalf("Couldn't read config file: %s", err.Error())
+		}
+
+		log.Infof("Using config file: %s", viper.ConfigFileUsed())
+	}
+
+	if envFile != "" {
+		if filepath.Ext(envFile) != ".env" {
+			log.Fatal("Provide .env file for --env flag")
+		}
+
+		if err := godotenv.Load(envFile); err != nil {
+			log.Fatalf("Failed to set environment variable from env file: %s", err.Error())
+		}
+
+		log.Infof("Using env file: %s", envFile)
 	}
 }

cmd/serve.go

@@ -16,31 +16,34 @@
 package cmd
 
 import (
+	"errors"
+	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
+	"strings"
 	"sync"
+	"time"
 
-	"github.com/vulcanize/ipld-eth-server/pkg/graphql"
-
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
 	"github.com/ethereum/go-ethereum/rpc"
-	log "github.com/sirupsen/logrus"
+	"github.com/mailgun/groupcache/v2"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-
-	srpc "github.com/vulcanize/ipld-eth-server/pkg/rpc"
-	s "github.com/vulcanize/ipld-eth-server/pkg/serve"
-	v "github.com/vulcanize/ipld-eth-server/version"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/graphql"
+	srpc "github.com/cerc-io/ipld-eth-server/v5/pkg/rpc"
+	s "github.com/cerc-io/ipld-eth-server/v5/pkg/serve"
+	v "github.com/cerc-io/ipld-eth-server/v5/version"
 )
 
+var ErrNoRpcEndpoints = errors.New("no rpc endpoints is available")
+
 // serveCmd represents the serve command
 var serveCmd = &cobra.Command{
 	Use:   "serve",
 	Short: "serve chain data from PG-IPFS",
-	Long: `This command configures a VulcanizeDB ipld-eth-server.
-
-`,
+	Long:  `This command configures a VulcanizeDB ipld-eth-server.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		subCommand = cmd.CalledAs()
 		logWithCommand = *log.WithField("SubCommand", subCommand)
@@ -49,34 +52,50 @@ var serveCmd = &cobra.Command{
 }
 
 func serve() {
-	logWithCommand.Infof("running ipld-eth-server version: %s", v.VersionWithMeta)
+	logWithCommand.Infof("ipld-eth-server version: %s", v.VersionWithMeta)
 
-	var forwardPayloadChan chan eth.ConvertedPayload
 	wg := new(sync.WaitGroup)
-	logWithCommand.Debug("loading server configuration variables")
 	serverConfig, err := s.NewConfig()
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
-	logWithCommand.Infof("server config: %+v", serverConfig)
-	logWithCommand.Debug("initializing new server service")
+	logWithCommand.Debug("server config: %+v", serverConfig)
 	server, err := s.NewServer(serverConfig)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
+	if serverConfig.ForwardEthCalls {
+		logWithCommand.Info("Fowarding eth_call")
+	}
+	if serverConfig.ForwardGetStorageAt {
+		logWithCommand.Info("Fowarding eth_getStorageAt")
+	}
+	if serverConfig.ProxyOnError {
+		logWithCommand.Info("Proxy on error is enabled")
+	}
 
-	logWithCommand.Info("starting up server servers")
-	forwardPayloadChan = make(chan eth.ConvertedPayload, s.PayloadChanBufferSize)
-	server.Serve(wg, forwardPayloadChan)
+	server.Serve(wg)
 	if err := startServers(server, serverConfig); err != nil {
 		logWithCommand.Fatal(err)
 	}
-	graphQL, err := startGraphQL(server)
+	graphQL, err := startEthGraphQL(server, serverConfig)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
 
-	shutdown := make(chan os.Signal)
+	err = startGroupCacheService(serverConfig)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	if serverConfig.StateValidationEnabled {
+		go startStateTrieValidator(serverConfig, server)
+		logWithCommand.Info("state validator enabled")
+	} else {
+		logWithCommand.Debug("state validator disabled")
+	}
+
+	shutdown := make(chan os.Signal, 1)
 	signal.Notify(shutdown, os.Interrupt)
 	<-shutdown
 	if graphQL != nil {
@@ -87,27 +106,43 @@ func serve() {
 }
 
 func startServers(server s.Server, settings *s.Config) error {
-	logWithCommand.Info("starting up IPC server")
-	_, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
-	if err != nil {
-		return err
+	if settings.IPCEnabled {
+		logWithCommand.Debug("starting up IPC server")
+		_, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
+		if err != nil {
+			return err
+		}
+	} else {
+		logWithCommand.Debug("IPC server is disabled")
 	}
-	logWithCommand.Info("starting up WS server")
-	_, _, err = srpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb"}, nil, true)
-	if err != nil {
-		return err
+
+	if settings.WSEnabled {
+		logWithCommand.Debug("starting up WS server")
+		_, _, err := srpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb", "net"}, nil)
+		if err != nil {
+			return err
+		}
+	} else {
+		logWithCommand.Debug("WS server is disabled")
 	}
-	logWithCommand.Info("starting up HTTP server")
-	_, _, err = srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"eth"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
-	return err
+
+	if settings.HTTPEnabled {
+		logWithCommand.Debug("starting up HTTP server")
+		_, err := srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"vdb", "eth", "debug", "net"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
+		if err != nil {
+			return err
+		}
+	} else {
+		logWithCommand.Debug("HTTP server is disabled")
+	}
+
+	return nil
 }
 
-func startGraphQL(server s.Server) (graphQLServer *graphql.Service, err error) {
-	viper.BindEnv("server.graphql", "SERVER_GRAPHQL")
-	if viper.GetBool("server.graphql") {
-		logWithCommand.Info("starting up GraphQL server")
-		viper.BindEnv("server.graphqlEndpoint", "SERVER_GRAPHQL_ENDPOINT")
-		endPoint := viper.GetString("server.graphqlEndpoint")
+func startEthGraphQL(server s.Server, settings *s.Config) (graphQLServer *graphql.Service, err error) {
+	if settings.EthGraphqlEnabled {
+		logWithCommand.Debug("starting up ETH GraphQL server")
+		endPoint := settings.EthGraphqlEndpoint
 		if endPoint != "" {
 			graphQLServer, err = graphql.New(server.Backend(), endPoint, nil, []string{"*"}, rpc.HTTPTimeouts{})
 			if err != nil {
@@ -115,19 +150,126 @@ func startGraphQL(server s.Server) (graphQLServer *graphql.Service, err error) {
 			}
 			err = graphQLServer.Start(nil)
 		}
+	} else {
+		logWithCommand.Debug("ETH GraphQL server is disabled")
 	}
+
 	return
 }
 
+func startGroupCacheService(settings *s.Config) error {
+	gcc := settings.GroupCache
+
+	if gcc.Pool.Enabled {
+		logWithCommand.Debug("starting up groupcache pool HTTTP server")
+
+		pool := groupcache.NewHTTPPoolOpts(gcc.Pool.HttpEndpoint, &groupcache.HTTPPoolOptions{})
+		pool.Set(gcc.Pool.PeerHttpEndpoints...)
+
+		httpURL, err := url.Parse(gcc.Pool.HttpEndpoint)
+		if err != nil {
+			return err
+		}
+
+		server := http.Server{
+			Addr:    httpURL.Host,
+			Handler: pool,
+		}
+
+		// Start a HTTP server to listen for peer requests from the groupcache
+		go server.ListenAndServe()
+
+		logWithCommand.Infof("groupcache pool endpoint opened at %s", httpURL)
+	} else {
+		logWithCommand.Debug("Groupcache pool is disabled")
+	}
+
+	return nil
+}
+
+func startStateTrieValidator(config *s.Config, server s.Server) {
+	validateEveryNthBlock := config.StateValidationEveryNthBlock
+
+	var lastBlockNumber uint64
+	backend := server.Backend()
+
+	for {
+		time.Sleep(5 * time.Second)
+
+		block, err := backend.CurrentBlock()
+		if err != nil {
+			log.Error("Error fetching current block for state trie validator")
+			continue
+		}
+
+		stateRoot := block.Root()
+		blockNumber := block.NumberU64()
+		blockHash := block.Hash()
+
+		if validateEveryNthBlock <= 0 || // Used for static replicas where block number doesn't progress.
+			(blockNumber > lastBlockNumber) && (blockNumber%validateEveryNthBlock == 0) {
+
+			// The validate trie call will take a long time on mainnet, e.g. a few hours.
+			if err = backend.ValidateTrie(stateRoot); err != nil {
+				log.Fatalf("Error validating trie for block number %d hash %s state root %s",
+					blockNumber,
+					blockHash,
+					stateRoot,
+				)
+			}
+
+			log.Infof("Successfully validated trie for block number %d hash %s state root %s",
+				blockNumber,
+				blockHash,
+				stateRoot,
+			)
+
+			if validateEveryNthBlock <= 0 {
+				// Static replica, sleep a long-ish time (1/2 of cache expiry time) since we only need to keep the cache warm.
+				time.Sleep((time.Minute * time.Duration(config.GroupCache.StateDB.CacheExpiryInMins)) / 2)
+			}
+
+			lastBlockNumber = blockNumber
+		}
+	}
+}
+
+func parseRpcAddresses(value string) ([]*rpc.Client, error) {
+	rpcAddresses := strings.Split(value, ",")
+	rpcClients := make([]*rpc.Client, 0, len(rpcAddresses))
+	for _, address := range rpcAddresses {
+		rpcClient, err := rpc.Dial(address)
+		if err != nil {
+			logWithCommand.Errorf("couldn't connect to %s. Error: %s", address, err)
+			continue
+		}
+
+		rpcClients = append(rpcClients, rpcClient)
+	}
+
+	if len(rpcClients) == 0 {
+		logWithCommand.Error(ErrNoRpcEndpoints)
+		return nil, ErrNoRpcEndpoints
+	}
+
+	return rpcClients, nil
+}
+
 func init() {
 	rootCmd.AddCommand(serveCmd)
 
+	addDatabaseFlags(serveCmd)
+
 	// flags for all config variables
-	serveCmd.PersistentFlags().Bool("server-graphql", false, "turn on the graphql server")
-	serveCmd.PersistentFlags().String("server-graphql-endpoint", "", "endpoint url for graphql server")
-	serveCmd.PersistentFlags().String("server-ws-path", "", "vdb server ws path")
-	serveCmd.PersistentFlags().String("server-http-path", "", "vdb server http path")
-	serveCmd.PersistentFlags().String("server-ipc-path", "", "vdb server ipc path")
+	// eth graphql and json-rpc parameters
+	serveCmd.PersistentFlags().Bool("server-graphql", false, "turn on the eth graphql server")
+	serveCmd.PersistentFlags().String("server-graphql-path", "", "endpoint url for eth graphql server (host:port)")
+	serveCmd.PersistentFlags().Bool("server-http", true, "turn on the eth http json-rpc server")
+	serveCmd.PersistentFlags().String("server-http-path", "", "endpoint url for eth http json-rpc server (host:port)")
+	serveCmd.PersistentFlags().Bool("server-ws", false, "turn on the eth websocket json-rpc server")
+	serveCmd.PersistentFlags().String("server-ws-path", "", "endpoint url for eth websocket json-rpc server (host:port)")
+	serveCmd.PersistentFlags().Bool("server-ipc", false, "turn on the eth ipc json-rpc server")
+	serveCmd.PersistentFlags().String("server-ipc-path", "", "path for eth ipc json-rpc server")
 
 	serveCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
 	serveCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
@@ -137,12 +279,38 @@ func init() {
 	serveCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
 	serveCmd.PersistentFlags().String("eth-default-sender", "", "default sender address")
 	serveCmd.PersistentFlags().String("eth-rpc-gas-cap", "", "rpc gas cap (for eth_Call execution)")
+	serveCmd.PersistentFlags().String("eth-chain-config", "", "json chain config file location")
+	serveCmd.PersistentFlags().Bool("eth-supports-state-diff", false, "whether the proxy ethereum client supports statediffing endpoints")
+	serveCmd.PersistentFlags().Bool("eth-forward-eth-calls", false, "whether to immediately forward eth_calls to proxy client")
+	serveCmd.PersistentFlags().Bool("eth-proxy-on-error", true, "whether to forward all failed calls to proxy client")
+
+	// groupcache flags
+	serveCmd.PersistentFlags().Bool("gcache-pool-enabled", false, "turn on the groupcache pool")
+	serveCmd.PersistentFlags().String("gcache-pool-http-path", "", "http url for groupcache node")
+	serveCmd.PersistentFlags().StringArray("gcache-pool-http-peers", []string{}, "http urls for groupcache peers")
+	serveCmd.PersistentFlags().Int("gcache-statedb-cache-size", 16, "state DB cache size in MB")
+	serveCmd.PersistentFlags().Int("gcache-statedb-cache-expiry", 60, "state DB cache expiry time in mins")
+	serveCmd.PersistentFlags().Int("gcache-statedb-log-stats-interval", 60, "state DB cache stats log interval in secs")
+
+	// state validator flags
+	serveCmd.PersistentFlags().Bool("validator-enabled", false, "turn on the state validator")
+	serveCmd.PersistentFlags().Uint("validator-every-nth-block", 1500, "only validate every Nth block")
 
 	// and their bindings
+	// eth graphql server
 	viper.BindPFlag("server.graphql", serveCmd.PersistentFlags().Lookup("server-graphql"))
-	viper.BindPFlag("server.graphqlEndpoint", serveCmd.PersistentFlags().Lookup("server-graphql-endpoint"))
-	viper.BindPFlag("server.wsPath", serveCmd.PersistentFlags().Lookup("server-ws-path"))
+	viper.BindPFlag("server.graphqlPath", serveCmd.PersistentFlags().Lookup("server-graphql-path"))
+
+	// eth http json-rpc server
+	viper.BindPFlag("server.http", serveCmd.PersistentFlags().Lookup("server-http"))
 	viper.BindPFlag("server.httpPath", serveCmd.PersistentFlags().Lookup("server-http-path"))
+
+	// eth websocket json-rpc server
+	viper.BindPFlag("server.ws", serveCmd.PersistentFlags().Lookup("server-ws"))
+	viper.BindPFlag("server.wsPath", serveCmd.PersistentFlags().Lookup("server-ws-path"))
+
+	// eth ipc json-rpc server
+	viper.BindPFlag("server.ipc", serveCmd.PersistentFlags().Lookup("server-ipc"))
 	viper.BindPFlag("server.ipcPath", serveCmd.PersistentFlags().Lookup("server-ipc-path"))
 
 	viper.BindPFlag("ethereum.httpPath", serveCmd.PersistentFlags().Lookup("eth-http-path"))
@@ -151,6 +319,23 @@ func init() {
 	viper.BindPFlag("ethereum.genesisBlock", serveCmd.PersistentFlags().Lookup("eth-genesis-block"))
 	viper.BindPFlag("ethereum.networkID", serveCmd.PersistentFlags().Lookup("eth-network-id"))
 	viper.BindPFlag("ethereum.chainID", serveCmd.PersistentFlags().Lookup("eth-chain-id"))
-	viper.BindPFlag("ethereum.defaultSender", serveCmd.PersistentFlags().Lookup("eth-default-sender"))
 	viper.BindPFlag("ethereum.rpcGasCap", serveCmd.PersistentFlags().Lookup("eth-rpc-gas-cap"))
+	viper.BindPFlag("ethereum.chainConfig", serveCmd.PersistentFlags().Lookup("eth-chain-config"))
+	viper.BindPFlag("ethereum.supportsStateDiff", serveCmd.PersistentFlags().Lookup("eth-supports-state-diff"))
+	viper.BindPFlag("ethereum.forwardEthCalls", serveCmd.PersistentFlags().Lookup("eth-forward-eth-calls"))
+	viper.BindPFlag("ethereum.forwardGetStorageAt", serveCmd.PersistentFlags().Lookup("eth-forward-get-storage-at"))
+	viper.BindPFlag("ethereum.proxyOnError", serveCmd.PersistentFlags().Lookup("eth-proxy-on-error"))
+	viper.BindPFlag("ethereum.getLogsBlockLimit", serveCmd.PersistentFlags().Lookup("eth-getlogs-block-limit"))
+
+	// groupcache flags
+	viper.BindPFlag("groupcache.pool.enabled", serveCmd.PersistentFlags().Lookup("gcache-pool-enabled"))
+	viper.BindPFlag("groupcache.pool.httpEndpoint", serveCmd.PersistentFlags().Lookup("gcache-pool-http-path"))
+	viper.BindPFlag("groupcache.pool.peerHttpEndpoints", serveCmd.PersistentFlags().Lookup("gcache-pool-http-peers"))
+	viper.BindPFlag("groupcache.statedb.cacheSizeInMB", serveCmd.PersistentFlags().Lookup("gcache-statedb-cache-size"))
+	viper.BindPFlag("groupcache.statedb.cacheExpiryInMins", serveCmd.PersistentFlags().Lookup("gcache-statedb-cache-expiry"))
+	viper.BindPFlag("groupcache.statedb.logStatsIntervalInSecs", serveCmd.PersistentFlags().Lookup("gcache-statedb-log-stats-interval"))
+
+	// state validator flags
+	viper.BindPFlag("validator.enabled", serveCmd.PersistentFlags().Lookup("validator-enabled"))
+	viper.BindPFlag("validator.everyNthBlock", serveCmd.PersistentFlags().Lookup("validator-every-nth-block"))
 }

cmd/subscribe.go

@@ -1,173 +0,0 @@
-// Copyright © 2019 Vulcanize, Inc
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bytes"
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/rpc"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/client"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	w "github.com/vulcanize/ipld-eth-server/pkg/serve"
-)
-
-// subscribeCmd represents the subscribe command
-var subscribeCmd = &cobra.Command{
-	Use:   "subscribe",
-	Short: "This command is used to subscribe to the eth ipfs watcher data stream with the provided filters",
-	Long: `This command is for demo and testing purposes and is used to subscribe to the watcher with the provided subscription configuration parameters.
-It does not do anything with the data streamed from the watcher other than unpack it and print it out for demonstration purposes.`,
-	Run: func(cmd *cobra.Command, args []string) {
-		subCommand = cmd.CalledAs()
-		logWithCommand = *log.WithField("SubCommand", subCommand)
-		subscribe()
-	},
-}
-
-func init() {
-	rootCmd.AddCommand(subscribeCmd)
-}
-
-func subscribe() {
-	// Prep the subscription config/filters to be sent to the server
-	ethSubConfig, err := eth.NewEthSubscriptionConfig()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	// Create a new rpc client and a subscription streamer with that client
-	rpcClient, err := getRPCClient()
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	subClient := client.NewClient(rpcClient)
-
-	// Buffered channel for reading subscription payloads
-	payloadChan := make(chan w.SubscriptionPayload, 20000)
-
-	// Subscribe to the watcher service with the given config/filter parameters
-	sub, err := subClient.Stream(payloadChan, *ethSubConfig)
-	if err != nil {
-		logWithCommand.Fatal(err)
-	}
-	logWithCommand.Info("awaiting payloads")
-	// Receive response payloads and print out the results
-	for {
-		select {
-		case payload := <-payloadChan:
-			if payload.Err != "" {
-				logWithCommand.Error(payload.Err)
-				continue
-			}
-			var ethData eth.IPLDs
-			if err := rlp.DecodeBytes(payload.Data, &ethData); err != nil {
-				logWithCommand.Error(err)
-				continue
-			}
-			var header types.Header
-			err = rlp.Decode(bytes.NewBuffer(ethData.Header.Data), &header)
-			if err != nil {
-				logWithCommand.Error(err)
-				continue
-			}
-			fmt.Printf("Header number %d, hash %s\n", header.Number.Int64(), header.Hash().Hex())
-			fmt.Printf("header: %v\n", header)
-			for _, trxRlp := range ethData.Transactions {
-				var trx types.Transaction
-				buff := bytes.NewBuffer(trxRlp.Data)
-				stream := rlp.NewStream(buff, 0)
-				err := trx.DecodeRLP(stream)
-				if err != nil {
-					logWithCommand.Error(err)
-					continue
-				}
-				fmt.Printf("Transaction with hash %s\n", trx.Hash().Hex())
-				fmt.Printf("trx: %v\n", trx)
-			}
-			for _, rctRlp := range ethData.Receipts {
-				var rct types.Receipt
-				buff := bytes.NewBuffer(rctRlp.Data)
-				stream := rlp.NewStream(buff, 0)
-				err = rct.DecodeRLP(stream)
-				if err != nil {
-					logWithCommand.Error(err)
-					continue
-				}
-				fmt.Printf("Receipt with block hash %s, trx hash %s\n", rct.BlockHash.Hex(), rct.TxHash.Hex())
-				fmt.Printf("rct: %v\n", rct)
-				for _, l := range rct.Logs {
-					if len(l.Topics) < 1 {
-						logWithCommand.Error(fmt.Sprintf("log only has %d topics", len(l.Topics)))
-						continue
-					}
-					fmt.Printf("Log for block hash %s, trx hash %s, address %s, and with topic0 %s\n",
-						l.BlockHash.Hex(), l.TxHash.Hex(), l.Address.Hex(), l.Topics[0].Hex())
-					fmt.Printf("log: %v\n", l)
-				}
-			}
-			// This assumes leafs only
-			for _, stateNode := range ethData.StateNodes {
-				var acct state.Account
-				err = rlp.DecodeBytes(stateNode.IPLD.Data, &acct)
-				if err != nil {
-					logWithCommand.Error(err)
-					continue
-				}
-				fmt.Printf("Account for key %s, and root %s, with balance %s\n",
-					stateNode.StateLeafKey.Hex(), acct.Root.Hex(), acct.Balance.String())
-				fmt.Printf("state account: %+v\n", acct)
-			}
-			for _, storageNode := range ethData.StorageNodes {
-				fmt.Printf("Storage for state key %s ", storageNode.StateLeafKey.Hex())
-				fmt.Printf("with storage key %s\n", storageNode.StorageLeafKey.Hex())
-				var i []interface{}
-				err := rlp.DecodeBytes(storageNode.IPLD.Data, &i)
-				if err != nil {
-					logWithCommand.Error(err)
-					continue
-				}
-				// if a value node
-				if len(i) == 1 {
-					valueBytes, ok := i[0].([]byte)
-					if !ok {
-						continue
-					}
-					fmt.Printf("Storage leaf key: %s, and value hash: %s\n",
-						storageNode.StorageLeafKey.Hex(), common.BytesToHash(valueBytes).Hex())
-				}
-			}
-		case err = <-sub.Err():
-			logWithCommand.Fatal(err)
-		}
-	}
-}
-
-func getRPCClient() (*rpc.Client, error) {
-	vulcPath := viper.GetString("watcher.ethSubscription.wsPath")
-	if vulcPath == "" {
-		vulcPath = "ws://127.0.0.1:8080" // default to and try the default ws url if no path is provided
-	}
-	return rpc.Dial(vulcPath)
-}

cmd/validate.go

@@ -0,0 +1,87 @@
+// Copyright © 2021 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"time"
+
+	validator "github.com/cerc-io/eth-ipfs-state-validator/v5/pkg"
+	ipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	s "github.com/cerc-io/ipld-eth-server/v5/pkg/serve"
+)
+
+const GroupName = "statedb-validate"
+const CacheExpiryInMins = 8 * 60 // 8 hours
+const CacheSizeInMB = 16         // 16 MB
+
+var validateCmd = &cobra.Command{
+	Use:   "validate",
+	Short: "validate state",
+	Long:  `This command validates the trie for the given state root`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *log.WithField("SubCommand", subCommand)
+		validate()
+	},
+}
+
+func validate() {
+	config, err := s.NewConfig()
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+
+	stateRootStr := viper.GetString("stateRoot")
+	if stateRootStr == "" {
+		logWithCommand.Fatal("must provide a state root for state validation")
+	}
+
+	stateRoot := common.HexToHash(stateRootStr)
+	cacheSize := viper.GetInt("cacheSize")
+
+	ethDB := ipfsethdb.NewDatabase(config.DB, ipfsethdb.CacheConfig{
+		Name:           GroupName,
+		Size:           cacheSize * 1024 * 1024,
+		ExpiryDuration: time.Minute * time.Duration(CacheExpiryInMins),
+	})
+
+	val := validator.NewValidator(nil, ethDB)
+	if err = val.ValidateTrie(stateRoot); err != nil {
+		log.Fatal("Error validating state root")
+	}
+
+	stats := ethDB.(*ipfsethdb.Database).GetCacheStats()
+	log.Debugf("groupcache stats %+v", stats)
+
+	log.Info("Successfully validated state root")
+}
+
+func init() {
+	rootCmd.AddCommand(validateCmd)
+
+	addDatabaseFlags(validateCmd)
+
+	validateCmd.PersistentFlags().String("state-root", "", "root of the state trie we wish to validate")
+	viper.BindPFlag("stateRoot", validateCmd.PersistentFlags().Lookup("state-root"))
+
+	validateCmd.PersistentFlags().Int("cache-size", CacheSizeInMB, "cache size in MB")
+	viper.BindPFlag("cacheSize", validateCmd.PersistentFlags().Lookup("cache-size"))
+}

cmd/version.go

@@ -16,19 +16,17 @@
 package cmd
 
 import (
-	log "github.com/sirupsen/logrus"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
 	"github.com/spf13/cobra"
 
-	v "github.com/vulcanize/ipld-eth-server/version"
+	v "github.com/cerc-io/ipld-eth-server/v5/version"
 )
 
 // versionCmd represents the version command
 var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Prints the version of ipld-eth-server",
-	Long: `Use this command to fetch the version of ipld-eth-server
-
-Usage: ./ipld-eth-server version`,
+	Long:  `Use this command to fetch the version of ipld-eth-server`,
 	Run: func(cmd *cobra.Command, args []string) {
 		subCommand = cmd.CalledAs()
 		logWithCommand = *log.WithField("SubCommand", subCommand)

db/migrations/00001_create_ipfs_blocks_table.sql

@@ -1,8 +0,0 @@
--- +goose Up
-CREATE TABLE IF NOT EXISTS public.blocks (
-  key TEXT UNIQUE NOT NULL,
-  data BYTEA NOT NULL
-);
-
--- +goose Down
-DROP TABLE public.blocks;

db/migrations/00002_create_nodes_table.sql

@@ -1,12 +0,0 @@
--- +goose Up
-CREATE TABLE nodes (
-  id            SERIAL PRIMARY KEY,
-  client_name   VARCHAR,
-  genesis_block VARCHAR(66),
-  network_id    VARCHAR,
-  node_id       VARCHAR(128),
-  CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id)
-);
-
--- +goose Down
-DROP TABLE nodes;

db/migrations/00003_create_eth_schema.sql

@@ -1,5 +0,0 @@
--- +goose Up
-CREATE SCHEMA eth;
-
--- +goose Down
-DROP SCHEMA eth;

db/migrations/00004_create_eth_header_cids_table.sql

@@ -1,23 +0,0 @@
--- +goose Up
-CREATE TABLE eth.header_cids (
-  id                    SERIAL PRIMARY KEY,
-  block_number          BIGINT NOT NULL,
-  block_hash            VARCHAR(66) NOT NULL,
-  parent_hash           VARCHAR(66) NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  td                    NUMERIC NOT NULL,
-  node_id               INTEGER NOT NULL REFERENCES nodes (id) ON DELETE CASCADE,
-  reward                NUMERIC NOT NULL,
-  state_root            VARCHAR(66) NOT NULL,
-  tx_root               VARCHAR(66) NOT NULL,
-  receipt_root          VARCHAR(66) NOT NULL,
-  uncle_root            VARCHAR(66) NOT NULL,
-  bloom                 BYTEA NOT NULL,
-  timestamp             NUMERIC NOT NULL,
-  times_validated       INTEGER NOT NULL DEFAULT 1,
-  UNIQUE (block_number, block_hash)
-);
-
--- +goose Down
-DROP TABLE eth.header_cids;

db/migrations/00005_create_eth_uncle_cids_table.sql

@@ -1,14 +0,0 @@
--- +goose Up
-CREATE TABLE eth.uncle_cids (
-  id                    SERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  block_hash            VARCHAR(66) NOT NULL,
-  parent_hash           VARCHAR(66) NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  reward                NUMERIC NOT NULL,
-  UNIQUE (header_id, block_hash)
-);
-
--- +goose Down
-DROP TABLE eth.uncle_cids;

db/migrations/00006_create_eth_transaction_cids_table.sql

@@ -1,17 +0,0 @@
--- +goose Up
-CREATE TABLE eth.transaction_cids (
-  id                    SERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  tx_hash               VARCHAR(66) NOT NULL,
-  index                 INTEGER NOT NULL,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  dst                   VARCHAR(66) NOT NULL,
-  src                   VARCHAR(66) NOT NULL,
-  deployment            BOOL NOT NULL,
-  tx_data               BYTEA,
-  UNIQUE (header_id, tx_hash)
-);
-
--- +goose Down
-DROP TABLE eth.transaction_cids;

db/migrations/00007_create_eth_receipt_cids_table.sql

@@ -1,18 +0,0 @@
--- +goose Up
-CREATE TABLE eth.receipt_cids (
-  id                    SERIAL PRIMARY KEY,
-  tx_id                 INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  contract              VARCHAR(66),
-  contract_hash         VARCHAR(66),
-  topic0s               VARCHAR(66)[],
-  topic1s               VARCHAR(66)[],
-  topic2s               VARCHAR(66)[],
-  topic3s               VARCHAR(66)[],
-  log_contracts         VARCHAR(66)[],
-  UNIQUE (tx_id)
-);
-
--- +goose Down
-DROP TABLE eth.receipt_cids;

db/migrations/00008_create_eth_state_cids_table.sql

@@ -1,15 +0,0 @@
--- +goose Up
-CREATE TABLE eth.state_cids (
-  id                    SERIAL PRIMARY KEY,
-  header_id             INTEGER NOT NULL REFERENCES eth.header_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  state_leaf_key        VARCHAR(66),
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  state_path            BYTEA,
-  node_type             INTEGER NOT NULL,
-  diff                  BOOLEAN NOT NULL DEFAULT FALSE,
-  UNIQUE (header_id, state_path)
-);
-
--- +goose Down
-DROP TABLE eth.state_cids;

db/migrations/00009_create_eth_storage_cids_table.sql

@@ -1,15 +0,0 @@
--- +goose Up
-CREATE TABLE eth.storage_cids (
-  id                    SERIAL PRIMARY KEY,
-  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  storage_leaf_key      VARCHAR(66),
-  cid                   TEXT NOT NULL,
-  mh_key                TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  storage_path          BYTEA,
-  node_type             INTEGER NOT NULL,
-  diff                  BOOLEAN NOT NULL DEFAULT FALSE,
-  UNIQUE (state_id, storage_path)
-);
-
--- +goose Down
-DROP TABLE eth.storage_cids;

db/migrations/00010_create_eth_state_accouts_table.sql

@@ -1,13 +0,0 @@
--- +goose Up
-CREATE TABLE eth.state_accounts (
-  id                    SERIAL PRIMARY KEY,
-  state_id              INTEGER NOT NULL REFERENCES eth.state_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
-  balance               NUMERIC NOT NULL,
-  nonce                 INTEGER NOT NULL,
-  code_hash             BYTEA NOT NULL,
-  storage_root          VARCHAR(66) NOT NULL,
-  UNIQUE (state_id)
-);
-
--- +goose Down
-DROP TABLE eth.state_accounts;

db/migrations/00011_create_postgraphile_comments.sql

@@ -1,6 +0,0 @@
--- +goose Up
-COMMENT ON TABLE public.nodes IS E'@name NodeInfo';
-COMMENT ON TABLE eth.transaction_cids IS E'@name EthTransactionCids';
-COMMENT ON TABLE eth.header_cids IS E'@name EthHeaderCids';
-COMMENT ON COLUMN public.nodes.node_id IS E'@name ChainNodeID';
-COMMENT ON COLUMN eth.header_cids.node_id IS E'@name EthNodeID';

db/migrations/00012_add_chain_id_to_nodes.sql

@@ -1,21 +0,0 @@
--- +goose Up
-ALTER TABLE public.nodes
-ADD COLUMN chain_id INTEGER DEFAULT 1;
-
-ALTER TABLE public.nodes
-DROP CONSTRAINT node_uc;
-
-ALTER TABLE public.nodes
-ADD CONSTRAINT node_uc
-UNIQUE (genesis_block, network_id, node_id, chain_id);
-
--- +goose Down
-ALTER TABLE public.nodes
-DROP CONSTRAINT node_uc;
-
-ALTER TABLE public.nodes
-ADD CONSTRAINT node_uc
-UNIQUE (genesis_block, network_id, node_id);
-
-ALTER TABLE public.nodes
-DROP COLUMN chain_id;

db/migrations/00013_potgraphile_triggers.sql

@@ -1,69 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-CREATE FUNCTION eth.graphql_subscription() returns TRIGGER as $$
-declare
-    table_name text = TG_ARGV[0];
-    attribute text = TG_ARGV[1];
-    id text;
-begin
-    execute 'select $1.' || quote_ident(attribute)
-        using new
-        into id;
-    perform pg_notify('postgraphile:' || table_name,
-                      json_build_object(
-                              '__node__', json_build_array(
-                              table_name,
-                              id
-                          )
-                          )::text
-        );
-    return new;
-end;
-$$ language plpgsql;
--- +goose StatementEnd
-
-CREATE TRIGGER header_cids_ai
-    after INSERT ON eth.header_cids
-    for each row
-    execute procedure eth.graphql_subscription('header_cids', 'id');
-
-CREATE TRIGGER receipt_cids_ai
-    after INSERT ON eth.receipt_cids
-    for each row
-    execute procedure eth.graphql_subscription('receipt_cids', 'id');
-
-CREATE TRIGGER state_accounts_ai
-    after INSERT ON eth.state_accounts
-    for each row
-    execute procedure eth.graphql_subscription('state_accounts', 'id');
-
-CREATE TRIGGER state_cids_ai
-    after INSERT ON eth.state_cids
-    for each row
-    execute procedure eth.graphql_subscription('state_cids', 'id');
-
-CREATE TRIGGER storage_cids_ai
-    after INSERT ON eth.storage_cids
-    for each row
-    execute procedure eth.graphql_subscription('storage_cids', 'id');
-
-CREATE TRIGGER transaction_cids_ai
-    after INSERT ON eth.transaction_cids
-    for each row
-    execute procedure eth.graphql_subscription('transaction_cids', 'id');
-
-CREATE TRIGGER uncle_cids_ai
-    after INSERT ON eth.uncle_cids
-    for each row
-    execute procedure eth.graphql_subscription('uncle_cids', 'id');
-
--- +goose Down
-DROP TRIGGER uncle_cids_ai ON eth.uncle_cids;
-DROP TRIGGER transaction_cids_ai ON eth.transaction_cids;
-DROP TRIGGER storage_cids_ai ON eth.storage_cids;
-DROP TRIGGER state_cids_ai ON eth.state_cids;
-DROP TRIGGER state_accounts_ai ON eth.state_accounts;
-DROP TRIGGER receipt_cids_ai ON eth.receipt_cids;
-DROP TRIGGER header_cids_ai ON eth.header_cids;
-
-DROP FUNCTION eth.graphql_subscription();

db/migrations/00014_create_cid_indexes.sql

@@ -1,121 +0,0 @@
--- +goose Up
--- header indexes
-CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
-
-CREATE INDEX block_hash_index ON eth.header_cids USING btree (block_hash);
-
-CREATE INDEX header_cid_index ON eth.header_cids USING btree (cid);
-
-CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
-
-CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
-
-CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp);
-
--- transaction indexes
-CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
-
-CREATE INDEX tx_hash_index ON eth.transaction_cids USING btree (tx_hash);
-
-CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
-
-CREATE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
-
-CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
-
-CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
-
--- receipt indexes
-CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id);
-
-CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
-
-CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
-
-CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
-
-CREATE INDEX rct_contract_hash_index ON eth.receipt_cids USING btree (contract_hash);
-
-CREATE INDEX rct_topic0_index ON eth.receipt_cids USING gin (topic0s);
-
-CREATE INDEX rct_topic1_index ON eth.receipt_cids USING gin (topic1s);
-
-CREATE INDEX rct_topic2_index ON eth.receipt_cids USING gin (topic2s);
-
-CREATE INDEX rct_topic3_index ON eth.receipt_cids USING gin (topic3s);
-
-CREATE INDEX rct_log_contract_index ON eth.receipt_cids USING gin (log_contracts);
-
--- state node indexes
-CREATE INDEX state_header_id_index ON eth.state_cids USING btree (header_id);
-
-CREATE INDEX state_leaf_key_index ON eth.state_cids USING btree (state_leaf_key);
-
-CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid);
-
-CREATE INDEX state_mh_index ON eth.state_cids USING btree (mh_key);
-
-CREATE INDEX state_path_index ON eth.state_cids USING btree (state_path);
-
--- storage node indexes
-CREATE INDEX storage_state_id_index ON eth.storage_cids USING btree (state_id);
-
-CREATE INDEX storage_leaf_key_index ON eth.storage_cids USING btree (storage_leaf_key);
-
-CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid);
-
-CREATE INDEX storage_mh_index ON eth.storage_cids USING btree (mh_key);
-
-CREATE INDEX storage_path_index ON eth.storage_cids USING btree (storage_path);
-
--- state accounts indexes
-CREATE INDEX account_state_id_index ON eth.state_accounts USING btree (state_id);
-
-CREATE INDEX storage_root_index ON eth.state_accounts USING btree (storage_root);
-
--- +goose Down
--- state account indexes
-DROP INDEX eth.storage_root_index;
-DROP INDEX eth.account_state_id_index;
-
--- storage node indexes
-DROP INDEX eth.storage_path_index;
-DROP INDEX eth.storage_mh_index;
-DROP INDEX eth.storage_cid_index;
-DROP INDEX eth.storage_leaf_key_index;
-DROP INDEX eth.storage_state_id_index;
-
--- state node indexes
-DROP INDEX eth.state_path_index;
-DROP INDEX eth.state_mh_index;
-DROP INDEX eth.state_cid_index;
-DROP INDEX eth.state_leaf_key_index;
-DROP INDEX eth.state_header_id_index;
-
--- receipt indexes
-DROP INDEX eth.rct_log_contract_index;
-DROP INDEX eth.rct_topic3_index;
-DROP INDEX eth.rct_topic2_index;
-DROP INDEX eth.rct_topic1_index;
-DROP INDEX eth.rct_topic0_index;
-DROP INDEX eth.rct_contract_hash_index;
-DROP INDEX eth.rct_contract_index;
-DROP INDEX eth.rct_mh_index;
-DROP INDEX eth.rct_cid_index;
-DROP INDEX eth.rct_tx_id_index;
-
--- transaction indexes
-DROP INDEX eth.tx_src_index;
-DROP INDEX eth.tx_dst_index;
-DROP INDEX eth.tx_mh_index;
-DROP INDEX eth.tx_cid_index;
-DROP INDEX eth.tx_hash_index;
-DROP INDEX eth.tx_header_id_index;
-
--- header indexes
-DROP INDEX eth.timestamp_index;
-DROP INDEX eth.state_root_index;
-DROP INDEX eth.header_mh_index;
-DROP INDEX eth.header_cid_index;
-DROP INDEX eth.block_hash_index;
-DROP INDEX eth.block_number_index;

db/migrations/00015_create_canonical_hash_finder_functions.sql

@@ -1,48 +0,0 @@
--- +goose Up
--- +goose StatementBegin
--- returns the number of child headers that reference backwards to the header with the provided hash
-CREATE OR REPLACE FUNCTION header_weight(hash VARCHAR(66)) RETURNS BIGINT
-AS $$
-  WITH RECURSIVE validator AS (
-          SELECT block_hash, parent_hash, block_number
-          FROM eth.header_cids
-          WHERE block_hash = hash
-      UNION
-          SELECT eth.header_cids.block_hash, eth.header_cids.parent_hash, eth.header_cids.block_number
-          FROM eth.header_cids
-          INNER JOIN validator
-            ON eth.header_cids.parent_hash = validator.block_hash
-            AND eth.header_cids.block_number = validator.block_number + 1
-  )
-  SELECT COUNT(*) FROM validator;
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose StatementBegin
--- returns the id for the header at the provided height which is heaviest
-CREATE OR REPLACE FUNCTION canonical_header(height BIGINT) RETURNS INT AS
-$BODY$
-DECLARE
-  current_weight INT;
-  heaviest_weight INT DEFAULT 0;
-  heaviest_id INT;
-  r eth.header_cids%ROWTYPE;
-BEGIN
-  FOR r IN SELECT * FROM eth.header_cids
-  WHERE block_number = height
-  LOOP
-    SELECT INTO current_weight * FROM header_weight(r.block_hash);
-    IF current_weight > heaviest_weight THEN
-        heaviest_weight := current_weight;
-        heaviest_id := r.id;
-    END IF;
-  END LOOP;
-  RETURN heaviest_id;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose Down
-DROP FUNCTION header_weight;
-DROP FUNCTION canonical_header;

db/migrations/00016_remove_deployment_from_tx_cids.sql

@@ -1,7 +0,0 @@
--- +goose Up
-ALTER TABLE eth.transaction_cids
-DROP COLUMN deployment;
-
--- +goose Down
-ALTER TABLE eth.transaction_cids
-ADD COLUMN deployment BOOL NOT NULL DEFAULT FALSE;

db/migrations/00017_state_and_storage_ids_use_big_serial.sql

@@ -1,25 +0,0 @@
--- +goose Up
-ALTER TABLE eth.storage_cids
-ALTER COLUMN state_id TYPE BIGINT;
-
-ALTER TABLE eth.state_accounts
-ALTER COLUMN state_id TYPE BIGINT;
-
-ALTER TABLE eth.state_cids
-ALTER COLUMN id TYPE BIGINT;
-
-ALTER TABLE eth.storage_cids
-ALTER COLUMN id TYPE BIGINT;
-
--- +goose Down
-ALTER TABLE eth.storage_cids
-ALTER COLUMN id TYPE INTEGER;
-
-ALTER TABLE eth.state_cids
-ALTER COLUMN id TYPE INTEGER;
-
-ALTER TABLE eth.state_accounts
-ALTER COLUMN state_id TYPE INTEGER;
-
-ALTER TABLE eth.storage_cids
-ALTER COLUMN state_id TYPE INTEGER;

db/migrations/00018_create_check_if_removed_functions.sql

@@ -1,39 +0,0 @@
--- +goose Up
--- +goose StatementBegin
--- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash
-CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
-AS $$
-  SELECT exists(SELECT 1
-					FROM eth.storage_cids
-            INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
-            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-					WHERE storage_path = path
-					AND block_number > height
-					AND block_number <= (SELECT block_number
-										FROM eth.header_cids
-										WHERE block_hash = hash)
-					AND storage_cids.node_type = 3
-					LIMIT 1);
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose StatementBegin
--- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash
-CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
-AS $$
-  SELECT exists(SELECT 1
-					FROM eth.state_cids
-            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-					WHERE state_path = path
-					AND block_number > height
-					AND block_number <= (SELECT block_number
-										FROM eth.header_cids
-										WHERE block_hash = hash)
-					AND state_cids.node_type = 3
-					LIMIT 1);
-$$ LANGUAGE SQL;
--- +goose StatementEnd
-
--- +goose Down
-DROP FUNCTION was_storage_removed;
-DROP FUNCTION was_state_removed;

db/migrations/00019_updated_canonical_hash_finder_functions.sql

@@ -1,121 +0,0 @@
--- +goose Up
--- +goose StatementBegin
-CREATE TYPE child_result AS (
-  has_child BOOLEAN,
-  children eth.header_cids[]
-);
-
-CREATE OR REPLACE FUNCTION has_child(hash VARCHAR(66), height BIGINT) RETURNS child_result AS
-$BODY$
-DECLARE
-  child_height INT;
-  temp_child eth.header_cids;
-  new_child_result child_result;
-BEGIN
-  child_height = height + 1;
-  -- short circuit if there are no children
-  SELECT exists(SELECT 1
-                FROM eth.header_cids
-                WHERE parent_hash = hash
-                AND block_number = child_height
-                LIMIT 1)
-  INTO new_child_result.has_child;
-  -- collect all the children for this header
-  IF new_child_result.has_child THEN
-    FOR temp_child IN
-      SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
-    LOOP
-      new_child_result.children = array_append(new_child_result.children, temp_child);
-    END LOOP;
-  END IF;
-  RETURN new_child_result;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose StatementBegin
-CREATE OR REPLACE FUNCTION canonical_header_from_array(headers eth.header_cids[]) RETURNS eth.header_cids AS
-$BODY$
-DECLARE
-  canonical_header eth.header_cids;
-  canonical_child eth.header_cids;
-  header eth.header_cids;
-  current_child_result child_result;
-  child_headers eth.header_cids[];
-  current_header_with_child eth.header_cids;
-  has_children_count INT DEFAULT 0;
-BEGIN
-  -- for each header in the provided set
-  FOREACH header IN ARRAY headers
-  LOOP
-    -- check if it has any children
-    current_child_result = has_child(header.block_hash, header.block_number);
-    IF current_child_result.has_child THEN
-      -- if it does, take note
-      has_children_count = has_children_count + 1;
-      current_header_with_child = header;
-      -- and add the children to the growing set of child headers
-      child_headers = array_cat(child_headers, current_child_result.children);
-    END IF;
-  END LOOP;
-  -- if none of the headers had children, none is more canonical than the other
-  IF has_children_count = 0 THEN
-    -- return the first one selected
-    SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
-  -- if only one header had children, it can be considered the heaviest/canonical header of the set
-  ELSIF has_children_count = 1 THEN
-    -- return the only header with a child
-    canonical_header = current_header_with_child;
-  -- if there are multiple headers with children
-  ELSE
-    -- find the canonical header from the child set
-    canonical_child = canonical_header_from_array(child_headers);
-    -- the header that is parent to this header, is the canonical header at this level
-    SELECT * INTO canonical_header FROM unnest(headers)
-    WHERE block_hash = canonical_child.parent_hash;
-  END IF;
-  RETURN canonical_header;
-END
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose StatementBegin
-CREATE OR REPLACE FUNCTION canonical_header_id(height BIGINT) RETURNS INTEGER AS
-$BODY$
-DECLARE
-  canonical_header eth.header_cids;
-  headers eth.header_cids[];
-  header_count INT;
-  temp_header eth.header_cids;
-BEGIN
-  -- collect all headers at this height
-  FOR temp_header IN
-    SELECT * FROM eth.header_cids WHERE block_number = height
-  LOOP
-    headers = array_append(headers, temp_header);
-  END LOOP;
-  -- count the number of headers collected
-  header_count = array_length(headers, 1);
-  -- if we have less than 1 header, return NULL
-  IF header_count IS NULL OR header_count < 1 THEN
-    RETURN NULL;
-  -- if we have one header, return its id
-  ELSIF header_count = 1 THEN
-    RETURN headers[1].id;
-  -- if we have multiple headers we need to determine which one is canonical
-  ELSE
-    canonical_header = canonical_header_from_array(headers);
-    RETURN canonical_header.id;
-  END IF;
-END;
-$BODY$
-LANGUAGE 'plpgsql';
--- +goose StatementEnd
-
--- +goose Down
-DROP FUNCTION canonical_header_id;
-DROP FUNCTION canonical_header_from_array;
-DROP FUNCTION has_child;
-DROP TYPE child_result;

docker-compose.yml

@@ -1,78 +0,0 @@
-version: '3.2'
-
-services:
-  dapptools:
-    restart: unless-stopped
-    image: vulcanize/dapptools:v0.29.0-statediff-0.0.2
-    ports:
-      - "127.0.0.1:8545:8545"
-      - "127.0.0.1:8546:8546"
-
-  db:
-    restart: always
-    image: postgres:10.12-alpine
-    environment:
-      POSTGRES_USER: "vdbm"
-      POSTGRES_DB: "vulcanize_public"
-      POSTGRES_PASSWORD: "password"
-    volumes:
-     - vdb_db_eth_server:/var/lib/postgresql/data
-    ports:
-     - "127.0.0.1:8077:5432"
-
-  eth-indexer:
-    restart: unless-stopped
-    depends_on:
-      - db
-      - dapptools
-    image: vulcanize/ipld-eth-indexer:v0.3.0-alpha
-    environment:
-      DATABASE_NAME: vulcanize_public
-      DATABASE_HOSTNAME: db
-      DATABASE_PORT: 5432
-      DATABASE_USER: vdbm
-      DATABASE_PASSWORD: password
-      ETH_WS_PATH: "dapptools:8546"
-      ETH_HTTP_PATH: "dapptools:8545"
-      ETH_CHAIN_ID: 4
-      ETH_NETWORK_ID: 4
-      VDB_COMMAND: sync
-
-  eth-server:
-    depends_on:
-      - db
-    build:
-      context: ./
-      cache_from:
-       - alpine:latest
-       - golang:1.13-alpine
-    environment:
-      VDB_COMMAND: "serve"
-      DATABASE_NAME: "vulcanize_public"
-      DATABASE_HOSTNAME: "db"
-      DATABASE_PORT: 5432
-      DATABASE_USER: "vdbm"
-      DATABASE_PASSWORD: "password"
-      SERVER_WS_PATH: "0.0.0.0:8081"
-      SERVER_HTTP_PATH: "0.0.0.0:8082"
-    ports:
-     - "127.0.0.1:8080:8080"
-     - "127.0.0.1:8081:8081"
-
-  graphql:
-    restart: unless-stopped
-    depends_on:
-      - db
-    image: vulcanize/postgraphile:v1.0.1
-    environment:
-      - PG_HOST=db
-      - PG_PORT=5432
-      - PG_DATABASE=vulcanize_public
-      - PG_USER=vdbm
-      - PG_PASSWORD=password
-      - SCHEMA=public,eth
-    ports:
-      - "127.0.0.1:5000:5000"
-
-volumes:
-  vdb_db_eth_server:

documentation/apis.md

@@ -12,7 +12,7 @@ We can expose a number of different APIs for remote access to ipld-eth-server da
 ipld-eth-server stores all processed data in Postgres using PG-IPFS, this includes all of the IPLD objects.
 [Postgraphile](https://www.graphile.org/postgraphile/) can be used to expose GraphQL endpoints for the Postgres tables.
 
-e.g. 
+e.g.
 
 `postgraphile --plugins @graphile/pg-pubsub --subscriptions --simple-subscriptions -c postgres://localhost:5432/vulcanize_public?sslmode=disable -s public,btc,eth -a -j`
 
@@ -33,16 +33,16 @@ by ipld-eth-server to filter and return a requested subset of chain data to the
 An example of how to subscribe to a real-time Ethereum data feed from ipld-eth-server using the `Stream` RPC method is provided below
 
 ```go
-    package main 
+    package main
 
     import (
     	"github.com/ethereum/go-ethereum/rlp"
     	"github.com/ethereum/go-ethereum/rpc"
     	"github.com/spf13/viper"
-    	
-        "github.com/vulcanize/ipld-eth-server/pkg/client"
-        "github.com/vulcanize/ipld-eth-server/pkg/eth"
-        "github.com/vulcanize/ipld-eth-server/pkg/watch"
+
+        "github.com/cerc-io/ipld-eth-server/v5/pkg/client"
+        "github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+        "github.com/cerc-io/ipld-eth-server/v5/pkg/watch"
     )
 
     config, _ := eth.NewEthSubscriptionConfig()
@@ -153,16 +153,16 @@ the addresses in the `addresses` fields are pre-hashed ETH addresses.
 An example of how to subscribe to a real-time Bitcoin data feed from ipld-eth-server using the `Stream` RPC method is provided below
 
 ```go
-    package main 
+    package main
 
     import (
     	"github.com/ethereum/go-ethereum/rlp"
     	"github.com/ethereum/go-ethereum/rpc"
     	"github.com/spf13/viper"
-    	
-    	"github.com/vulcanize/ipld-eth-server/pkg/btc"
-    	"github.com/vulcanize/ipld-eth-server/pkg/client"
-    	"github.com/vulcanize/ipld-eth-server/pkg/watch"
+
+    	"github.com/vulcanize/ipld-eth-server/v5/pkg/btc"
+    	"github.com/vulcanize/ipld-eth-server/v5/pkg/client"
+    	"github.com/vulcanize/ipld-eth-server/v5/pkg/watch"
     )
 
     config, _ := btc.NewBtcSubscriptionConfig()

entrypoint.sh

@@ -1,25 +1,14 @@
 #!/bin/sh
-# Runs the db migrations and starts the watcher services
-
-# Construct the connection string for postgres
-VDB_PG_CONNECT=postgresql://$DATABASE_USER:$DATABASE_PASSWORD@$DATABASE_HOSTNAME:$DATABASE_PORT/$DATABASE_NAME?sslmode=disable
-
-# Run the DB migrations
-echo "Connecting with: $VDB_PG_CONNECT"
-echo "Running database migrations"
-./goose -dir migrations/vulcanizedb postgres "$VDB_PG_CONNECT" up
-rv=$?
-
-if [ $rv != 0 ]; then
-  echo "Could not run migrations. Are the database details correct?"
-  exit 1
-fi
-
 
 echo "Beginning the ipld-eth-server process"
 
-echo running: ./ipld-eth-server ${VDB_COMMAND} --config=config.toml
-./ipld-eth-server ${VDB_COMMAND} --config=config.toml
+START_CMD="./ipld-eth-server"
+if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
+    START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec `pwd`/ipld-eth-server --continue --"
+fi
+
+echo running: $START_CMD ${VDB_COMMAND} --config=`pwd`/config.toml
+$START_CMD ${VDB_COMMAND} --config=`pwd`/config.toml
 rv=$?
 
 if [ $rv != 0 ]; then

environments/example.toml

@@ -1,5 +1,5 @@
 [database]
-    name     = "vulcanize_public" # $DATABASE_NAME
+    name     = "cerc_testing" # $DATABASE_NAME
     hostname = "localhost" # $DATABASE_HOSTNAME
     port     = 5432 # $DATABASE_PORT
     user     = "postgres" # $DATABASE_USER
@@ -9,18 +9,25 @@
     level = "info" # $LOGRUS_LEVEL
 
 [server]
+    ipc = false
     ipcPath = "~/.vulcanize/vulcanize.ipc" # $SERVER_IPC_PATH
-    wsPath = "127.0.0.1:8081" # $SERVER_WS_PATH
-    httpPath = "127.0.0.1:8082" # $SERVER_HTTP_PATH
+    ws = true
+    wsPath = "127.0.0.1:8080" # $SERVER_WS_PATH
+    http = true
+    httpPath = "127.0.0.1:8081" # $SERVER_HTTP_PATH
     graphql = true # $SERVER_GRAPHQL
-    graphqlEndpoint = "127.0.0.1:8083" # $SERVER_GRAPHQL_ENDPOINT
+    graphqlPath = "127.0.0.1:8082" # $SERVER_GRAPHQL_PATH
 
 [ethereum]
+    chainConfig = "./chain.json" # ETH_CHAIN_CONFIG
     chainID = "1" # $ETH_CHAIN_ID
-    defaultSender = "" # $ETH_DEFAULT_SENDER_ADDR
     rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
     httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF
+    stateDiffTimeout = "4m" # $ETH_STATEDIFF_TIMEOUT
+    forwardEthCalls = false # $ETH_FORWARD_ETH_CALLS
+    proxyOnError = true # $ETH_PROXY_ON_ERROR
     nodeID = "arch1" # $ETH_NODE_ID
     clientName = "Geth" # $ETH_CLIENT_NAME
     genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
-    networkID = "1" # $ETH_NETWORK_ID
+    networkID = "1" # $ETH_NETWORK_ID

environments/subscribeExample.toml

@@ -1,30 +0,0 @@
-[watcher]
-    [watcher.ethSubscription]
-        historicalData = false
-        historicalDataOnly = false
-        startingBlock = 0
-        endingBlock = 0
-        wsPath = "ws://127.0.0.1:8080"
-        [watcher.ethSubscription.headerFilter]
-            off = false
-            uncles = false
-        [watcher.ethSubscription.txFilter]
-            off = false
-            src = []
-            dst = []
-        [watcher.ethSubscription.receiptFilter]
-            off = false
-            contracts = []
-            topic0s = []
-            topic1s = []
-            topic2s = []
-            topic3s = []
-        [watcher.ethSubscription.stateFilter]
-            off = false
-            addresses = []
-           intermediateNodes = false
-        [watcher.ethSubscription.storageFilter]
-            off = true
-            addresses = []
-            storageKeys = []
-            intermediateNodes = false

go.mod

@@ -1,26 +1,288 @@
-module github.com/vulcanize/ipld-eth-server
+module github.com/cerc-io/ipld-eth-server/v5
 
-go 1.13
+go 1.19
 
 require (
-	github.com/ethereum/go-ethereum v1.9.11
-	github.com/graph-gophers/graphql-go v0.0.0-20201003130358-c5bdf3b1108e
-	github.com/ipfs/go-block-format v0.0.2
-	github.com/ipfs/go-cid v0.0.5
-	github.com/ipfs/go-ipfs-blockstore v1.0.0
-	github.com/ipfs/go-ipfs-ds-help v1.0.0
-	github.com/ipfs/go-ipld-format v0.2.0
-	github.com/jmoiron/sqlx v1.2.0
-	github.com/lib/pq v1.5.2
-	github.com/multiformats/go-multihash v0.0.13
-	github.com/onsi/ginkgo v1.12.1
-	github.com/onsi/gomega v1.10.1
-	github.com/prometheus/client_golang v1.5.1
-	github.com/sirupsen/logrus v1.6.0
-	github.com/spf13/cobra v1.0.0
-	github.com/spf13/viper v1.7.0
-	github.com/vulcanize/ipld-eth-indexer v0.6.0-alpha
-	github.com/vulcanize/pg-ipfs-ethdb v0.0.1-alpha
+	github.com/cerc-io/eth-ipfs-state-validator/v5 v5.0.0-alpha
+	github.com/cerc-io/ipfs-ethdb/v5 v5.0.0-alpha
+	github.com/cerc-io/ipld-eth-statedb v0.0.5-alpha
+	github.com/ethereum/go-ethereum v1.11.6
+	github.com/google/uuid v1.3.0
+	github.com/graph-gophers/graphql-go v1.3.0
+	github.com/ipfs/go-cid v0.3.2
+	github.com/jmoiron/sqlx v1.3.5
+	github.com/joho/godotenv v1.4.0
+	github.com/lib/pq v1.10.7
+	github.com/machinebox/graphql v0.2.2
+	github.com/mailgun/groupcache/v2 v2.3.0
+	github.com/onsi/ginkgo/v2 v2.9.2
+	github.com/onsi/gomega v1.27.4
+	github.com/prometheus/client_golang v1.14.0
+	github.com/sirupsen/logrus v1.9.0
+	github.com/spf13/cobra v1.4.0
+	github.com/spf13/viper v1.11.0
+	gorm.io/driver/postgres v1.3.7
+	gorm.io/gorm v1.23.5
 )
 
-replace github.com/ethereum/go-ethereum v1.9.11 => github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.8
+require (
+	bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect
+	github.com/DataDog/zstd v1.5.2 // indirect
+	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
+	github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.9.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect
+	github.com/cockroachdb/redact v1.1.3 // indirect
+	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
+	github.com/cskr/pubsub v1.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
+	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/deepmap/oapi-codegen v1.8.2 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/edsrzf/mmap-go v1.0.0 // indirect
+	github.com/elastic/gosigar v0.14.2 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
+	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
+	github.com/flynn/noise v1.0.0 // indirect
+	github.com/francoispqt/gojay v1.2.13 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
+	github.com/georgysavva/scany v0.2.9 // indirect
+	github.com/getsentry/sentry-go v0.18.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
+	github.com/golang/mock v1.6.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-bexpr v0.1.10 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
+	github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect
+	github.com/huin/goupnp v1.0.3 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
+	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c // indirect
+	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-bitfield v1.0.0 // indirect
+	github.com/ipfs/go-bitswap v0.11.0 // indirect
+	github.com/ipfs/go-block-format v0.0.3 // indirect
+	github.com/ipfs/go-blockservice v0.5.0 // indirect
+	github.com/ipfs/go-cidutil v0.1.0 // indirect
+	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-delegated-routing v0.7.0 // indirect
+	github.com/ipfs/go-ds-measure v0.2.0 // indirect
+	github.com/ipfs/go-fetcher v1.6.1 // indirect
+	github.com/ipfs/go-filestore v1.2.0 // indirect
+	github.com/ipfs/go-fs-lock v0.0.7 // indirect
+	github.com/ipfs/go-graphsync v0.14.1 // indirect
+	github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
+	github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
+	github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
+	github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
+	github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect
+	github.com/ipfs/go-ipfs-keystore v0.1.0 // indirect
+	github.com/ipfs/go-ipfs-pinner v0.2.1 // indirect
+	github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
+	github.com/ipfs/go-ipfs-pq v0.0.2 // indirect
+	github.com/ipfs/go-ipfs-provider v0.8.1 // indirect
+	github.com/ipfs/go-ipfs-routing v0.3.0 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
+	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
+	github.com/ipfs/go-ipld-format v0.4.0 // indirect
+	github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
+	github.com/ipfs/go-ipns v0.3.0 // indirect
+	github.com/ipfs/go-libipfs v0.2.0 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.5.1 // indirect
+	github.com/ipfs/go-merkledag v0.9.0 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-mfs v0.2.1 // indirect
+	github.com/ipfs/go-namesys v0.6.0 // indirect
+	github.com/ipfs/go-path v0.3.0 // indirect
+	github.com/ipfs/go-peertaskqueue v0.8.0 // indirect
+	github.com/ipfs/go-unixfs v0.4.2 // indirect
+	github.com/ipfs/go-unixfsnode v1.5.1 // indirect
+	github.com/ipfs/go-verifcid v0.0.2 // indirect
+	github.com/ipfs/interface-go-ipfs-core v0.8.2 // indirect
+	github.com/ipfs/kubo v0.18.1 // indirect
+	github.com/ipld/edelweiss v0.2.0 // indirect
+	github.com/ipld/go-codec-dagpb v1.5.0 // indirect
+	github.com/ipld/go-ipld-prime v0.19.0 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.14.0 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.2 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgtype v1.14.0 // indirect
+	github.com/jackc/pgx/v4 v4.18.1 // indirect
+	github.com/jackc/puddle v1.3.0 // indirect
+	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
+	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.4 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.1 // indirect
+	github.com/koron/go-ssdp v0.0.3 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/libp2p/go-cidranger v1.1.0 // indirect
+	github.com/libp2p/go-doh-resolver v0.4.0 // indirect
+	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
+	github.com/libp2p/go-libp2p v0.24.2 // indirect
+	github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
+	github.com/libp2p/go-libp2p-kad-dht v0.20.0 // indirect
+	github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect
+	github.com/libp2p/go-libp2p-pubsub v0.8.3 // indirect
+	github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
+	github.com/libp2p/go-libp2p-record v0.2.0 // indirect
+	github.com/libp2p/go-libp2p-routing-helpers v0.6.0 // indirect
+	github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
+	github.com/libp2p/go-mplex v0.7.0 // indirect
+	github.com/libp2p/go-msgio v0.2.0 // indirect
+	github.com/libp2p/go-nat v0.1.0 // indirect
+	github.com/libp2p/go-netroute v0.2.1 // indirect
+	github.com/libp2p/go-openssl v0.1.0 // indirect
+	github.com/libp2p/go-reuseport v0.2.0 // indirect
+	github.com/libp2p/go-yamux/v4 v4.0.0 // indirect
+	github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
+	github.com/lucas-clemente/quic-go v0.31.1 // indirect
+	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/marten-seemann/qpack v0.3.0 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect
+	github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect
+	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
+	github.com/marten-seemann/webtransport-go v0.4.3 // indirect
+	github.com/matryer/is v1.4.1 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-pointer v0.0.1 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/miekg/dns v1.1.50 // indirect
+	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
+	github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/mitchellh/pointerstructure v1.2.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr v0.8.0 // indirect
+	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
+	github.com/multiformats/go-multibase v0.1.1 // indirect
+	github.com/multiformats/go-multicodec v0.7.0 // indirect
+	github.com/multiformats/go-multihash v0.2.1 // indirect
+	github.com/multiformats/go-multistream v0.3.3 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/opencontainers/runtime-spec v1.0.2 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
+	github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 // indirect
+	github.com/pganalyze/pg_query_go/v2 v2.1.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.39.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/raulk/go-watchdog v1.3.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/rs/cors v1.7.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/samber/lo v1.36.0 // indirect
+	github.com/segmentio/fasthash v1.0.3 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
+	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/afero v1.8.2 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/status-im/keycard-go v0.2.0 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/stretchr/testify v1.8.1 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
+	github.com/thoas/go-funk v0.9.2 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
+	github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa // indirect
+	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
+	github.com/whyrusleeping/cbor-gen v0.0.0-20221220214510-0333c149dec0 // indirect
+	github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
+	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
+	github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
+	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/otel v1.7.0 // indirect
+	go.opentelemetry.io/otel/trace v1.7.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/dig v1.15.0 // indirect
+	go.uber.org/fx v1.18.2 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
+	golang.org/x/crypto v0.6.0 // indirect
+	golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
+	golang.org/x/mod v0.9.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
+	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	lukechampine.com/blake3 v1.1.7 // indirect
+)
+
+replace github.com/ethereum/go-ethereum v1.11.6 => github.com/cerc-io/go-ethereum v1.11.6-statediff-5.0.8

integration/deploy_helpers.go

@@ -0,0 +1,82 @@
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/big"
+	"net/http"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff/types"
+)
+
+type ContractDeployed struct {
+	Address         common.Address `json:"address"`
+	TransactionHash common.Hash    `json:"txHash"`
+	BlockNumber     int64          `json:"blockNumber"`
+	BlockHash       common.Hash    `json:"blockHash"`
+}
+
+type ContractDestroyed struct {
+	BlockNumber int64 `json:"blockNumber"`
+}
+
+type Tx struct {
+	From            string   `json:"from"`
+	To              string   `json:"to"`
+	Value           *big.Int `json:"value"`
+	TransactionHash string   `json:"txHash"`
+	BlockNumber     int64    `json:"blockNumber"`
+	BlockHash       string   `json:"blockHash"`
+}
+
+type StorageKey struct {
+	Key string `json:"key"`
+}
+
+type CountIncremented struct {
+	BlockNumber *big.Int `json:"blockNumber"`
+}
+
+const ContractServerUrl = "http://localhost:3000"
+
+// Factory which creates endpoint functions
+func MakeGetAndDecodeFunc[R any](format string) func(...interface{}) (*R, error) {
+	return func(params ...interface{}) (*R, error) {
+		params = append([]interface{}{ContractServerUrl}, params...)
+		url := fmt.Sprintf(format, params...)
+		res, err := http.Get(url)
+		if err != nil {
+			return nil, err
+		}
+		defer res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("%s: %s", url, res.Status)
+		}
+
+		var data R
+		decoder := json.NewDecoder(res.Body)
+		return &data, decoder.Decode(&data)
+	}
+}
+
+var (
+	DeployContract     = MakeGetAndDecodeFunc[ContractDeployed]("%s/v1/deployContract")
+	DestroyContract    = MakeGetAndDecodeFunc[ContractDestroyed]("%s/v1/destroyContract?addr=%s")
+	DeploySLVContract  = MakeGetAndDecodeFunc[ContractDeployed]("%s/v1/deploySLVContract")
+	DestroySLVContract = MakeGetAndDecodeFunc[ContractDestroyed]("%s/v1/destroySLVContract?addr=%s")
+	SendEth            = MakeGetAndDecodeFunc[Tx]("%s/v1/sendEth?to=%s&value=%s")
+	GetStorageSlotKey  = MakeGetAndDecodeFunc[StorageKey]("%s/v1/getStorageKey?contract=%s&label=%s")
+	IncrementCount     = MakeGetAndDecodeFunc[CountIncremented]("%s/v1/incrementCount%s?addr=%s")
+	Create2Contract    = MakeGetAndDecodeFunc[ContractDeployed]("%s/v1/create2Contract?contract=%s&salt=%s")
+)
+
+func ClearWatchedAddresses(gethRPCClient *rpc.Client) error {
+	gethMethod := "statediff_watchAddress"
+	args := []types.WatchAddressArg{}
+
+	// Clear watched addresses
+	return gethRPCClient.Call(nil, gethMethod, types.Clear, args)
+}

integration/direct_proxy_integration_test.go

@@ -0,0 +1,378 @@
+package integration_test
+
+import (
+	"context"
+	"math/big"
+	"math/rand"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/cerc-io/ipld-eth-server/v5/integration"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+)
+
+var _ = Describe("Direct proxy integration test", Label("proxy"), func() {
+	ctx := context.Background()
+
+	var contract *integration.ContractDeployed
+	var tx *integration.Tx
+	var contractErr error
+	var txErr error
+
+	Describe("get Block", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("get not existing block by number", func() {
+			blockNum := contract.BlockNumber + 100
+
+			gethBlock, err := gethClient.BlockByNumber(ctx, big.NewInt(blockNum))
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(gethBlock).To(BeZero())
+
+			ipldBlock, err := ipldClient.BlockByNumber(ctx, big.NewInt(blockNum))
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(ipldBlock).To(BeZero())
+		})
+
+		It("get not existing block by hash", func() {
+			gethBlock, err := gethClient.BlockByHash(ctx, nonExistingBlockHash)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(gethBlock).To(BeZero())
+
+			ipldBlock, err := ipldClient.BlockByHash(ctx, nonExistingBlockHash)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(ipldBlock).To(BeZero())
+		})
+
+		It("get block by number", func() {
+			blockNum := contract.BlockNumber
+
+			_, err := gethClient.BlockByNumber(ctx, big.NewInt(blockNum))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BlockByNumber(ctx, big.NewInt(blockNum))
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("get block by hash", func() {
+			_, err := gethClient.BlockByHash(ctx, contract.BlockHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BlockByHash(ctx, contract.BlockHash)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("Transaction", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("Get tx by hash", func() {
+			_, _, err := gethClient.TransactionByHash(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, _, err = ipldClient.TransactionByHash(ctx, contract.TransactionHash)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("not found"))
+		})
+
+		It("Get tx by block hash and index", func() {
+			_, err := gethClient.TransactionInBlock(ctx, contract.BlockHash, 0)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.TransactionInBlock(ctx, contract.BlockHash, 0)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("not found"))
+		})
+	})
+
+	Describe("Receipt", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("Get tx receipt", func() {
+			_, err := gethClient.TransactionReceipt(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.TransactionReceipt(ctx, contract.TransactionHash)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("not found"))
+		})
+	})
+
+	Describe("FilterLogs", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("with blockhash", func() {
+			blockHash := contract.BlockHash
+			filterQuery := ethereum.FilterQuery{
+				//Addresses: addresses,
+				BlockHash: &blockHash,
+				Topics:    [][]common.Hash{},
+			}
+
+			gethLogs, err := gethClient.FilterLogs(ctx, filterQuery)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldLogs, err := ipldClient.FilterLogs(ctx, filterQuery)
+			Expect(err).ToNot(HaveOccurred())
+
+			// not empty list
+			Expect(gethLogs).ToNot(BeEmpty())
+			// empty list
+			Expect(ipldLogs).To(BeEmpty())
+		})
+	})
+
+	Describe("CodeAt", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("gets code of deployed contract with block number", func() {
+			_, err := gethClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldCode).To(BeEmpty())
+		})
+		It("gets code of contract that doesn't exist at this height", func() {
+			gethCode, err := gethClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethCode).To(BeEmpty())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+
+		It("gets code at non-existing address without block number", func() {
+			gethCode, err := gethClient.CodeAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethCode).To(BeEmpty())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+		It("gets code of deployed contract without block number", func() {
+			_, err := gethClient.CodeAt(ctx, contract.Address, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldCode).To(BeEmpty())
+		})
+	})
+
+	Describe("Get balance", func() {
+		var newAddress common.Address
+		rand.Read(newAddress[:])
+
+		BeforeEach(func() {
+			tx, txErr = integration.SendEth(newAddress, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+		})
+
+		It("gets balance for an account with eth without block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethBalance.String()).To(Equal("10000000000000000"))
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, newAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldBalance.String()).To(Equal("0"))
+		})
+		It("gets balance for an account with eth with block number", func() {
+			_, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber))
+			Expect(err).To(MatchError("header not found"))
+		})
+		It("gets historical balance for an account with eth with block number", func() {
+			_, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber-1))
+			Expect(err).To(MatchError("header not found"))
+		})
+
+		It("gets balance for a non-existing account without block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+		It("gets balance for an non-existing block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber+3))
+			Expect(err).To(MatchError("header not found"))
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, nonExistingAddress, big.NewInt(tx.BlockNumber+3))
+			Expect(err).To(MatchError("header not found"))
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+	})
+
+	Describe("Get Storage", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("gets ERC20 total supply (without block number)", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethStorage)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldStorage).To(Equal(make([]byte, 32)))
+		})
+
+		It("gets ERC20 total supply (with block number)", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethStorage)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			_, err = ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).To(MatchError("header not found"))
+		})
+
+		It("gets storage for non-existing account", func() {
+			_, err := gethClient.StorageAt(ctx, nonExistingAddress, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.StorageAt(ctx, nonExistingAddress, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).To(MatchError("header not found"))
+		})
+
+		It("gets storage for non-existing contract slot", func() {
+			_, err := gethClient.StorageAt(ctx, contract.Address, randomHash, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.StorageAt(ctx, contract.Address, randomHash, big.NewInt(contract.BlockNumber))
+			Expect(err).To(MatchError("header not found"))
+		})
+
+		It("gets storage for non-existing contract", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(0))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for non-existing block number", func() {
+			blockNum := contract.BlockNumber + 100
+
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(blockNum))
+			Expect(err).To(MatchError("header not found"))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(blockNum))
+			Expect(err).To(MatchError("header not found"))
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("get storage after self destruct", func() {
+			tx, err := integration.DestroyContract(contract.Address)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethStorage1, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+			gethStorage2, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethStorage1).NotTo(Equal(gethStorage2))
+			Expect(gethStorage2).To(Equal(eth.EmptyNodeValue))
+
+			_, err = ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber-1))
+			Expect(err).To(MatchError("header not found"))
+
+			_, err = ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber))
+			Expect(err).To(MatchError("header not found"))
+
+			// Query the current block
+			ipldStorage3, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(eth.EmptyNodeValue).To(Equal(ipldStorage3))
+		})
+	})
+
+	Describe("eth call", func() {
+		var msg ethereum.CallMsg
+
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			msg = ethereum.CallMsg{
+				To:   &contract.Address,
+				Data: common.Hex2Bytes("18160ddd"), // totalSupply()
+			}
+		})
+
+		It("calls totalSupply() without block number", func() {
+			gethResult, err := gethClient.CallContract(ctx, msg, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContract(ctx, msg, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+
+		It("calls totalSupply() with block number", func() {
+			gethResult, err := gethClient.CallContract(ctx, msg, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContract(ctx, msg, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+	})
+
+	Describe("Chain ID", func() {
+		It("Check chain id", func() {
+			_, err := gethClient.ChainID(ctx)
+			Expect(err).ToNot(HaveOccurred())
+
+			_, err = ipldClient.ChainID(ctx)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+})

integration/integration_suite_test.go

@@ -0,0 +1,53 @@
+package integration_test
+
+import (
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestIntegration(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "integration test suite")
+}
+
+var (
+	gethHttpPath    = "http://127.0.0.1:8545"
+	ipldEthHttpPath = "http://127.0.0.1:8081"
+
+	gethClient    *ethclient.Client
+	ipldClient    *ethclient.Client
+	gethRPCClient *rpc.Client
+	ipldRPCClient *rpc.Client
+
+	testChainId int64 = 99
+)
+
+var _ = BeforeSuite(func() {
+	var err error
+
+	envChainID := os.Getenv("ETH_CHAIN_ID")
+	if len(envChainID) == 0 {
+		panic("ETH_CHAIN_ID must be set")
+	}
+	testChainId, err = strconv.ParseInt(envChainID, 10, 64)
+	Expect(err).ToNot(HaveOccurred())
+
+	if path := os.Getenv("ETH_HTTP_PATH"); len(path) != 0 {
+		gethHttpPath = "http://" + path
+	}
+	if path := os.Getenv("SERVER_HTTP_PATH"); len(path) != 0 {
+		ipldEthHttpPath = "http://" + path
+	}
+
+	gethClient, err = ethclient.Dial(gethHttpPath)
+	Expect(err).ToNot(HaveOccurred())
+
+	ipldClient, err = ethclient.Dial(ipldEthHttpPath)
+	Expect(err).ToNot(HaveOccurred())
+})

integration/integration_test.go

@@ -0,0 +1,547 @@
+package integration_test
+
+import (
+	"context"
+	"math/big"
+	"math/rand"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	integration "github.com/cerc-io/ipld-eth-server/v5/integration"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+)
+
+var (
+	nonExistingBlockHash = common.HexToHash("0x111111111111111111111111111111111111111111111111111111111111111")
+	nonExistingAddress   = common.HexToAddress("0x1111111111111111111111111111111111111111")
+	randomAddr           = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
+	randomHash           = crypto.Keccak256Hash(randomAddr.Bytes())
+
+	erc20TotalSupply, _ = new(big.Int).SetString("1000000000000000000000", 10)
+	ercTotalSupplyIndex = common.HexToHash("0x2")
+)
+
+var _ = Describe("Basic integration test", func() {
+	ctx := context.Background()
+
+	var contract *integration.ContractDeployed
+	var tx *integration.Tx
+	var contractErr error
+	var txErr error
+
+	Describe("get Block", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			err := waitForBlock(ctx, ipldClient, contract.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("get not existing block by number", func() {
+			blockNum := big.NewInt(contract.BlockNumber + 100)
+
+			gethBlock, err := gethClient.BlockByNumber(ctx, blockNum)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(gethBlock).To(BeZero())
+
+			ipldBlock, err := ipldClient.BlockByNumber(ctx, blockNum)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(ipldBlock).To(BeZero())
+		})
+
+		It("get not existing block by hash", func() {
+			gethBlock, err := gethClient.BlockByHash(ctx, nonExistingBlockHash)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(gethBlock).To(BeZero())
+
+			ipldBlock, err := ipldClient.BlockByHash(ctx, nonExistingBlockHash)
+			Expect(err).To(MatchError(ethereum.NotFound))
+			Expect(ipldBlock).To(BeZero())
+		})
+
+		It("get block by number", func() {
+			blockNum := big.NewInt(contract.BlockNumber)
+
+			gethBlock, err := gethClient.BlockByNumber(ctx, blockNum)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBlock, err := ipldClient.BlockByNumber(ctx, blockNum)
+			Expect(err).ToNot(HaveOccurred())
+
+			// check headers are equals
+			Expect(gethBlock.Header()).To(Equal(ipldBlock.Header()))
+
+			gethTxs := gethBlock.Transactions()
+			ipldTxs := ipldBlock.Transactions()
+
+			Expect(gethTxs.Len()).To(Equal(ipldTxs.Len()))
+			Expect(types.TxDifference(gethTxs, ipldTxs).Len()).To(Equal(0))
+		})
+
+		It("get block by hash", func() {
+			gethBlock, err := gethClient.BlockByHash(ctx, contract.BlockHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBlock, err := ipldClient.BlockByHash(ctx, contract.BlockHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			// check headers are equals
+			compareBlocks(gethBlock, ipldBlock)
+
+			gethTxs := gethBlock.Transactions()
+			ipldTxs := ipldBlock.Transactions()
+
+			Expect(gethTxs.Len()).To(Equal(ipldTxs.Len()))
+			Expect(types.TxDifference(gethTxs, ipldTxs).Len()).To(Equal(0))
+		})
+	})
+
+	Describe("Transaction", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("Get tx by hash", func() {
+			gethTx, _, err := gethClient.TransactionByHash(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldTx, _, err := ipldClient.TransactionByHash(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			compareTxs(gethTx, ipldTx)
+
+			Expect(gethTx.Hash()).To(Equal(ipldTx.Hash()))
+		})
+
+		It("Get tx by block hash and index", func() {
+			gethTx, err := gethClient.TransactionInBlock(ctx, contract.BlockHash, 0)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldTx, err := ipldClient.TransactionInBlock(ctx, contract.BlockHash, 0)
+			Expect(err).ToNot(HaveOccurred())
+
+			compareTxs(gethTx, ipldTx)
+		})
+	})
+
+	Describe("Receipt", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("Get tx receipt", func() {
+			gethReceipt, err := gethClient.TransactionReceipt(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldReceipt, err := ipldClient.TransactionReceipt(ctx, contract.TransactionHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethReceipt).To(Equal(ipldReceipt))
+
+			rlpGeth, err := gethReceipt.MarshalBinary()
+			Expect(err).ToNot(HaveOccurred())
+
+			rlpIpld, err := ipldReceipt.MarshalBinary()
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(rlpGeth).To(Equal(rlpIpld))
+		})
+	})
+
+	Describe("FilterLogs", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("with blockhash", func() {
+			blockHash := contract.BlockHash
+			filterQuery := ethereum.FilterQuery{
+				//Addresses: addresses,
+				BlockHash: &blockHash,
+				Topics:    [][]common.Hash{},
+			}
+
+			gethLogs, err := gethClient.FilterLogs(ctx, filterQuery)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldLogs, err := ipldClient.FilterLogs(ctx, filterQuery)
+			Expect(err).ToNot(HaveOccurred())
+
+			// not empty list
+			Expect(gethLogs).ToNot(BeEmpty())
+
+			Expect(len(gethLogs)).To(Equal(len(ipldLogs)))
+			Expect(gethLogs).To(Equal(ipldLogs))
+		})
+	})
+
+	Describe("CodeAt", func() {
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+		})
+
+		It("gets code at non-existing address without block number", func() {
+			gethCode, err := gethClient.CodeAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethCode).To(BeEmpty())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+		It("gets code of deployed contract without block number", func() {
+			gethCode, err := gethClient.CodeAt(ctx, contract.Address, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+		It("gets code of deployed contract with block number", func() {
+			gethCode, err := gethClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+		It("gets code of contract that doesn't exist at this height", func() {
+			gethCode, err := gethClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCode, err := ipldClient.CodeAt(ctx, contract.Address, big.NewInt(contract.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethCode).To(BeEmpty())
+			Expect(gethCode).To(Equal(ipldCode))
+		})
+	})
+
+	Describe("Get balance", func() {
+		var newAddress common.Address
+		rand.Read(newAddress[:])
+
+		BeforeEach(func() {
+			tx, txErr = integration.SendEth(newAddress, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+		})
+
+		It("gets balance for an account with eth with block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+		It("gets balance for an account with eth without block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, newAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+
+		It("gets historical balance for an account with eth with block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+		It("gets balance for a non-existing account without block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, nonExistingAddress, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+		It("gets balance for an non-existing block number", func() {
+			gethBalance, err := gethClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber+3))
+			Expect(err).To(MatchError("header not found"))
+
+			ipldBalance, err := ipldClient.BalanceAt(ctx, newAddress, big.NewInt(tx.BlockNumber+3))
+			Expect(err).To(MatchError("header not found"))
+
+			Expect(gethBalance).To(Equal(ipldBalance))
+		})
+	})
+
+	Describe("Get Storage", func() {
+		var contractSalt string
+		countAIndex := common.HexToHash("0x5")
+
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+			Expect(contract.BlockNumber).ToNot(BeZero())
+
+			contractSalt = common.Bytes2Hex(contract.BlockHash[:10])
+
+			err := waitForBlock(ctx, ipldClient, contract.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("gets ERC20 total supply (with block number)", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethStorage)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets ERC20 total supply (without block number)", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethStorage)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for non-existing account", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, nonExistingAddress, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, nonExistingAddress, ercTotalSupplyIndex, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for non-existing contract slot", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, randomHash, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, randomHash, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for non-existing contract", func() {
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(0))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for non-existing block number", func() {
+			blockNum := contract.BlockNumber + 100
+			gethStorage, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(blockNum))
+			Expect(err).To(MatchError("header not found"))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(blockNum))
+			Expect(err).To(MatchError("header not found"))
+			Expect(gethStorage).To(Equal(ipldStorage))
+		})
+
+		It("gets storage for SLV countA after tx", func() {
+			slvContract, contractErr := integration.Create2Contract("SLVToken", contractSalt)
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			gethStorage, err := gethClient.StorageAt(ctx, slvContract.Address, countAIndex, big.NewInt(slvContract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			slvCountA := new(big.Int).SetBytes(gethStorage)
+
+			err = waitForBlock(ctx, ipldClient, slvContract.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, slvContract.Address, countAIndex, big.NewInt(slvContract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCountA := new(big.Int).SetBytes(ipldStorage)
+			Expect(ipldCountA).To(Equal(slvCountA))
+
+			inc, err := integration.IncrementCount("A", slvContract.Address)
+			Expect(err).ToNot(HaveOccurred())
+			slvCountA.Add(slvCountA, big.NewInt(1))
+
+			ipldStorage, err = ipldClient.StorageAt(ctx, slvContract.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldCountA = new(big.Int).SetBytes(ipldStorage)
+			Expect(ipldCountA).To(Equal(slvCountA))
+		})
+
+		It("gets storage after destruction and redeploy", func() {
+			slvContract, contractErr := integration.Create2Contract("SLVToken", contractSalt)
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			tx, err := integration.DestroyContract(contract.Address)
+			Expect(err).ToNot(HaveOccurred())
+
+			slvTx, err := integration.DestroyContract(slvContract.Address)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethStorage1, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+			gethStorage2, err := gethClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethStorage1).NotTo(Equal(gethStorage2))
+			Expect(gethStorage2).To(Equal(eth.EmptyNodeValue))
+
+			ipldStorage1, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber-1))
+			Expect(err).ToNot(HaveOccurred())
+			ipldStorage2, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(ipldStorage1).To(Equal(gethStorage1))
+			Expect(ipldStorage2).To(Equal(gethStorage2))
+
+			// Query the current block
+			ipldStorage3, err := ipldClient.StorageAt(ctx, contract.Address, ercTotalSupplyIndex, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(ipldStorage2).To(Equal(ipldStorage3))
+
+			// Check for SLV contract
+			gethStorage, err := gethClient.StorageAt(ctx, slvContract.Address, countAIndex, big.NewInt(slvTx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gethStorage).To(Equal(eth.EmptyNodeValue))
+
+			ipldStorage, err := ipldClient.StorageAt(ctx, slvContract.Address, countAIndex, big.NewInt(slvTx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(ipldStorage).To(Equal(gethStorage))
+
+			// Redeploy to same address
+			slvContract, contractErr = integration.Create2Contract("SLVToken", contractSalt)
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			gethStorage, err = gethClient.StorageAt(ctx, slvContract.Address, countAIndex, big.NewInt(slvContract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldStorage, err = ipldClient.StorageAt(ctx, slvContract.Address, countAIndex, big.NewInt(slvContract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethStorage).To(Equal(ipldStorage))
+			ipldCountA := new(big.Int).SetBytes(ipldStorage)
+			Expect(ipldCountA.String()).To(Equal("0"))
+
+		})
+	})
+
+	Describe("eth call", func() {
+		var msg ethereum.CallMsg
+
+		BeforeEach(func() {
+			contract, contractErr = integration.DeployContract()
+			Expect(contractErr).ToNot(HaveOccurred())
+
+			msg = ethereum.CallMsg{
+				To:   &contract.Address,
+				Data: common.Hex2Bytes("18160ddd"), // totalSupply()
+			}
+		})
+
+		It("calls totalSupply() without block number", func() {
+			gethResult, err := gethClient.CallContract(ctx, msg, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContract(ctx, msg, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+
+		It("calls totalSupply() with block number", func() {
+			gethResult, err := gethClient.CallContract(ctx, msg, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContract(ctx, msg, big.NewInt(contract.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+
+		It("calls totalSupply() with block hash", func() {
+			gethResult, err := gethClient.CallContractAtHash(ctx, msg, contract.BlockHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			gethTotalSupply := new(big.Int).SetBytes(gethResult)
+			Expect(gethTotalSupply).To(Equal(erc20TotalSupply))
+
+			ipldResult, err := ipldClient.CallContractAtHash(ctx, msg, contract.BlockHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethResult).To(Equal(ipldResult))
+		})
+	})
+
+	Describe("Chain ID", func() {
+		It("Check chain id", func() {
+			gethChainId, err := gethClient.ChainID(ctx)
+			Expect(err).ToNot(HaveOccurred())
+
+			ipldChainId, err := ipldClient.ChainID(ctx)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(gethChainId).To(Equal(ipldChainId))
+		})
+	})
+})
+
+func compareBlocks(block1 *types.Block, block2 *types.Block) {
+	GinkgoHelper()
+	Expect(block1.Header()).To(Equal(block2.Header()))
+	Expect(block1.Uncles()).To(Equal(block2.Uncles()))
+
+	txs1 := block1.Transactions()
+	txs2 := block2.Transactions()
+
+	Expect(len(txs1)).To(Equal(len(txs2)))
+	for i, tx := range txs1 {
+		compareTxs(tx, txs2[i])
+	}
+}
+
+func compareTxs(tx1 *types.Transaction, tx2 *types.Transaction) {
+	GinkgoHelper()
+	Expect(tx1.Data()).To(Equal(tx2.Data()))
+	Expect(tx1.Hash()).To(Equal(tx2.Hash()))
+	Expect(tx1.Size()).To(Equal(tx2.Size()))
+
+	signer := types.NewLondonSigner(big.NewInt(testChainId))
+
+	gethSender, err := types.Sender(signer, tx1)
+	Expect(err).ToNot(HaveOccurred())
+
+	ipldSender, err := types.Sender(signer, tx2)
+	Expect(err).ToNot(HaveOccurred())
+
+	Expect(gethSender).To(Equal(ipldSender))
+}

integration/util_test.go

@@ -0,0 +1,29 @@
+package integration_test
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+)
+
+func waitForBlock(ctx context.Context, client *ethclient.Client, target int64) error {
+	timeout := 10 * time.Second
+	for {
+		select {
+		case <-time.After(timeout):
+			return errors.New("timed out")
+		default:
+			latest, err := client.BlockNumber(ctx)
+			if err != nil {
+				return err
+			}
+			if uint64(target) <= latest {
+				return nil
+			}
+			time.Sleep(time.Second)
+		}
+	}
+
+}

integration/watch_address_integration_test.go

@@ -0,0 +1,627 @@
+package integration_test
+
+import (
+	"context"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rpc"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	integration "github.com/cerc-io/ipld-eth-server/v5/integration"
+)
+
+var (
+	gethMethod = "statediff_watchAddress"
+	ipldMethod = "vdb_watchAddress"
+)
+
+var _ = Describe("WatchAddress integration test", func() {
+	var (
+		ctx = context.Background()
+
+		txErr       error
+		contractErr error
+		incErr      error
+
+		tx  *integration.Tx
+		inc *integration.CountIncremented
+
+		contract1 *integration.ContractDeployed
+		contract2 *integration.ContractDeployed
+		contract3 *integration.ContractDeployed
+
+		countAIndex common.Hash
+
+		prevBalance1 *big.Int
+		prevBalance2 *big.Int
+		prevBalance3 *big.Int
+
+		actualBalance1 *big.Int
+		actualBalance2 *big.Int
+		actualBalance3 *big.Int
+
+		prevCountA1 *big.Int
+		prevCountA2 *big.Int
+		prevCountA3 *big.Int
+
+		actualCountA1 *big.Int
+		actualCountA2 *big.Int
+		actualCountA3 *big.Int
+	)
+
+	BeforeEach(func() {
+		var err error
+
+		gethRPCClient, err = rpc.Dial(gethHttpPath)
+		Expect(err).ToNot(HaveOccurred())
+
+		ipldRPCClient, err = rpc.Dial(ipldEthHttpPath)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	It("test init", func() {
+		// Deploy three contracts
+		contract1, contractErr = integration.DeploySLVContract()
+		Expect(contractErr).ToNot(HaveOccurred())
+
+		contract2, contractErr = integration.DeploySLVContract()
+		Expect(contractErr).ToNot(HaveOccurred())
+
+		contract3, contractErr = integration.DeploySLVContract()
+		Expect(contractErr).ToNot(HaveOccurred())
+
+		// Get the storage slot key
+		storageSlotAKey, err := integration.GetStorageSlotKey("SLVToken", "countA")
+		Expect(err).ToNot(HaveOccurred())
+		countAIndex = common.HexToHash(storageSlotAKey.Key)
+
+		// Clear out watched addresses
+		err = integration.ClearWatchedAddresses(gethRPCClient)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Get initial balances for all the contracts
+		// Contract 1
+		actualBalance1 = big.NewInt(0)
+		initBalance1, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(contract1.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(initBalance1.String()).To(Equal(actualBalance1.String()))
+		prevBalance1 = big.NewInt(0)
+
+		// Contract 2
+		actualBalance2 = big.NewInt(0)
+		initBalance2, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(contract2.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(initBalance2.String()).To(Equal(actualBalance2.String()))
+		prevBalance2 = big.NewInt(0)
+
+		// Contract 3
+		actualBalance3 = big.NewInt(0)
+		initBalance3, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(contract3.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		Expect(initBalance3.String()).To(Equal(actualBalance3.String()))
+		prevBalance3 = big.NewInt(0)
+
+		// Get initial storage values for the contracts
+		// Contract 1, countA
+		actualCountA1 = big.NewInt(0)
+		ipldCountA1Storage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, big.NewInt(contract1.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		ipldCountA1 := new(big.Int).SetBytes(ipldCountA1Storage)
+		Expect(ipldCountA1.String()).To(Equal(actualCountA1.String()))
+		prevCountA1 = big.NewInt(0)
+
+		// Contract 2, countA
+		actualCountA2 = big.NewInt(0)
+		ipldCountA2Storage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, big.NewInt(contract2.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		ipldCountA2 := new(big.Int).SetBytes(ipldCountA2Storage)
+		Expect(ipldCountA2.String()).To(Equal("0"))
+		prevCountA2 = big.NewInt(0)
+
+		// Contract 3, countA
+		actualCountA3 = big.NewInt(0)
+		ipldCountA3Storage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, big.NewInt(contract3.BlockNumber))
+		Expect(err).ToNot(HaveOccurred())
+		ipldCountA3 := new(big.Int).SetBytes(ipldCountA3Storage)
+		Expect(ipldCountA3.String()).To(Equal("0"))
+		prevCountA3 = big.NewInt(0)
+	})
+
+	defer It("test cleanup", func() {
+		// Clear out watched addresses
+		err := integration.ClearWatchedAddresses(gethRPCClient)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Context("no contracts being watched", func() {
+		It("indexes state for all the contracts", func() {
+			// WatchedAddresses = []
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(actualBalance1.String()))
+			prevBalance1.Set(actualBalance1)
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(actualBalance2.String()))
+			prevBalance2.Set(actualBalance2)
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(actualBalance3.String()))
+			prevBalance3.Set(actualBalance3)
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(actualCountA1.String()))
+			prevCountA1.Set(actualCountA1)
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(actualCountA2.String()))
+			prevCountA2.Set(actualCountA2)
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(actualCountA3.String()))
+			prevCountA3.Set(actualCountA3)
+		})
+	})
+
+	Context("one contract being watched", func() {
+		It("indexes state only for the watched contract", func() {
+			operation := sdtypes.Add
+			args := []sdtypes.WatchAddressArg{
+				{
+					Address:   contract1.Address.String(),
+					CreatedAt: uint64(contract1.BlockNumber),
+				},
+			}
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).ToNot(HaveOccurred())
+
+			// WatchedAddresses = [Contract1]
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(actualBalance1.String()))
+			prevBalance1.Set(actualBalance1)
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(prevBalance2.String()))
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(prevBalance3.String()))
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(actualCountA1.String()))
+			prevCountA1.Set(actualCountA1)
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(prevCountA2.String()))
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(prevCountA3.String()))
+		})
+	})
+
+	Context("contract added to a non-empty watch-list", func() {
+		It("indexes state only for the watched contracts", func() {
+			operation := sdtypes.Add
+			args := []sdtypes.WatchAddressArg{
+				{
+					Address:   contract2.Address.String(),
+					CreatedAt: uint64(contract2.BlockNumber),
+				},
+			}
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).ToNot(HaveOccurred())
+
+			// WatchedAddresses = [Contract1, Contract2]
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(actualBalance1.String()))
+			prevBalance1.Set(actualBalance1)
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(actualBalance2.String()))
+			prevBalance2.Set(actualBalance2)
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(prevBalance3.String()))
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(actualCountA1.String()))
+			prevCountA1.Set(actualCountA1)
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(actualCountA2.String()))
+			prevCountA2.Set(actualCountA2)
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(prevCountA3.String()))
+		})
+	})
+
+	Context("contract removed from the watch-list", func() {
+		It("indexes state only for the watched contract", func() {
+			operation := sdtypes.Remove
+			args := []sdtypes.WatchAddressArg{
+				{
+					Address:   contract1.Address.String(),
+					CreatedAt: uint64(contract1.BlockNumber),
+				},
+			}
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).ToNot(HaveOccurred())
+
+			// WatchedAddresses = [Contract2]
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(prevBalance1.String()))
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(actualBalance2.String()))
+			prevBalance2.Set(actualBalance2)
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(prevBalance3.String()))
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(prevCountA1.String()))
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(actualCountA2.String()))
+			prevCountA2.Set(actualCountA2)
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(prevCountA3.String()))
+		})
+	})
+
+	Context("list of watched addresses set", func() {
+		It("indexes state only for the watched contracts", func() {
+			operation := sdtypes.Set
+			args := []sdtypes.WatchAddressArg{
+				{
+					Address:   contract1.Address.String(),
+					CreatedAt: uint64(contract1.BlockNumber),
+				},
+				{
+					Address:   contract3.Address.String(),
+					CreatedAt: uint64(contract3.BlockNumber),
+				},
+			}
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).ToNot(HaveOccurred())
+
+			// WatchedAddresses = [Contract1, Contract3]
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(actualBalance1.String()))
+			prevBalance1.Set(actualBalance1)
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(prevBalance2.String()))
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(actualBalance3.String()))
+			prevBalance3.Set(actualBalance3)
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(actualCountA1.String()))
+			prevCountA1.Set(actualCountA1)
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(prevCountA2.String()))
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(actualCountA3.String()))
+			prevCountA3.Set(actualCountA3)
+		})
+	})
+
+	Context("list of watched addresses cleared", func() {
+		It("indexes state for all the contracts", func() {
+			operation := sdtypes.Clear
+			args := []sdtypes.WatchAddressArg{}
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).ToNot(HaveOccurred())
+
+			// WatchedAddresses = []
+			// Send eth to all three contract accounts
+			// Contract 1
+			tx, txErr = integration.SendEth(contract1.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance1.Add(actualBalance1, big.NewInt(10000000000000000))
+
+			balance1AfterTransfer, err := ipldClient.BalanceAt(ctx, contract1.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance1AfterTransfer.String()).To(Equal(actualBalance1.String()))
+			prevBalance1.Set(actualBalance1)
+
+			// Contract 2
+			tx, txErr = integration.SendEth(contract2.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance2.Add(actualBalance2, big.NewInt(10000000000000000))
+
+			balance2AfterTransfer, err := ipldClient.BalanceAt(ctx, contract2.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance2AfterTransfer.String()).To(Equal(actualBalance2.String()))
+			prevBalance2.Set(actualBalance2)
+
+			// Contract 3
+			tx, txErr = integration.SendEth(contract3.Address, "0.01")
+			Expect(txErr).ToNot(HaveOccurred())
+			actualBalance3.Add(actualBalance3, big.NewInt(10000000000000000))
+
+			balance3AfterTransfer, err := ipldClient.BalanceAt(ctx, contract3.Address, big.NewInt(tx.BlockNumber))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(balance3AfterTransfer.String()).To(Equal(actualBalance3.String()))
+			prevBalance3.Set(actualBalance3)
+
+			// Increment counts
+			// Contract 1, countA
+			inc, incErr = integration.IncrementCount("A", contract1.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA1.Add(actualCountA1, big.NewInt(1))
+
+			countA1AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract1.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA1AfterIncrement := new(big.Int).SetBytes(countA1AfterIncrementStorage)
+			Expect(countA1AfterIncrement.String()).To(Equal(actualCountA1.String()))
+			prevCountA1.Set(actualCountA1)
+
+			// Contract 2, countA
+			inc, incErr = integration.IncrementCount("A", contract2.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA2.Add(actualCountA2, big.NewInt(1))
+
+			countA2AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract2.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA2AfterIncrement := new(big.Int).SetBytes(countA2AfterIncrementStorage)
+			Expect(countA2AfterIncrement.String()).To(Equal(actualCountA2.String()))
+			prevCountA2.Set(actualCountA2)
+
+			// Contract 3, countA
+			inc, incErr = integration.IncrementCount("A", contract3.Address)
+			Expect(incErr).ToNot(HaveOccurred())
+			actualCountA3.Add(actualCountA3, big.NewInt(1))
+
+			countA3AfterIncrementStorage, err := ipldClient.StorageAt(ctx, contract3.Address, countAIndex, inc.BlockNumber)
+			Expect(err).ToNot(HaveOccurred())
+			countA3AfterIncrement := new(big.Int).SetBytes(countA3AfterIncrementStorage)
+			Expect(countA3AfterIncrement.String()).To(Equal(actualCountA3.String()))
+			prevCountA3.Set(actualCountA3)
+		})
+	})
+
+	Context("with invalid args", func() {
+		It("returns an error on invalid operation arg", func() {
+			operation := "WrongOp"
+			args := []sdtypes.WatchAddressArg{}
+
+			gethErr := gethRPCClient.Call(nil, gethMethod, operation, args)
+			Expect(gethErr).To(HaveOccurred())
+			Expect(gethErr.Error()).To(ContainSubstring("unexpected operation"))
+
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).To(HaveOccurred())
+			Expect(ipldErr.Error()).To(ContainSubstring("unexpected operation"))
+
+			Expect(ipldErr).To(Equal(gethErr))
+		})
+
+		It("returns an error on args of invalid type", func() {
+			operation := sdtypes.Add
+			args := []string{"WrongArg"}
+
+			gethErr := gethRPCClient.Call(nil, gethMethod, operation, args)
+			Expect(gethErr).To(HaveOccurred())
+			Expect(gethErr.Error()).To(ContainSubstring("WatchAddressArg"))
+
+			ipldErr := ipldRPCClient.Call(nil, ipldMethod, operation, args)
+			Expect(ipldErr).To(HaveOccurred())
+			Expect(ipldErr.Error()).To(ContainSubstring("WatchAddressArg"))
+
+			Expect(ipldErr).To(Equal(gethErr))
+		})
+	})
+})

main.go

@@ -16,14 +16,9 @@
 package main
 
 import (
-	"github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/ipld-eth-server/cmd"
+	"github.com/cerc-io/ipld-eth-server/v5/cmd"
 )
 
 func main() {
-	logrus.SetFormatter(&logrus.TextFormatter{
-		FullTimestamp: true,
-	})
 	cmd.Execute()
 }

pkg/client/client.go

@@ -1,45 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-// Client is used by watchers to stream chain IPLD data from a vulcanizedb ipld-eth-server
-package client
-
-import (
-	"context"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-
-	"github.com/ethereum/go-ethereum/rpc"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/serve"
-)
-
-// Client is used to subscribe to the ipld-eth-server ipld data stream
-type Client struct {
-	c *rpc.Client
-}
-
-// NewClient creates a new Client
-func NewClient(c *rpc.Client) *Client {
-	return &Client{
-		c: c,
-	}
-}
-
-// Stream is the main loop for subscribing to iplds from an ipld-eth-server server
-func (c *Client) Stream(payloadChan chan serve.SubscriptionPayload, params eth.SubscriptionSettings) (*rpc.ClientSubscription, error) {
-	return c.c.Subscribe(context.Background(), "vdb", payloadChan, "stream", params)
-}

pkg/debug/backend.go

@@ -0,0 +1,51 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package debug
+
+import (
+	"context"
+	"errors"
+
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/eth/tracers"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+)
+
+var _ tracers.Backend = &Backend{}
+
+var (
+	errMethodNotSupported = errors.New("backend method not supported")
+)
+
+// Backend implements tracers.Backend interface
+type Backend struct {
+	eth.Backend
+}
+
+// StateAtBlock retrieves the state database associated with a certain block
+func (b *Backend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) {
+	return nil, func() {}, errMethodNotSupported
+}
+
+// StateAtTransaction returns the execution environment of a certain transaction
+func (b *Backend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (*core.Message, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) {
+	return nil, vm.BlockContext{}, nil, func() {}, errMethodNotSupported
+}

pkg/eth/api_test/api_test.go

@@ -14,38 +14,43 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package eth_test
+package eth_api_test
 
 import (
 	"context"
+	"math/big"
 	"strconv"
 
-	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/eth/filters"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
-	. "github.com/onsi/ginkgo"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-	shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
 )
 
 var (
-	randomAddr    = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
-	randomHash    = crypto.Keccak256Hash(randomAddr.Bytes())
-	number        = rpc.BlockNumber(test_helpers.BlockNumber.Int64())
-	wrongNumber   = rpc.BlockNumber(number + 1)
-	blockHash     = test_helpers.MockBlock.Header().Hash()
-	ctx           = context.Background()
+	randomAddr     = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
+	randomHash     = crypto.Keccak256Hash(randomAddr.Bytes())
+	number         = rpc.BlockNumber(test_helpers.BlockNumber.Int64())
+	londonBlockNum = rpc.BlockNumber(test_helpers.LondonBlockNum.Int64())
+	wrongNumber    = number + 1
+	blockHash      = test_helpers.MockBlock.Header().Hash()
+	baseFee        = test_helpers.MockLondonBlock.BaseFee()
+	ctx            = context.Background()
+
 	expectedBlock = map[string]interface{}{
 		"number":           (*hexutil.Big)(test_helpers.MockBlock.Number()),
 		"hash":             test_helpers.MockBlock.Hash(),
@@ -126,13 +131,14 @@ var (
 		"receiptsRoot":     test_helpers.MockUncles[1].ReceiptHash,
 		"uncles":           []common.Hash{},
 	}
-	expectedTransaction  = eth.NewRPCTransaction(test_helpers.MockTransactions[0], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 0)
-	expectedTransaction2 = eth.NewRPCTransaction(test_helpers.MockTransactions[1], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 1)
-	expectedTransaction3 = eth.NewRPCTransaction(test_helpers.MockTransactions[2], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 2)
-	expectRawTx, _       = rlp.EncodeToBytes(test_helpers.MockTransactions[0])
-	expectRawTx2, _      = rlp.EncodeToBytes(test_helpers.MockTransactions[1])
-	expectRawTx3, _      = rlp.EncodeToBytes(test_helpers.MockTransactions[2])
-	expectedReceipt      = map[string]interface{}{
+	expectedTransaction       = eth.NewRPCTransaction(test_helpers.MockTransactions[0], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 0, test_helpers.MockBlock.BaseFee())
+	expectedTransaction2      = eth.NewRPCTransaction(test_helpers.MockTransactions[1], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 1, test_helpers.MockBlock.BaseFee())
+	expectedTransaction3      = eth.NewRPCTransaction(test_helpers.MockTransactions[2], test_helpers.MockBlock.Hash(), test_helpers.MockBlock.NumberU64(), 2, test_helpers.MockBlock.BaseFee())
+	expectedLondonTransaction = eth.NewRPCTransaction(test_helpers.MockLondonTransactions[0], test_helpers.MockLondonBlock.Hash(), test_helpers.MockLondonBlock.NumberU64(), 0, test_helpers.MockLondonBlock.BaseFee())
+	expectRawTx, _            = test_helpers.MockTransactions[0].MarshalBinary()
+	expectRawTx2, _           = test_helpers.MockTransactions[1].MarshalBinary()
+	expectRawTx3, _           = test_helpers.MockTransactions[2].MarshalBinary()
+	expectedReceipt           = map[string]interface{}{
 		"blockHash":         blockHash,
 		"blockNumber":       hexutil.Uint64(uint64(number.Int64())),
 		"transactionHash":   expectedTransaction.Hash,
@@ -144,7 +150,9 @@ var (
 		"contractAddress":   nil,
 		"logs":              test_helpers.MockReceipts[0].Logs,
 		"logsBloom":         test_helpers.MockReceipts[0].Bloom,
-		"root":              hexutil.Bytes(test_helpers.MockReceipts[0].PostState),
+		"status":            hexutil.Uint(test_helpers.MockReceipts[0].Status),
+		"effectiveGasPrice": (*hexutil.Big)(big.NewInt(100)),
+		"type":              hexutil.Uint64(types.LegacyTxType),
 	}
 	expectedReceipt2 = map[string]interface{}{
 		"blockHash":         blockHash,
@@ -159,6 +167,8 @@ var (
 		"logs":              test_helpers.MockReceipts[1].Logs,
 		"logsBloom":         test_helpers.MockReceipts[1].Bloom,
 		"root":              hexutil.Bytes(test_helpers.MockReceipts[1].PostState),
+		"effectiveGasPrice": (*hexutil.Big)(big.NewInt(200)),
+		"type":              hexutil.Uint64(types.LegacyTxType),
 	}
 	expectedReceipt3 = map[string]interface{}{
 		"blockHash":         blockHash,
@@ -169,41 +179,83 @@ var (
 		"to":                expectedTransaction3.To,
 		"gasUsed":           hexutil.Uint64(test_helpers.MockReceipts[2].GasUsed),
 		"cumulativeGasUsed": hexutil.Uint64(test_helpers.MockReceipts[2].CumulativeGasUsed),
-		"contractAddress":   nil,
+		"contractAddress":   test_helpers.ContractAddress,
 		"logs":              test_helpers.MockReceipts[2].Logs,
 		"logsBloom":         test_helpers.MockReceipts[2].Bloom,
 		"root":              hexutil.Bytes(test_helpers.MockReceipts[2].PostState),
+		"effectiveGasPrice": (*hexutil.Big)(big.NewInt(150)),
+		"type":              hexutil.Uint64(types.LegacyTxType),
 	}
 )
+var (
+	db          *sqlx.DB
+	api         *eth.PublicEthAPI
+	chainConfig = params.TestChainConfig
+)
+
+var _ = BeforeSuite(func() {
+	var (
+		err error
+		tx  interfaces.Batch
+	)
+
+	db = shared.SetupDB()
+	indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
+
+	backend, err := eth.NewEthBackend(db, &eth.Config{
+		ChainConfig: chainConfig,
+		VMConfig:    vm.Config{},
+		RPCGasCap:   big.NewInt(10000000000), // Max gas capacity for a rpc call.
+		GroupCacheConfig: &shared.GroupCacheConfig{
+			StateDB: shared.GroupConfig{
+				Name:                   "api_test",
+				CacheSizeInMB:          8,
+				CacheExpiryInMins:      60,
+				LogStatsIntervalInSecs: 0,
+			},
+		},
+	})
+	Expect(err).ToNot(HaveOccurred())
+	api, _ = eth.NewPublicEthAPI(backend, nil, eth.APIConfig{StateDiffTimeout: shared.DefaultStateDiffTimeout})
+	tx, err = indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+	Expect(err).ToNot(HaveOccurred())
+
+	ipld := sdtypes.IPLD{
+		CID:     ipld.Keccak256ToCid(ipld.RawBinary, test_helpers.CodeHash.Bytes()).String(),
+		Content: test_helpers.ContractCode,
+	}
+	err = indexAndPublisher.PushIPLD(tx, ipld)
+	Expect(err).ToNot(HaveOccurred())
+
+	for _, node := range test_helpers.MockStateNodes {
+		err = indexAndPublisher.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	err = tx.Submit(err)
+	Expect(err).ToNot(HaveOccurred())
+
+	uncles := test_helpers.MockBlock.Uncles()
+	uncleHashes := make([]common.Hash, len(uncles))
+	for i, uncle := range uncles {
+		uncleHashes[i] = uncle.Hash()
+	}
+	expectedBlock["uncles"] = uncleHashes
+
+	// setting chain config to for london block
+	chainConfig.LondonBlock = big.NewInt(2)
+	indexAndPublisher = shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
+
+	tx, err = indexAndPublisher.PushBlock(test_helpers.MockLondonBlock, test_helpers.MockLondonReceipts, test_helpers.MockLondonBlock.Difficulty())
+	Expect(err).ToNot(HaveOccurred())
+
+	err = tx.Submit(err)
+	Expect(err).ToNot(HaveOccurred())
+})
+
+var _ = AfterSuite(func() { shared.TearDownDB(db) })
 
 var _ = Describe("API", func() {
-	var (
-		db  *postgres.DB
-		api *eth.PublicEthAPI
-	)
-	// Test db setup, rather than using BeforeEach we only need to setup once since the tests do not mutate the database
-	// Note: if you focus one of the tests be sure to focus this and the defered It()
-	It("test init", func() {
-		var err error
-		db, err = shared.SetupDB()
-		Expect(err).ToNot(HaveOccurred())
-		indexAndPublisher := eth2.NewIPLDPublisher(db)
-		backend, err := eth.NewEthBackend(db, &eth.Config{})
-		Expect(err).ToNot(HaveOccurred())
-		api = eth.NewPublicEthAPI(backend, nil)
-		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
-		Expect(err).ToNot(HaveOccurred())
-		err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)
-		Expect(err).ToNot(HaveOccurred())
-		uncles := test_helpers.MockBlock.Uncles()
-		uncleHashes := make([]common.Hash, len(uncles))
-		for i, uncle := range uncles {
-			uncleHashes[i] = uncle.Hash()
-		}
-		expectedBlock["uncles"] = uncleHashes
-	})
-	// Single test db tear down at end of all tests
-	defer It("test teardown", func() { eth.TearDownDB(db) })
 	/*
 
 	   Headers and blocks
@@ -221,8 +273,6 @@ var _ = Describe("API", func() {
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
 			Expect(header).To(BeNil())
-			_, err = api.B.DB.Beginx()
-			Expect(err).ToNot(HaveOccurred())
 		})
 	})
 
@@ -243,7 +293,7 @@ var _ = Describe("API", func() {
 			bn := api.BlockNumber()
 			ubn := (uint64)(bn)
 			subn := strconv.FormatUint(ubn, 10)
-			Expect(subn).To(Equal(test_helpers.BlockNumber.String()))
+			Expect(subn).To(Equal(test_helpers.LondonBlockNum.String()))
 		})
 	})
 
@@ -272,10 +322,32 @@ var _ = Describe("API", func() {
 				Expect(val).To(Equal(block[key]))
 			}
 		})
-		It("Throws an error if a block cannot be found", func() {
-			_, err := api.GetBlockByNumber(ctx, wrongNumber, false)
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		It("Returns `nil` if a block cannot be found", func() {
+			block, err := api.GetBlockByNumber(ctx, wrongNumber, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(block).To(BeNil())
+		})
+		It("Fetch BaseFee from london block by block number, returns `nil` for legacy block", func() {
+			block, err := api.GetBlockByNumber(ctx, number, false)
+			Expect(err).ToNot(HaveOccurred())
+			_, ok := block["baseFeePerGas"]
+			Expect(ok).To(Equal(false))
+
+			block, err = api.GetBlockByNumber(ctx, londonBlockNum, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(block["baseFeePerGas"]).To(Equal((*hexutil.Big)(baseFee)))
+		})
+		It("Retrieves a block by number with uncles in correct order", func() {
+			block, err := api.GetBlockByNumber(ctx, londonBlockNum, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedUncles := []common.Hash{
+				test_helpers.MockLondonUncles[0].Hash(),
+				test_helpers.MockLondonUncles[1].Hash(),
+			}
+			Expect(block["uncles"]).To(Equal(expectedUncles))
+			Expect(block["sha3Uncles"]).To(Equal(test_helpers.MockLondonBlock.UncleHash()))
+			Expect(block["hash"]).To(Equal(test_helpers.MockLondonBlock.Hash()))
 		})
 	})
 
@@ -304,10 +376,31 @@ var _ = Describe("API", func() {
 				Expect(val).To(Equal(block[key]))
 			}
 		})
-		It("Throws an error if a block cannot be found", func() {
-			_, err := api.GetBlockByHash(ctx, randomHash, false)
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		It("Returns `nil` if a block cannot be found", func() {
+			block, err := api.GetBlockByHash(ctx, randomHash, false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(block).To(BeZero())
+		})
+		It("Fetch BaseFee from london block by block hash, returns `nil` for legacy block", func() {
+			block, err := api.GetBlockByHash(ctx, test_helpers.MockBlock.Hash(), true)
+			Expect(err).ToNot(HaveOccurred())
+			_, ok := block["baseFeePerGas"]
+			Expect(ok).To(Equal(false))
+			block, err = api.GetBlockByHash(ctx, test_helpers.MockLondonBlock.Hash(), false)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(block["baseFeePerGas"]).To(Equal((*hexutil.Big)(baseFee)))
+		})
+		It("Retrieves a block by hash with uncles in correct order", func() {
+			block, err := api.GetBlockByHash(ctx, test_helpers.MockLondonBlock.Hash(), false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedUncles := []common.Hash{
+				test_helpers.MockLondonUncles[0].Hash(),
+				test_helpers.MockLondonUncles[1].Hash(),
+			}
+			Expect(block["uncles"]).To(Equal(expectedUncles))
+			Expect(block["sha3Uncles"]).To(Equal(test_helpers.MockLondonBlock.UncleHash()))
+			Expect(block["hash"]).To(Equal(test_helpers.MockLondonBlock.Hash()))
 		})
 	})
 
@@ -326,10 +419,10 @@ var _ = Describe("API", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(uncle2).To(Equal(expectedUncle2))
 		})
-		It("Throws an error if an block for blocknumber cannot be found", func() {
-			_, err := api.GetUncleByBlockNumberAndIndex(ctx, wrongNumber, 0)
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		It("Returns `nil` if an block for block number cannot be found", func() {
+			block, err := api.GetUncleByBlockNumberAndIndex(ctx, wrongNumber, 0)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(block).To(BeNil())
 		})
 		It("Returns `nil` if an uncle at the provided index does not exist for the block found for the provided block number", func() {
 			uncle, err := api.GetUncleByBlockNumberAndIndex(ctx, number, 2)
@@ -347,10 +440,10 @@ var _ = Describe("API", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(uncle2).To(Equal(expectedUncle2))
 		})
-		It("Throws an error if an block for blockhash cannot be found", func() {
-			_, err := api.GetUncleByBlockHashAndIndex(ctx, randomHash, 0)
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		It("Returns `nil` if a block for blockhash cannot be found", func() {
+			block, err := api.GetUncleByBlockHashAndIndex(ctx, randomHash, 0)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(block).To(BeNil())
 		})
 		It("Returns `nil` if an uncle at the provided index does not exist for the block with the provided hash", func() {
 			uncle, err := api.GetUncleByBlockHashAndIndex(ctx, blockHash, 2)
@@ -362,6 +455,7 @@ var _ = Describe("API", func() {
 	Describe("eth_getUncleCountByBlockNumber", func() {
 		It("Retrieves the number of uncles for the canonical block with the provided number", func() {
 			count := api.GetUncleCountByBlockNumber(ctx, number)
+			Expect(*count).NotTo(Equal(nil))
 			Expect(uint64(*count)).To(Equal(uint64(2)))
 		})
 	})
@@ -369,6 +463,7 @@ var _ = Describe("API", func() {
 	Describe("eth_getUncleCountByBlockHash", func() {
 		It("Retrieves the number of uncles for the block with the provided hash", func() {
 			count := api.GetUncleCountByBlockHash(ctx, blockHash)
+			Expect(*count).NotTo(Equal(nil))
 			Expect(uint64(*count)).To(Equal(uint64(2)))
 		})
 	})
@@ -403,14 +498,14 @@ var _ = Describe("API", func() {
 	Describe("eth_getBlockTransactionCountByNumber", func() {
 		It("Retrieves the number of transactions in the canonical block with the provided number", func() {
 			count := api.GetBlockTransactionCountByNumber(ctx, number)
-			Expect(uint64(*count)).To(Equal(uint64(3)))
+			Expect(uint64(*count)).To(Equal(uint64(4)))
 		})
 	})
 
 	Describe("eth_getBlockTransactionCountByHash", func() {
 		It("Retrieves the number of transactions in the block with the provided hash ", func() {
 			count := api.GetBlockTransactionCountByHash(ctx, blockHash)
-			Expect(uint64(*count)).To(Equal(uint64(3)))
+			Expect(uint64(*count)).To(Equal(uint64(4)))
 		})
 	})
 
@@ -428,6 +523,13 @@ var _ = Describe("API", func() {
 			Expect(tx).ToNot(BeNil())
 			Expect(tx).To(Equal(expectedTransaction3))
 		})
+		It("Retrieves the GasFeeCap and GasTipCap for dynamic transaction from the london block hash", func() {
+			tx := api.GetTransactionByBlockNumberAndIndex(ctx, londonBlockNum, 0)
+			Expect(tx).ToNot(BeNil())
+			Expect(tx.GasFeeCap).To(Equal((*hexutil.Big)(test_helpers.MockLondonTransactions[0].GasFeeCap())))
+			Expect(tx.GasTipCap).To(Equal((*hexutil.Big)(test_helpers.MockLondonTransactions[0].GasTipCap())))
+			Expect(tx).To(Equal(expectedLondonTransaction))
+		})
 	})
 
 	Describe("eth_getTransactionByBlockHashAndIndex", func() {
@@ -444,6 +546,14 @@ var _ = Describe("API", func() {
 			Expect(tx).ToNot(BeNil())
 			Expect(tx).To(Equal(expectedTransaction3))
 		})
+
+		It("Retrieves the GasFeeCap and GasTipCap for dynamic transaction from the london block hash", func() {
+			tx := api.GetTransactionByBlockHashAndIndex(ctx, test_helpers.MockLondonBlock.Hash(), 0)
+			Expect(tx).ToNot(BeNil())
+			Expect(tx.GasFeeCap).To(Equal((*hexutil.Big)(test_helpers.MockLondonTransactions[0].GasFeeCap())))
+			Expect(tx.GasTipCap).To(Equal((*hexutil.Big)(test_helpers.MockLondonTransactions[0].GasTipCap())))
+			Expect(tx).To(Equal(expectedLondonTransaction))
+		})
 	})
 
 	Describe("eth_getRawTransactionByBlockNumberAndIndex", func() {
@@ -555,7 +665,65 @@ var _ = Describe("API", func() {
 
 	Describe("eth_getLogs", func() {
 		It("Retrieves receipt logs that match the provided topics within the provided range", func() {
-			crit := ethereum.FilterQuery{
+			crit := filters.FilterCriteria{
+				Topics: [][]common.Hash{
+					{
+						common.HexToHash("0x0c"),
+					},
+					{
+						common.HexToHash("0x0a"),
+					},
+					{
+						common.HexToHash("0x0b"),
+					},
+				},
+				FromBlock: test_helpers.MockBlock.Number(),
+				ToBlock:   test_helpers.MockBlock.Number(),
+			}
+			logs, err := api.GetLogs(ctx, crit)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(0))
+
+			crit = filters.FilterCriteria{
+				Topics: [][]common.Hash{
+					{
+						common.HexToHash("0x08"),
+					},
+					{
+						common.HexToHash("0x0a"),
+					},
+					{
+						common.HexToHash("0x0c"),
+					},
+				},
+				FromBlock: test_helpers.MockBlock.Number(),
+				ToBlock:   test_helpers.MockBlock.Number(),
+			}
+			logs, err = api.GetLogs(ctx, crit)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(0))
+
+			crit = filters.FilterCriteria{
+				Topics: [][]common.Hash{
+					{
+						common.HexToHash("0x09"),
+					},
+					{
+						common.HexToHash("0x0a"),
+					},
+					{
+						common.HexToHash("0x0b"),
+					},
+				},
+				FromBlock: test_helpers.MockBlock.Number(),
+				ToBlock:   test_helpers.MockBlock.Number(),
+			}
+			logs, err = api.GetLogs(ctx, crit)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(1))
+			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog4}))
+
+			crit = filters.FilterCriteria{
 				Topics: [][]common.Hash{
 					{
 						common.HexToHash("0x04"),
@@ -564,12 +732,12 @@ var _ = Describe("API", func() {
 				FromBlock: test_helpers.MockBlock.Number(),
 				ToBlock:   test_helpers.MockBlock.Number(),
 			}
-			logs, err := api.GetLogs(ctx, crit)
+			logs, err = api.GetLogs(ctx, crit)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				Topics: [][]common.Hash{
 					{
 						common.HexToHash("0x04"),
@@ -584,7 +752,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(2))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				Topics: [][]common.Hash{
 					{
 						common.HexToHash("0x04"),
@@ -599,7 +767,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				Topics: [][]common.Hash{
 					{
 						common.HexToHash("0x04"),
@@ -615,7 +783,7 @@ var _ = Describe("API", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(logs)).To(Equal(0))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				Topics: [][]common.Hash{
 					{
 						common.HexToHash("0x04"),
@@ -632,7 +800,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				Topics: [][]common.Hash{
 					{
 						common.HexToHash("0x05"),
@@ -649,7 +817,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				Topics: [][]common.Hash{
 					{
 						common.HexToHash("0x05"),
@@ -667,7 +835,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				Topics: [][]common.Hash{
 					{
 						common.HexToHash("0x04"),
@@ -686,7 +854,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(2))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				Topics: [][]common.Hash{
 					{},
 					{
@@ -701,7 +869,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				Topics: [][]common.Hash{
 					{},
 					{
@@ -716,20 +884,20 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				Topics:    [][]common.Hash{},
 				FromBlock: test_helpers.MockBlock.Number(),
 				ToBlock:   test_helpers.MockBlock.Number(),
 			}
 			logs, err = api.GetLogs(ctx, crit)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
-			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2}))
+			Expect(len(logs)).To(Equal(6))
+			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2, test_helpers.MockLog3, test_helpers.MockLog4, test_helpers.MockLog5, test_helpers.MockLog6}))
 		})
 
 		It("Uses the provided blockhash if one is provided", func() {
 			hash := test_helpers.MockBlock.Hash()
-			crit := ethereum.FilterQuery{
+			crit := filters.FilterCriteria{
 				BlockHash: &hash,
 				Topics: [][]common.Hash{
 					{},
@@ -743,7 +911,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				BlockHash: &hash,
 				Topics: [][]common.Hash{
 					{
@@ -759,7 +927,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				BlockHash: &hash,
 				Topics: [][]common.Hash{
 					{},
@@ -773,7 +941,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				BlockHash: &hash,
 				Topics: [][]common.Hash{
 					{
@@ -789,7 +957,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				BlockHash: &hash,
 				Topics: [][]common.Hash{
 					{
@@ -804,7 +972,7 @@ var _ = Describe("API", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(logs)).To(Equal(0))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				BlockHash: &hash,
 				Topics: [][]common.Hash{
 					{
@@ -821,7 +989,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(1))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog2}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				BlockHash: &hash,
 				Topics: [][]common.Hash{
 					{
@@ -835,7 +1003,7 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(2))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				BlockHash: &hash,
 				Topics: [][]common.Hash{
 					{
@@ -853,19 +1021,19 @@ var _ = Describe("API", func() {
 			Expect(len(logs)).To(Equal(2))
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2}))
 
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				BlockHash: &hash,
 				Topics:    [][]common.Hash{},
 			}
 			logs, err = api.GetLogs(ctx, crit)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(len(logs)).To(Equal(2))
-			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2}))
+			Expect(len(logs)).To(Equal(6))
+			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2, test_helpers.MockLog3, test_helpers.MockLog4, test_helpers.MockLog5, test_helpers.MockLog6}))
 		})
 
 		It("Filters on contract address if any are provided", func() {
 			hash := test_helpers.MockBlock.Hash()
-			crit := ethereum.FilterQuery{
+			crit := filters.FilterCriteria{
 				BlockHash: &hash,
 				Addresses: []common.Address{
 					test_helpers.Address,
@@ -887,7 +1055,7 @@ var _ = Describe("API", func() {
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1}))
 
 			hash = test_helpers.MockBlock.Hash()
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				BlockHash: &hash,
 				Addresses: []common.Address{
 					test_helpers.Address,
@@ -910,7 +1078,7 @@ var _ = Describe("API", func() {
 			Expect(logs).To(Equal([]*types.Log{test_helpers.MockLog1, test_helpers.MockLog2}))
 
 			hash = test_helpers.MockBlock.Hash()
-			crit = ethereum.FilterQuery{
+			crit = filters.FilterCriteria{
 				BlockHash: &hash,
 				Addresses: []common.Address{
 					test_helpers.Address,
@@ -949,8 +1117,17 @@ var _ = Describe("API", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
 		})
-		It("Throws an error for an account it cannot find the balance for", func() {
-			_, err := api.GetBalance(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blockHash, true))
+		It("Retrieves the eth balance for the non-existing account address at the block with the provided hash", func() {
+			bal, err := api.GetBalance(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blockHash, true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+		})
+		It("Throws an error for an account of a non-existing block hash", func() {
+			_, err := api.GetBalance(ctx, test_helpers.AccountAddresss, rpc.BlockNumberOrHashWithHash(randomHash, true))
+			Expect(err).To(HaveOccurred())
+		})
+		It("Throws an error for an account of a non-existing block number", func() {
+			_, err := api.GetBalance(ctx, test_helpers.AccountAddresss, rpc.BlockNumberOrHashWithNumber(wrongNumber))
 			Expect(err).To(HaveOccurred())
 		})
 	})
@@ -966,26 +1143,10 @@ var _ = Describe("API", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
 		})
-		It("Throws an error for an account it cannot find the code for", func() {
-			_, err := api.GetCode(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blockHash, true))
-			Expect(err).To(HaveOccurred())
+		It("Returns `nil` for an account it cannot find the code for", func() {
+			code, err := api.GetCode(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blockHash, true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(BeEmpty())
 		})
 	})
 })
-
-func publishCode(db *postgres.DB, codeHash common.Hash, code []byte) error {
-	tx, err := db.Beginx()
-	if err != nil {
-		return err
-	}
-	mhKey, err := shared2.MultihashKeyFromKeccak256(codeHash)
-	if err != nil {
-		tx.Rollback()
-		return err
-	}
-	if err := shared2.PublishDirect(tx, mhKey, code); err != nil {
-		tx.Rollback()
-		return err
-	}
-	return tx.Commit()
-}

pkg/eth/api_test/eth_suite_test.go

@@ -14,24 +14,16 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package serve
+package eth_api_test
 
-import log "github.com/sirupsen/logrus"
+import (
+	"testing"
 
-func sendNonBlockingErr(sub Subscription, err error) {
-	log.Error(err)
-	select {
-	case sub.PayloadChan <- SubscriptionPayload{Data: nil, Err: err.Error(), Flag: EmptyFlag}:
-	default:
-		log.Infof("unable to send error to subscription %s", sub.ID)
-	}
-}
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
 
-func sendNonBlockingQuit(sub Subscription) {
-	select {
-	case sub.QuitChan <- true:
-		log.Infof("closing subscription %s", sub.ID)
-	default:
-		log.Infof("unable to close subscription %s; channel has no receiver", sub.ID)
-	}
+func TestETHSuite(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ipld-eth-server/pkg/eth/api_test")
 }

pkg/eth/backend_utils.go

@@ -17,26 +17,33 @@
 package eth
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/rpc"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/trie"
+	nodeiter "github.com/ethereum/go-ethereum/trie/concurrent_iterator"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/state"
 )
 
+var nullHashBytes = common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")
+var emptyCodeHash = crypto.Keccak256([]byte{})
+
 // RPCMarshalHeader converts the given header to the RPC output.
 // This function is eth/internal so we have to make our own version here...
 func RPCMarshalHeader(head *types.Header) map[string]interface{} {
-	return map[string]interface{}{
+	headerMap := map[string]interface{}{
 		"number":           (*hexutil.Big)(head.Number),
 		"hash":             head.Hash(),
 		"parentHash":       head.ParentHash,
@@ -55,6 +62,11 @@ func RPCMarshalHeader(head *types.Header) map[string]interface{} {
 		"transactionsRoot": head.TxHash,
 		"receiptsRoot":     head.ReceiptHash,
 	}
+
+	if head.BaseFee != nil {
+		headerMap["baseFeePerGas"] = (*hexutil.Big)(head.BaseFee)
+	}
+	return headerMap
 }
 
 // RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are
@@ -117,8 +129,8 @@ func RPCMarshalBlockWithUncleHashes(block *types.Block, uncleHashes []common.Has
 		}
 		fields["transactions"] = transactions
 	}
-	fields["uncles"] = uncleHashes
 
+	fields["uncles"] = uncleHashes
 	return fields, nil
 }
 
@@ -132,16 +144,23 @@ func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransa
 	return nil
 }
 
+// SignerForTx returns an appropriate Signer for this Transaction
+func SignerForTx(tx *types.Transaction) types.Signer {
+	var signer types.Signer
+	if tx.Protected() {
+		signer = types.LatestSignerForChainID(tx.ChainId())
+	} else {
+		signer = types.HomesteadSigner{}
+	}
+	return signer
+}
+
 // NewRPCTransaction returns a transaction that will serialize to the RPC
 // representation, with the given location metadata set (if available).
-func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
-	var signer types.Signer = types.FrontierSigner{}
-	if tx.Protected() {
-		signer = types.NewEIP155Signer(tx.ChainId())
-	}
+func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64, baseFee *big.Int) *RPCTransaction {
+	signer := SignerForTx(tx)
 	from, _ := types.Sender(signer, tx)
 	v, r, s := tx.RawSignatureValues()
-
 	result := &RPCTransaction{
 		From:     from,
 		Gas:      hexutil.Uint64(tx.Gas()),
@@ -151,6 +170,7 @@ func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber
 		Nonce:    hexutil.Uint64(tx.Nonce()),
 		To:       tx.To(),
 		Value:    (*hexutil.Big)(tx.Value()),
+		Type:     hexutil.Uint64(tx.Type()),
 		V:        (*hexutil.Big)(v),
 		R:        (*hexutil.Big)(r),
 		S:        (*hexutil.Big)(s),
@@ -160,6 +180,31 @@ func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber
 		result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber))
 		result.TransactionIndex = (*hexutil.Uint64)(&index)
 	}
+	switch tx.Type() {
+	case types.LegacyTxType:
+		// if a legacy transaction has an EIP-155 chain id, include it explicitly
+		if id := tx.ChainId(); id.Sign() != 0 {
+			result.ChainID = (*hexutil.Big)(id)
+		}
+	case types.AccessListTxType:
+		al := tx.AccessList()
+		result.Accesses = &al
+		result.ChainID = (*hexutil.Big)(tx.ChainId())
+	case types.DynamicFeeTxType:
+		al := tx.AccessList()
+		result.Accesses = &al
+		result.ChainID = (*hexutil.Big)(tx.ChainId())
+		result.GasFeeCap = (*hexutil.Big)(tx.GasFeeCap())
+		result.GasTipCap = (*hexutil.Big)(tx.GasTipCap())
+		// if the transaction has been mined, compute the effective gas price
+		if baseFee != nil && blockHash != (common.Hash{}) {
+			// price = min(tip, gasFeeCap - baseFee) + baseFee
+			price := math.BigMin(new(big.Int).Add(tx.GasTipCap(), baseFee), tx.GasFeeCap())
+			result.GasPrice = (*hexutil.Big)(price)
+		} else {
+			result.GasPrice = (*hexutil.Big)(tx.GasFeeCap())
+		}
+	}
 	return result
 }
 
@@ -230,7 +275,7 @@ func newRPCRawTransactionFromBlockIndex(b *types.Block, index uint64) hexutil.By
 	if index >= uint64(len(txs)) {
 		return nil
 	}
-	blob, _ := rlp.EncodeToBytes(txs[index])
+	blob, _ := txs[index].MarshalBinary()
 	return blob
 }
 
@@ -240,55 +285,7 @@ func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransacti
 	if index >= uint64(len(txs)) {
 		return nil
 	}
-	return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index)
-}
-
-// extractLogsOfInterest returns logs from the receipt IPLD
-func extractLogsOfInterest(rctIPLDs []ipfs.BlockModel, wantedTopics [][]string) ([]*types.Log, error) {
-	var logs []*types.Log
-	for _, rctIPLD := range rctIPLDs {
-		rctRLP := rctIPLD
-		var rct types.Receipt
-		if err := rlp.DecodeBytes(rctRLP.Data, &rct); err != nil {
-			return nil, err
-		}
-		for _, log := range rct.Logs {
-			if wanted := wantedLog(wantedTopics, log.Topics); wanted == true {
-				logs = append(logs, log)
-			}
-		}
-	}
-	return logs, nil
-}
-
-// returns true if the log matches on the filter
-func wantedLog(wantedTopics [][]string, actualTopics []common.Hash) bool {
-	// actualTopics will always have length <= 4
-	// wantedTopics will always have length 4
-	matches := 0
-	for i, actualTopic := range actualTopics {
-		// If we have topics in this filter slot, count as a match if the actualTopic matches one of the ones in this filter slot
-		if len(wantedTopics[i]) > 0 {
-			matches += sliceContainsHash(wantedTopics[i], actualTopic)
-		} else {
-			// Filter slot is empty, not matching any topics at this slot => counts as a match
-			matches++
-		}
-	}
-	if matches == len(actualTopics) {
-		return true
-	}
-	return false
-}
-
-// returns 1 if the slice contains the hash, 0 if it does not
-func sliceContainsHash(slice []string, hash common.Hash) int {
-	for _, str := range slice {
-		if str == hash.String() {
-			return 1
-		}
-	}
-	return 0
+	return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index, b.BaseFee())
 }
 
 func toFilterArg(q ethereum.FilterQuery) (interface{}, error) {
@@ -318,3 +315,103 @@ func toBlockNumArg(number *big.Int) string {
 	}
 	return hexutil.EncodeBig(number)
 }
+
+func getIteratorAtPath(t state.Trie, startKey []byte) (trie.NodeIterator, int64) {
+	startTime := makeTimestamp()
+	var it trie.NodeIterator
+
+	if len(startKey)%2 != 0 {
+		// Zero-pad for odd-length keys, required by HexToKeyBytes()
+		startKey = append(startKey, 0)
+		it = t.NodeIterator(nodeiter.HexToKeyBytes(startKey))
+	} else {
+		it = t.NodeIterator(nodeiter.HexToKeyBytes(startKey))
+		// Step to the required node (not required if original startKey was odd-length)
+		it.Next(true)
+	}
+
+	return it, makeTimestamp() - startTime
+}
+
+func fillSliceNodeData(
+	sdb state.Database,
+	nodesMap map[string]string,
+	leavesMap map[string]GetSliceResponseAccount,
+	node StateNode,
+	nodeElements []interface{},
+	storage bool,
+) (int64, error) {
+	// Populate the nodes map
+	nodeValHash := crypto.Keccak256Hash(node.NodeValue)
+	nodesMap[common.Bytes2Hex(nodeValHash.Bytes())] = common.Bytes2Hex(node.NodeValue)
+
+	// Extract account data if it's a Leaf node
+	leafStartTime := makeTimestamp()
+	if node.NodeType == Leaf && !storage {
+		stateLeafKey, storageRoot, code, err := extractContractAccountInfo(sdb, node, nodeElements)
+		if err != nil {
+			return 0, fmt.Errorf("GetSlice account lookup error: %s", err.Error())
+		}
+
+		if len(code) > 0 {
+			// Populate the leaves map
+			leavesMap[stateLeafKey] = GetSliceResponseAccount{
+				StorageRoot: storageRoot,
+				EVMCode:     common.Bytes2Hex(code),
+			}
+		}
+	}
+
+	return makeTimestamp() - leafStartTime, nil
+}
+
+func extractContractAccountInfo(sdb state.Database, node StateNode, nodeElements []interface{}) (string, string, []byte, error) {
+	var account types.StateAccount
+	if err := rlp.DecodeBytes(nodeElements[1].([]byte), &account); err != nil {
+		return "", "", nil, fmt.Errorf("error decoding account for leaf node at path %x nerror: %v", node.Path, err)
+	}
+
+	if bytes.Equal(account.CodeHash, emptyCodeHash) {
+		return "", "", nil, nil
+	}
+
+	// Extract state leaf key
+	partialPath := trie.CompactToHex(nodeElements[0].([]byte))
+	valueNodePath := append(node.Path, partialPath...)
+	encodedPath := trie.HexToCompact(valueNodePath)
+	leafKey := encodedPath[1:]
+	stateLeafKeyString := common.BytesToHash(leafKey).String()
+
+	storageRootString := account.Root.String()
+
+	// Extract codeHash and get code
+	codeHash := common.BytesToHash(account.CodeHash)
+	codeBytes, err := sdb.ContractCode(codeHash)
+	if err != nil {
+		return "", "", nil, err
+	}
+
+	return stateLeafKeyString, storageRootString, codeBytes, nil
+}
+
+// IsLeaf checks if the node we are at is a leaf
+func IsLeaf(elements []interface{}) (bool, error) {
+	if len(elements) > 2 {
+		return false, nil
+	}
+	if len(elements) < 2 {
+		return false, fmt.Errorf("node cannot be less than two elements in length")
+	}
+	switch elements[0].([]byte)[0] / 16 {
+	case '\x00':
+		return false, nil
+	case '\x01':
+		return false, nil
+	case '\x02':
+		return true, nil
+	case '\x03':
+		return true, nil
+	default:
+		return false, fmt.Errorf("unknown hex prefix")
+	}
+}

pkg/eth/cid_retriever.go

@@ -1,572 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"fmt"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/jmoiron/sqlx"
-	"github.com/lib/pq"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-// Retriever interface for substituting mocks in tests
-type Retriever interface {
-	RetrieveFirstBlockNumber() (int64, error)
-	RetrieveLastBlockNumber() (int64, error)
-	Retrieve(filter SubscriptionSettings, blockNumber int64) ([]CIDWrapper, bool, error)
-}
-
-// CIDRetriever satisfies the CIDRetriever interface for ethereum
-type CIDRetriever struct {
-	db *postgres.DB
-}
-
-// NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface
-func NewCIDRetriever(db *postgres.DB) *CIDRetriever {
-	return &CIDRetriever{
-		db: db,
-	}
-}
-
-// RetrieveFirstBlockNumber is used to retrieve the first block number in the db
-func (ecr *CIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
-	var blockNumber int64
-	err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number ASC LIMIT 1")
-	return blockNumber, err
-}
-
-// RetrieveLastBlockNumber is used to retrieve the latest block number in the db
-func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
-	var blockNumber int64
-	err := ecr.db.Get(&blockNumber, "SELECT block_number FROM eth.header_cids ORDER BY block_number DESC LIMIT 1 ")
-	return blockNumber, err
-}
-
-// Retrieve is used to retrieve all of the CIDs which conform to the passed StreamFilters
-func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64) ([]CIDWrapper, bool, error) {
-	log.Debug("retrieving cids")
-
-	// Begin new db tx
-	tx, err := ecr.db.Beginx()
-	if err != nil {
-		return nil, true, err
-	}
-	defer func() {
-		if p := recover(); p != nil {
-			shared.Rollback(tx)
-			panic(p)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = tx.Commit()
-		}
-	}()
-
-	// Retrieve cached header CIDs at this block height
-	headers, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
-	if err != nil {
-		log.Error("header cid retrieval error")
-		return nil, true, err
-	}
-	cws := make([]CIDWrapper, len(headers))
-	empty := true
-	for i, header := range headers {
-		cw := new(CIDWrapper)
-		cw.BlockNumber = big.NewInt(blockNumber)
-		if !filter.HeaderFilter.Off {
-			cw.Header = header
-			empty = false
-			if filter.HeaderFilter.Uncles {
-				// Retrieve uncle cids for this header id
-				uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, header.ID)
-				if err != nil {
-					log.Error("uncle cid retrieval error")
-					return nil, true, err
-				}
-				cw.Uncles = uncleCIDs
-			}
-		}
-		// Retrieve cached trx CIDs
-		if !filter.TxFilter.Off {
-			cw.Transactions, err = ecr.RetrieveTxCIDs(tx, filter.TxFilter, header.ID)
-			if err != nil {
-				log.Error("transaction cid retrieval error")
-				return nil, true, err
-			}
-			if len(cw.Transactions) > 0 {
-				empty = false
-			}
-		}
-		trxIds := make([]int64, len(cw.Transactions))
-		for j, tx := range cw.Transactions {
-			trxIds[j] = tx.ID
-		}
-		// Retrieve cached receipt CIDs
-		if !filter.ReceiptFilter.Off {
-			cw.Receipts, err = ecr.RetrieveRctCIDsByHeaderID(tx, filter.ReceiptFilter, header.ID, trxIds)
-			if err != nil {
-				log.Error("receipt cid retrieval error")
-				return nil, true, err
-			}
-			if len(cw.Receipts) > 0 {
-				empty = false
-			}
-		}
-		// Retrieve cached state CIDs
-		if !filter.StateFilter.Off {
-			cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, filter.StateFilter, header.ID)
-			if err != nil {
-				log.Error("state cid retrieval error")
-				return nil, true, err
-			}
-			if len(cw.StateNodes) > 0 {
-				empty = false
-			}
-		}
-		// Retrieve cached storage CIDs
-		if !filter.StorageFilter.Off {
-			cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, filter.StorageFilter, header.ID)
-			if err != nil {
-				log.Error("storage cid retrieval error")
-				return nil, true, err
-			}
-			if len(cw.StorageNodes) > 0 {
-				empty = false
-			}
-		}
-		cws[i] = *cw
-	}
-
-	return cws, empty, err
-}
-
-// RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
-func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]eth.HeaderModel, error) {
-	log.Debug("retrieving header cids for block ", blockNumber)
-	headers := make([]eth.HeaderModel, 0)
-	pgStr := `SELECT * FROM eth.header_cids
-				WHERE block_number = $1`
-	return headers, tx.Select(&headers, pgStr, blockNumber)
-}
-
-// RetrieveUncleCIDsByHeaderID retrieves and returns all of the uncle cids for the provided header
-func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth.UncleModel, error) {
-	log.Debug("retrieving uncle cids for block id ", headerID)
-	headers := make([]eth.UncleModel, 0)
-	pgStr := `SELECT * FROM eth.uncle_cids
-				WHERE header_id = $1`
-	return headers, tx.Select(&headers, pgStr, headerID)
-}
-
-// RetrieveTxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters
-// also returns the ids for the returned transaction cids
-func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]eth.TxModel, error) {
-	log.Debug("retrieving transaction cids for header id ", headerID)
-	args := make([]interface{}, 0, 3)
-	results := make([]eth.TxModel, 0)
-	id := 1
-	pgStr := fmt.Sprintf(`SELECT transaction_cids.id, transaction_cids.header_id,
- 			transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.mh_key,
- 			transaction_cids.dst, transaction_cids.src, transaction_cids.index, transaction_cids.tx_data
- 			FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-			WHERE header_cids.id = $%d`, id)
-	args = append(args, headerID)
-	id++
-	if len(txFilter.Dst) > 0 {
-		pgStr += fmt.Sprintf(` AND transaction_cids.dst = ANY($%d::VARCHAR(66)[])`, id)
-		args = append(args, pq.Array(txFilter.Dst))
-		id++
-	}
-	if len(txFilter.Src) > 0 {
-		pgStr += fmt.Sprintf(` AND transaction_cids.src = ANY($%d::VARCHAR(66)[])`, id)
-		args = append(args, pq.Array(txFilter.Src))
-	}
-	pgStr += ` ORDER BY transaction_cids.index`
-	return results, tx.Select(&results, pgStr, args...)
-}
-
-// RetrieveRctCIDsByHeaderID retrieves and returns all of the rct cids at the provided header ID that conform to the provided
-// filter parameters and correspond to the provided tx ids
-func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]eth.ReceiptModel, error) {
-	log.Debug("retrieving receipt cids for header id ", headerID)
-	args := make([]interface{}, 0, 4)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
- 			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
-			WHERE receipt_cids.tx_id = transaction_cids.id 
-			AND transaction_cids.header_id = header_cids.id
-			AND header_cids.id = $1`
-	id := 2
-	args = append(args, headerID)
-	if len(rctFilter.LogAddresses) > 0 {
-		// Filter on log contract addresses if there are any
-		pgStr += fmt.Sprintf(` AND ((receipt_cids.log_contracts && $%d::VARCHAR(66)[]`, id)
-		args = append(args, pq.Array(rctFilter.LogAddresses))
-		id++
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND ("
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
-			pgStr += ")"
-		}
-		pgStr += ")"
-		// Filter on txIDs if there are any and we are matching txs
-		if rctFilter.MatchTxs && len(trxIds) > 0 {
-			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-		pgStr += ")"
-	} else { // If there are no contract addresses to filter on
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND (("
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
-			pgStr += ")"
-			// Filter on txIDs if there are any and we are matching txs
-			if rctFilter.MatchTxs && len(trxIds) > 0 {
-				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-				args = append(args, pq.Array(trxIds))
-			}
-			pgStr += ")"
-		} else if rctFilter.MatchTxs && len(trxIds) > 0 {
-			// If there are no contract addresses or topics to filter on,
-			// Filter on txIDs if there are any and we are matching txs
-			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-	}
-	pgStr += ` ORDER BY transaction_cids.index`
-	receiptCids := make([]eth.ReceiptModel, 0)
-	return receiptCids, tx.Select(&receiptCids, pgStr, args...)
-}
-
-// RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight or block hash that conform to the provided
-// filter parameters and correspond to the provided tx ids
-func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]eth.ReceiptModel, error) {
-	log.Debug("retrieving receipt cids for block ", blockNumber)
-	args := make([]interface{}, 0, 5)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
- 			FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
-			WHERE receipt_cids.tx_id = transaction_cids.id 
-			AND transaction_cids.header_id = header_cids.id`
-	id := 1
-	if blockNumber > 0 {
-		pgStr += fmt.Sprintf(` AND header_cids.block_number = $%d`, id)
-		args = append(args, blockNumber)
-		id++
-	}
-	if blockHash != nil {
-		pgStr += fmt.Sprintf(` AND header_cids.block_hash = $%d`, id)
-		args = append(args, blockHash.String())
-		id++
-	}
-	if len(rctFilter.LogAddresses) > 0 {
-		// Filter on log contract addresses if there are any
-		pgStr += fmt.Sprintf(` AND ((receipt_cids.log_contracts && $%d::VARCHAR(66)[]`, id)
-		args = append(args, pq.Array(rctFilter.LogAddresses))
-		id++
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND ("
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
-			pgStr += ")"
-		}
-		pgStr += ")"
-		// Filter on txIDs if there are any and we are matching txs
-		if rctFilter.MatchTxs && len(trxIds) > 0 {
-			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-		pgStr += ")"
-	} else { // If there are no contract addresses to filter on
-		// Filter on topics if there are any
-		if hasTopics(rctFilter.Topics) {
-			pgStr += " AND (("
-			first := true
-			for i, topicSet := range rctFilter.Topics {
-				if i < 4 && len(topicSet) > 0 {
-					if first {
-						pgStr += fmt.Sprintf(`receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-						first = false
-					} else {
-						pgStr += fmt.Sprintf(` AND receipt_cids.topic%ds && $%d::VARCHAR(66)[]`, i, id)
-					}
-					args = append(args, pq.Array(topicSet))
-					id++
-				}
-			}
-			pgStr += ")"
-			// Filter on txIDs if there are any and we are matching txs
-			if rctFilter.MatchTxs && len(trxIds) > 0 {
-				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-				args = append(args, pq.Array(trxIds))
-			}
-			pgStr += ")"
-		} else if rctFilter.MatchTxs && len(trxIds) > 0 {
-			// If there are no contract addresses or topics to filter on,
-			// Filter on txIDs if there are any and we are matching txs
-			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d::INTEGER[])`, id)
-			args = append(args, pq.Array(trxIds))
-		}
-	}
-	pgStr += ` ORDER BY transaction_cids.index`
-	receiptCids := make([]eth.ReceiptModel, 0)
-	return receiptCids, tx.Select(&receiptCids, pgStr, args...)
-}
-
-func hasTopics(topics [][]string) bool {
-	for _, topicSet := range topics {
-		if len(topicSet) > 0 {
-			return true
-		}
-	}
-	return false
-}
-
-// RetrieveStateCIDs retrieves and returns all of the state node cids at the provided header ID that conform to the provided filter parameters
-func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID int64) ([]eth.StateNodeModel, error) {
-	log.Debug("retrieving state cids for header id ", headerID)
-	args := make([]interface{}, 0, 2)
-	pgStr := `SELECT state_cids.id, state_cids.header_id,
-			state_cids.state_leaf_key, state_cids.node_type, state_cids.cid, state_cids.mh_key, state_cids.state_path
-			FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-			WHERE header_cids.id = $1`
-	args = append(args, headerID)
-	addrLen := len(stateFilter.Addresses)
-	if addrLen > 0 {
-		keys := make([]string, addrLen)
-		for i, addr := range stateFilter.Addresses {
-			keys[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes()).String()
-		}
-		pgStr += ` AND state_cids.state_leaf_key = ANY($2::VARCHAR(66)[])`
-		args = append(args, pq.Array(keys))
-	}
-	if !stateFilter.IntermediateNodes {
-		pgStr += ` AND state_cids.node_type = 2`
-	}
-	stateNodeCIDs := make([]eth.StateNodeModel, 0)
-	return stateNodeCIDs, tx.Select(&stateNodeCIDs, pgStr, args...)
-}
-
-// RetrieveStorageCIDs retrieves and returns all of the storage node cids at the provided header id that conform to the provided filter parameters
-func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID int64) ([]eth.StorageNodeWithStateKeyModel, error) {
-	log.Debug("retrieving storage cids for header id ", headerID)
-	args := make([]interface{}, 0, 3)
-	pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_leaf_key, storage_cids.node_type,
- 			storage_cids.cid, storage_cids.mh_key, storage_cids.storage_path, state_cids.state_leaf_key
- 			FROM eth.storage_cids, eth.state_cids, eth.header_cids
-			WHERE storage_cids.state_id = state_cids.id 
-			AND state_cids.header_id = header_cids.id
-			AND header_cids.id = $1`
-	args = append(args, headerID)
-	id := 2
-	addrLen := len(storageFilter.Addresses)
-	if addrLen > 0 {
-		keys := make([]string, addrLen)
-		for i, addr := range storageFilter.Addresses {
-			keys[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes()).String()
-		}
-		pgStr += fmt.Sprintf(` AND state_cids.state_leaf_key = ANY($%d::VARCHAR(66)[])`, id)
-		args = append(args, pq.Array(keys))
-		id++
-	}
-	if len(storageFilter.StorageKeys) > 0 {
-		pgStr += fmt.Sprintf(` AND storage_cids.storage_leaf_key = ANY($%d::VARCHAR(66)[])`, id)
-		args = append(args, pq.Array(storageFilter.StorageKeys))
-	}
-	if !storageFilter.IntermediateNodes {
-		pgStr += ` AND storage_cids.node_type = 2`
-	}
-	storageNodeCIDs := make([]eth.StorageNodeWithStateKeyModel, 0)
-	return storageNodeCIDs, tx.Select(&storageNodeCIDs, pgStr, args...)
-}
-
-// RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash
-func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth.HeaderModel, []eth.UncleModel, []eth.TxModel, []eth.ReceiptModel, error) {
-	log.Debug("retrieving block cids for block hash ", blockHash.String())
-
-	// Begin new db tx
-	tx, err := ecr.db.Beginx()
-	if err != nil {
-		return eth.HeaderModel{}, nil, nil, nil, err
-	}
-	defer func() {
-		if p := recover(); p != nil {
-			shared.Rollback(tx)
-			panic(p)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = tx.Commit()
-		}
-	}()
-
-	headerCID, err := ecr.RetrieveHeaderCIDByHash(tx, blockHash)
-	if err != nil {
-		log.Error("header cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
-	}
-	uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.ID)
-	if err != nil {
-		log.Error("uncle cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
-	}
-	txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.ID)
-	if err != nil {
-		log.Error("tx cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
-	}
-	txIDs := make([]int64, len(txCIDs))
-	for i, txCID := range txCIDs {
-		txIDs[i] = txCID.ID
-	}
-	rctCIDs, err := ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
-	if err != nil {
-		log.Error("rct cid retrieval error")
-	}
-	return headerCID, uncleCIDs, txCIDs, rctCIDs, err
-}
-
-// RetrieveBlockByNumber returns all of the CIDs needed to compose an entire block, for a given block number
-func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth.HeaderModel, []eth.UncleModel, []eth.TxModel, []eth.ReceiptModel, error) {
-	log.Debug("retrieving block cids for block number ", blockNumber)
-
-	// Begin new db tx
-	tx, err := ecr.db.Beginx()
-	if err != nil {
-		return eth.HeaderModel{}, nil, nil, nil, err
-	}
-	defer func() {
-		if p := recover(); p != nil {
-			shared.Rollback(tx)
-			panic(p)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = tx.Commit()
-		}
-	}()
-
-	headerCID, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
-	if err != nil {
-		log.Error("header cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
-	}
-	if len(headerCID) < 1 {
-		return eth.HeaderModel{}, nil, nil, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber)
-	}
-	uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID[0].ID)
-	if err != nil {
-		log.Error("uncle cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
-	}
-	txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].ID)
-	if err != nil {
-		log.Error("tx cid retrieval error")
-		return eth.HeaderModel{}, nil, nil, nil, err
-	}
-	txIDs := make([]int64, len(txCIDs))
-	for i, txCID := range txCIDs {
-		txIDs[i] = txCID.ID
-	}
-	rctCIDs, err := ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
-	if err != nil {
-		log.Error("rct cid retrieval error")
-	}
-	return headerCID[0], uncleCIDs, txCIDs, rctCIDs, err
-}
-
-// RetrieveHeaderCIDByHash returns the header for the given block hash
-func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (eth.HeaderModel, error) {
-	log.Debug("retrieving header cids for block hash ", blockHash.String())
-	pgStr := `SELECT * FROM eth.header_cids
-			WHERE block_hash = $1`
-	var headerCID eth.HeaderModel
-	return headerCID, tx.Get(&headerCID, pgStr, blockHash.String())
-}
-
-// RetrieveTxCIDsByHeaderID retrieves all tx CIDs for the given header id
-func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth.TxModel, error) {
-	log.Debug("retrieving tx cids for block id ", headerID)
-	pgStr := `SELECT * FROM eth.transaction_cids
-			WHERE header_id = $1
-			ORDER BY index`
-	var txCIDs []eth.TxModel
-	return txCIDs, tx.Select(&txCIDs, pgStr, headerID)
-}
-
-// RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs
-func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]eth.ReceiptModel, error) {
-	log.Debugf("retrieving receipt cids for tx ids %v", txIDs)
-	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
- 			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
-			receipt_cids.topic2s, receipt_cids.topic3s, receipt_cids.log_contracts
-			FROM eth.receipt_cids, eth.transaction_cids
-			WHERE tx_id = ANY($1::INTEGER[])
-			AND receipt_cids.tx_id = transaction_cids.id
-			ORDER BY transaction_cids.index`
-	var rctCIDs []eth.ReceiptModel
-	return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txIDs))
-}

pkg/eth/cid_retriever_test.go

@@ -1,474 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-var (
-	openFilter = eth.SubscriptionSettings{
-		Start:         big.NewInt(0),
-		End:           big.NewInt(1),
-		HeaderFilter:  eth.HeaderFilter{},
-		TxFilter:      eth.TxFilter{},
-		ReceiptFilter: eth.ReceiptFilter{},
-		StateFilter:   eth.StateFilter{},
-		StorageFilter: eth.StorageFilter{},
-	}
-	rctAddressFilter = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			LogAddresses: []string{test_helpers.Address.String()},
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctTopicsFilter = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			Topics: [][]string{{"0x0000000000000000000000000000000000000000000000000000000000000004"}},
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctTopicsAndAddressFilter = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			Topics: [][]string{
-				{"0x0000000000000000000000000000000000000000000000000000000000000004"},
-				{"0x0000000000000000000000000000000000000000000000000000000000000006"},
-			},
-			LogAddresses: []string{test_helpers.Address.String()},
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctTopicsAndAddressFilterFail = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			Topics: [][]string{
-				{"0x0000000000000000000000000000000000000000000000000000000000000004"},
-				{"0x0000000000000000000000000000000000000000000000000000000000000007"}, // This topic won't match on the mocks.Address.String() contract receipt
-			},
-			LogAddresses: []string{test_helpers.Address.String()},
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctAddressesAndTopicFilter = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			Topics:       [][]string{{"0x0000000000000000000000000000000000000000000000000000000000000005"}},
-			LogAddresses: []string{test_helpers.Address.String(), test_helpers.AnotherAddress.String()},
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctsForAllCollectedTrxs = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{}, // Trx filter open so we will collect all trxs, therefore we will also collect all corresponding rcts despite rct filter
-		ReceiptFilter: eth.ReceiptFilter{
-			MatchTxs:     true,
-			Topics:       [][]string{{"0x0000000000000000000000000000000000000000000000000000000000000006"}}, // Topic0 isn't one of the topic0s we have
-			LogAddresses: []string{"0x0000000000000000000000000000000000000002"},                             // Contract isn't one of the contracts we have
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	rctsForSelectCollectedTrxs = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Dst: []string{test_helpers.AnotherAddress.String()}, // We only filter for one of the trxs so we will only get the one corresponding receipt
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			MatchTxs:     true,
-			Topics:       [][]string{{"0x0000000000000000000000000000000000000000000000000000000000000006"}}, // Topic0 isn't one of the topic0s we have
-			LogAddresses: []string{"0x0000000000000000000000000000000000000002"},                             // Contract isn't one of the contracts we have
-		},
-		StateFilter: eth.StateFilter{
-			Off: true,
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-	stateFilter = eth.SubscriptionSettings{
-		Start: big.NewInt(0),
-		End:   big.NewInt(1),
-		HeaderFilter: eth.HeaderFilter{
-			Off: true,
-		},
-		TxFilter: eth.TxFilter{
-			Off: true,
-		},
-		ReceiptFilter: eth.ReceiptFilter{
-			Off: true,
-		},
-		StateFilter: eth.StateFilter{
-			Addresses: []string{test_helpers.AccountAddresss.Hex()},
-		},
-		StorageFilter: eth.StorageFilter{
-			Off: true,
-		},
-	}
-)
-
-var _ = Describe("Retriever", func() {
-	var (
-		db        *postgres.DB
-		repo      *eth2.IPLDPublisher
-		retriever *eth.CIDRetriever
-	)
-	BeforeEach(func() {
-		var err error
-		db, err = shared.SetupDB()
-		Expect(err).ToNot(HaveOccurred())
-		repo = eth2.NewIPLDPublisher(db)
-		retriever = eth.NewCIDRetriever(db)
-	})
-	AfterEach(func() {
-		eth.TearDownDB(db)
-	})
-
-	Describe("Retrieve", func() {
-		BeforeEach(func() {
-			err := repo.Publish(test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-		})
-		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-			cids, empty, err := retriever.Retrieve(openFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids)).To(Equal(1))
-			Expect(cids[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			expectedHeaderCID := test_helpers.MockCIDWrapper.Header
-			expectedHeaderCID.ID = cids[0].Header.ID
-			expectedHeaderCID.NodeID = cids[0].Header.NodeID
-			Expect(cids[0].Header).To(Equal(expectedHeaderCID))
-			Expect(len(cids[0].Transactions)).To(Equal(3))
-			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[0].CID)).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[2].CID)).To(BeTrue())
-			Expect(len(cids[0].Receipts)).To(Equal(3))
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[0].CID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[2].CID)).To(BeTrue())
-			Expect(len(cids[0].StateNodes)).To(Equal(2))
-			for _, stateNode := range cids[0].StateNodes {
-				if stateNode.CID == test_helpers.State1CID.String() {
-					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(test_helpers.ContractLeafKey).Hex()))
-					Expect(stateNode.NodeType).To(Equal(2))
-					Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
-				}
-				if stateNode.CID == test_helpers.State2CID.String() {
-					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(test_helpers.AccountLeafKey).Hex()))
-					Expect(stateNode.NodeType).To(Equal(2))
-					Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
-				}
-			}
-			Expect(len(cids[0].StorageNodes)).To(Equal(1))
-			expectedStorageNodeCIDs := test_helpers.MockCIDWrapper.StorageNodes
-			expectedStorageNodeCIDs[0].ID = cids[0].StorageNodes[0].ID
-			expectedStorageNodeCIDs[0].StateID = cids[0].StorageNodes[0].StateID
-			Expect(cids[0].StorageNodes).To(Equal(expectedStorageNodeCIDs))
-		})
-
-		It("Applies filters from the provided config.Subscription", func() {
-			cids1, empty, err := retriever.Retrieve(rctAddressFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids1)).To(Equal(1))
-			Expect(cids1[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids1[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids1[0].Transactions)).To(Equal(0))
-			Expect(len(cids1[0].StateNodes)).To(Equal(0))
-			Expect(len(cids1[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids1[0].Receipts)).To(Equal(1))
-			expectedReceiptCID := test_helpers.MockCIDWrapper.Receipts[0]
-			expectedReceiptCID.ID = cids1[0].Receipts[0].ID
-			expectedReceiptCID.TxID = cids1[0].Receipts[0].TxID
-			Expect(cids1[0].Receipts[0]).To(Equal(expectedReceiptCID))
-
-			cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids2)).To(Equal(1))
-			Expect(cids2[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids2[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids2[0].Transactions)).To(Equal(0))
-			Expect(len(cids2[0].StateNodes)).To(Equal(0))
-			Expect(len(cids2[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids2[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
-			expectedReceiptCID.ID = cids2[0].Receipts[0].ID
-			expectedReceiptCID.TxID = cids2[0].Receipts[0].TxID
-			Expect(cids2[0].Receipts[0]).To(Equal(expectedReceiptCID))
-
-			cids3, empty, err := retriever.Retrieve(rctTopicsAndAddressFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids3)).To(Equal(1))
-			Expect(cids3[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids3[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids3[0].Transactions)).To(Equal(0))
-			Expect(len(cids3[0].StateNodes)).To(Equal(0))
-			Expect(len(cids3[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids3[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
-			expectedReceiptCID.ID = cids3[0].Receipts[0].ID
-			expectedReceiptCID.TxID = cids3[0].Receipts[0].TxID
-			Expect(cids3[0].Receipts[0]).To(Equal(expectedReceiptCID))
-
-			cids4, empty, err := retriever.Retrieve(rctAddressesAndTopicFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids4)).To(Equal(1))
-			Expect(cids4[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids4[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids4[0].Transactions)).To(Equal(0))
-			Expect(len(cids4[0].StateNodes)).To(Equal(0))
-			Expect(len(cids4[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids4[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
-			expectedReceiptCID.ID = cids4[0].Receipts[0].ID
-			expectedReceiptCID.TxID = cids4[0].Receipts[0].TxID
-			Expect(cids4[0].Receipts[0]).To(Equal(expectedReceiptCID))
-
-			cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids5)).To(Equal(1))
-			Expect(cids5[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids5[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids5[0].Transactions)).To(Equal(3))
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx1CID.String())).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx2CID.String())).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx3CID.String())).To(BeTrue())
-			Expect(len(cids5[0].StateNodes)).To(Equal(0))
-			Expect(len(cids5[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids5[0].Receipts)).To(Equal(3))
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct1CID.String())).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct2CID.String())).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct3CID.String())).To(BeTrue())
-
-			cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids6)).To(Equal(1))
-			Expect(cids6[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids6[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids6[0].Transactions)).To(Equal(1))
-			expectedTxCID := test_helpers.MockCIDWrapper.Transactions[1]
-			expectedTxCID.ID = cids6[0].Transactions[0].ID
-			expectedTxCID.HeaderID = cids6[0].Transactions[0].HeaderID
-			Expect(cids6[0].Transactions[0]).To(Equal(expectedTxCID))
-			Expect(len(cids6[0].StateNodes)).To(Equal(0))
-			Expect(len(cids6[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids6[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
-			expectedReceiptCID.ID = cids6[0].Receipts[0].ID
-			expectedReceiptCID.TxID = cids6[0].Receipts[0].TxID
-			Expect(cids6[0].Receipts[0]).To(Equal(expectedReceiptCID))
-
-			cids7, empty, err := retriever.Retrieve(stateFilter, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).ToNot(BeTrue())
-			Expect(len(cids7)).To(Equal(1))
-			Expect(cids7[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
-			Expect(cids7[0].Header).To(Equal(eth2.HeaderModel{}))
-			Expect(len(cids7[0].Transactions)).To(Equal(0))
-			Expect(len(cids7[0].Receipts)).To(Equal(0))
-			Expect(len(cids7[0].StorageNodes)).To(Equal(0))
-			Expect(len(cids7[0].StateNodes)).To(Equal(1))
-			Expect(cids7[0].StateNodes[0]).To(Equal(eth2.StateNodeModel{
-				ID:       cids7[0].StateNodes[0].ID,
-				HeaderID: cids7[0].StateNodes[0].HeaderID,
-				NodeType: 2,
-				StateKey: common.BytesToHash(test_helpers.AccountLeafKey).Hex(),
-				CID:      test_helpers.State2CID.String(),
-				MhKey:    test_helpers.State2MhKey,
-				Path:     []byte{'\x0c'},
-			}))
-
-			_, empty, err = retriever.Retrieve(rctTopicsAndAddressFilterFail, 1)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(empty).To(BeTrue())
-		})
-	})
-
-	Describe("RetrieveFirstBlockNumber", func() {
-		It("Throws an error if there are no blocks in the database", func() {
-			_, err := retriever.RetrieveFirstBlockNumber()
-			Expect(err).To(HaveOccurred())
-		})
-		It("Gets the number of the first block that has data in the database", func() {
-			err := repo.Publish(test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveFirstBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(1)))
-		})
-
-		It("Gets the number of the first block that has data in the database", func() {
-			payload := test_helpers.MockConvertedPayload
-			payload.Block = newMockBlock(1010101)
-			err := repo.Publish(payload)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveFirstBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(1010101)))
-		})
-
-		It("Gets the number of the first block that has data in the database", func() {
-			payload1 := test_helpers.MockConvertedPayload
-			payload1.Block = newMockBlock(1010101)
-			payload2 := payload1
-			payload2.Block = newMockBlock(5)
-			err := repo.Publish(payload1)
-			Expect(err).ToNot(HaveOccurred())
-			err = repo.Publish(payload2)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveFirstBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(5)))
-		})
-	})
-
-	Describe("RetrieveLastBlockNumber", func() {
-		It("Throws an error if there are no blocks in the database", func() {
-			_, err := retriever.RetrieveLastBlockNumber()
-			Expect(err).To(HaveOccurred())
-		})
-		It("Gets the number of the latest block that has data in the database", func() {
-			err := repo.Publish(test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveLastBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(1)))
-		})
-
-		It("Gets the number of the latest block that has data in the database", func() {
-			payload := test_helpers.MockConvertedPayload
-			payload.Block = newMockBlock(1010101)
-			err := repo.Publish(payload)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveLastBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(1010101)))
-		})
-
-		It("Gets the number of the latest block that has data in the database", func() {
-			payload1 := test_helpers.MockConvertedPayload
-			payload1.Block = newMockBlock(1010101)
-			payload2 := payload1
-			payload2.Block = newMockBlock(5)
-			err := repo.Publish(payload1)
-			Expect(err).ToNot(HaveOccurred())
-			err = repo.Publish(payload2)
-			Expect(err).ToNot(HaveOccurred())
-			num, err := retriever.RetrieveLastBlockNumber()
-			Expect(err).ToNot(HaveOccurred())
-			Expect(num).To(Equal(int64(1010101)))
-		})
-	})
-})
-
-func newMockBlock(blockNumber uint64) *types.Block {
-	header := test_helpers.MockHeader
-	header.Number.SetUint64(blockNumber)
-	return types.NewBlock(&test_helpers.MockHeader, test_helpers.MockTransactions, nil, test_helpers.MockReceipts)
-}

pkg/eth/errors.go

@@ -0,0 +1,17 @@
+package eth
+
+import "fmt"
+
+type RequiresProxyError struct {
+	method string
+}
+
+var _ error = RequiresProxyError{}
+
+func (e RequiresProxyError) SetMethod(method string) {
+	e.method = method
+}
+
+func (e RequiresProxyError) Error() string {
+	return fmt.Sprintf("%s requires a configured proxy geth node", e.method)
+}

pkg/eth/eth_state_test.go

@@ -1,467 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	"bytes"
-	"context"
-	"io/ioutil"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/statediff"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-var (
-	parsedABI abi.ABI
-)
-
-func init() {
-	// load abi
-	abiBytes, err := ioutil.ReadFile("./test_helpers/abi.json")
-	if err != nil {
-		panic(err)
-	}
-	parsedABI, err = abi.JSON(bytes.NewReader(abiBytes))
-	if err != nil {
-		panic(err)
-	}
-}
-
-var _ = Describe("eth state reading tests", func() {
-	var (
-		blocks                  []*types.Block
-		receipts                []types.Receipts
-		chain                   *core.BlockChain
-		db                      *postgres.DB
-		api                     *eth.PublicEthAPI
-		backend                 *eth.Backend
-		chainConfig             = params.TestChainConfig
-		mockTD                  = big.NewInt(1337)
-		expectedCanonicalHeader map[string]interface{}
-	)
-	It("test init", func() {
-		// db and type initializations
-		var err error
-		db, err = shared.SetupDB()
-		Expect(err).ToNot(HaveOccurred())
-		transformer := eth2.NewStateDiffTransformer(chainConfig, db)
-		backend, err = eth.NewEthBackend(db, &eth.Config{
-			ChainConfig: chainConfig,
-			VmConfig:    vm.Config{},
-			RPCGasCap:   big.NewInt(10000000000),
-		})
-		Expect(err).ToNot(HaveOccurred())
-		api = eth.NewPublicEthAPI(backend, nil)
-
-		// make the test blockchain (and state)
-		blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen)
-		params := statediff.Params{
-			IntermediateStateNodes:   true,
-			IntermediateStorageNodes: true,
-		}
-		canonicalHeader := blocks[1].Header()
-		expectedCanonicalHeader = map[string]interface{}{
-			"number":           (*hexutil.Big)(canonicalHeader.Number),
-			"hash":             canonicalHeader.Hash(),
-			"parentHash":       canonicalHeader.ParentHash,
-			"nonce":            canonicalHeader.Nonce,
-			"mixHash":          canonicalHeader.MixDigest,
-			"sha3Uncles":       canonicalHeader.UncleHash,
-			"logsBloom":        canonicalHeader.Bloom,
-			"stateRoot":        canonicalHeader.Root,
-			"miner":            canonicalHeader.Coinbase,
-			"difficulty":       (*hexutil.Big)(canonicalHeader.Difficulty),
-			"extraData":        hexutil.Bytes([]byte{}),
-			"size":             hexutil.Uint64(canonicalHeader.Size()),
-			"gasLimit":         hexutil.Uint64(canonicalHeader.GasLimit),
-			"gasUsed":          hexutil.Uint64(canonicalHeader.GasUsed),
-			"timestamp":        hexutil.Uint64(canonicalHeader.Time),
-			"transactionsRoot": canonicalHeader.TxHash,
-			"receiptsRoot":     canonicalHeader.ReceiptHash,
-			"totalDifficulty":  (*hexutil.Big)(mockTD),
-		}
-		// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
-		builder := statediff.NewBuilder(chain.StateCache())
-		for i, block := range blocks {
-			var args statediff.Args
-			var rcts types.Receipts
-			if i == 0 {
-				args = statediff.Args{
-					OldStateRoot: common.Hash{},
-					NewStateRoot: block.Root(),
-					BlockNumber:  block.Number(),
-					BlockHash:    block.Hash(),
-				}
-			} else {
-				args = statediff.Args{
-					OldStateRoot: blocks[i-1].Root(),
-					NewStateRoot: block.Root(),
-					BlockNumber:  block.Number(),
-					BlockHash:    block.Hash(),
-				}
-				rcts = receipts[i-1]
-			}
-			diff, err := builder.BuildStateDiffObject(args, params)
-			Expect(err).ToNot(HaveOccurred())
-			diffRlp, err := rlp.EncodeToBytes(diff)
-			Expect(err).ToNot(HaveOccurred())
-			blockRlp, err := rlp.EncodeToBytes(block)
-			Expect(err).ToNot(HaveOccurred())
-			receiptsRlp, err := rlp.EncodeToBytes(rcts)
-			Expect(err).ToNot(HaveOccurred())
-			payload := statediff.Payload{
-				StateObjectRlp:  diffRlp,
-				BlockRlp:        blockRlp,
-				ReceiptsRlp:     receiptsRlp,
-				TotalDifficulty: mockTD,
-			}
-			_, err = transformer.Transform(0, payload)
-			Expect(err).ToNot(HaveOccurred())
-		}
-
-		// Insert some non-canonical data into the database so that we test our ability to discern canonicity
-		indexAndPublisher := eth2.NewIPLDPublisher(db)
-		api = eth.NewPublicEthAPI(backend, nil)
-		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
-		Expect(err).ToNot(HaveOccurred())
-		// The non-canonical header has a child
-		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayloadForChild)
-		Expect(err).ToNot(HaveOccurred())
-		err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)
-		Expect(err).ToNot(HaveOccurred())
-	})
-	defer It("test teardown", func() {
-		eth.TearDownDB(db)
-		chain.Stop()
-	})
-
-	Describe("eth_call", func() {
-		It("Applies call args (tx data) on top of state, returning the result (e.g. a Getter method call)", func() {
-			data, err := parsedABI.Pack("data")
-			Expect(err).ToNot(HaveOccurred())
-			bdata := hexutil.Bytes(data)
-			callArgs := eth.CallArgs{
-				To:   &test_helpers.ContractAddr,
-				Data: &bdata,
-			}
-			// Before contract deployment, returns nil
-			res, err := api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(0), nil)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res).To(BeNil())
-
-			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(1), nil)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(res).To(BeNil())
-
-			// After deployment
-			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(2), nil)
-			Expect(err).ToNot(HaveOccurred())
-			expectedRes := hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
-			Expect(res).To(Equal(expectedRes))
-
-			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(3), nil)
-			Expect(err).ToNot(HaveOccurred())
-			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003"))
-			Expect(res).To(Equal(expectedRes))
-
-			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(4), nil)
-			Expect(err).ToNot(HaveOccurred())
-			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009"))
-			Expect(res).To(Equal(expectedRes))
-
-			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(5), nil)
-			Expect(err).ToNot(HaveOccurred())
-			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000"))
-			Expect(res).To(Equal(expectedRes))
-		})
-	})
-
-	var (
-		expectedContractBalance   = (*hexutil.Big)(common.Big0)
-		expectedBankBalanceBlock0 = (*hexutil.Big)(test_helpers.TestBankFunds)
-
-		expectedAcct1BalanceBlock1 = (*hexutil.Big)(big.NewInt(10000))
-		expectedBankBalanceBlock1  = (*hexutil.Big)(new(big.Int).Sub(test_helpers.TestBankFunds, big.NewInt(10000)))
-
-		expectedAcct2BalanceBlock2 = (*hexutil.Big)(big.NewInt(1000))
-		expectedBankBalanceBlock2  = (*hexutil.Big)(new(big.Int).Sub(expectedBankBalanceBlock1.ToInt(), big.NewInt(1000)))
-
-		expectedAcct2BalanceBlock3 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock2.ToInt(), test_helpers.MiningReward))
-
-		expectedAcct2BalanceBlock4 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock3.ToInt(), test_helpers.MiningReward))
-
-		expectedAcct1BalanceBlock5 = (*hexutil.Big)(new(big.Int).Add(expectedAcct1BalanceBlock1.ToInt(), test_helpers.MiningReward))
-	)
-
-	Describe("eth_getBalance", func() {
-		It("Retrieves the eth balance for the provided account address at the block with the provided number", func() {
-			bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(0))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock0))
-
-			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(1))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
-			_, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(1))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			_, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(1))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(1))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock1))
-
-			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(2))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
-			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(2))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct2BalanceBlock2))
-			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(2))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedContractBalance))
-			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(2))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock2))
-
-			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(3))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
-			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(3))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct2BalanceBlock3))
-			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedContractBalance))
-			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(3))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock2))
-
-			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(4))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
-			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(4))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
-			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(4))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedContractBalance))
-			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(4))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock2))
-
-			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(5))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct1BalanceBlock5))
-			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(5))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
-			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedContractBalance))
-			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(5))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock2))
-		})
-		It("Retrieves the eth balance for the provided account address at the block with the provided hash", func() {
-			bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock0))
-
-			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
-			_, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			_, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock1))
-
-			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
-			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct2BalanceBlock2))
-			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedContractBalance))
-			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock2))
-
-			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
-			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct2BalanceBlock3))
-			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedContractBalance))
-			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock2))
-
-			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
-			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
-			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedContractBalance))
-			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock2))
-
-			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct1BalanceBlock5))
-			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
-			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedContractBalance))
-			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(bal).To(Equal(expectedBankBalanceBlock2))
-		})
-		It("Throws an error for an account it cannot find the balance for an account at the provided block number", func() {
-			_, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(0))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			_, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(0))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			_, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-		})
-		It("Throws an error for an account it cannot find the balance for an account at the provided block hash", func() {
-			_, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			_, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-			_, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-		})
-	})
-
-	Describe("eth_getCode", func() {
-		It("Retrieves the code for the provided contract address at the block with the provided number", func() {
-			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
-
-			code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
-		})
-		It("Retrieves the code for the provided contract address at the block with the provided hash", func() {
-			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
-
-			code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
-		})
-		It("Throws an error for an account it cannot find the code for", func() {
-			_, err := api.GetCode(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
-			Expect(err).To(HaveOccurred())
-		})
-		It("Throws an error for a contract that doesn't exist at this hieght", func() {
-			_, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0))
-			Expect(err).To(HaveOccurred())
-		})
-	})
-
-	Describe("eth_getStorageAt", func() {
-		It("Throws an error if it tries to access a contract which does not exist", func() {
-			_, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(0))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-
-			_, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(1))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-		})
-		It("Throws an error if it tries to access a contract slot which does not exist", func() {
-			_, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, randomHash.Hex(), rpc.BlockNumberOrHashWithNumber(2))
-			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
-		})
-		It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() {
-			// After deployment
-			val, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(2))
-			Expect(err).ToNot(HaveOccurred())
-			expectedRes := hexutil.Bytes(common.Hex2Bytes("01"))
-			Expect(val).To(Equal(expectedRes))
-
-			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(3))
-			Expect(err).ToNot(HaveOccurred())
-			expectedRes = hexutil.Bytes(common.Hex2Bytes("03"))
-			Expect(val).To(Equal(expectedRes))
-
-			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(4))
-			Expect(err).ToNot(HaveOccurred())
-			expectedRes = hexutil.Bytes(common.Hex2Bytes("09"))
-			Expect(val).To(Equal(expectedRes))
-
-			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(5))
-			Expect(err).ToNot(HaveOccurred())
-			Expect(val).To(Equal(hexutil.Bytes{}))
-		})
-	})
-
-	Describe("eth_getHeaderByNumber", func() {
-		It("Finds the canonical header based on the header's weight relative to others at the provided height", func() {
-			header, err := api.GetHeaderByNumber(ctx, number)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(header).To(Equal(expectedCanonicalHeader))
-		})
-	})
-})

pkg/eth/eth_suite_test.go

@@ -17,19 +17,13 @@
 package eth_test
 
 import (
-	"io/ioutil"
 	"testing"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"github.com/sirupsen/logrus"
 )
 
-func TestETHWatcher(t *testing.T) {
+func TestETHSuite(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "eth ipld server eth suite test")
+	RunSpecs(t, "ipld-eth-server/pkg/eth")
 }
-
-var _ = BeforeSuite(func() {
-	logrus.SetOutput(ioutil.Discard)
-})

pkg/eth/filterer.go

@@ -1,323 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"bytes"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/statediff"
-	"github.com/multiformats/go-multihash"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
-)
-
-// Filterer interface for substituing mocks in tests
-type Filterer interface {
-	Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error)
-}
-
-// ResponseFilterer satisfies the ResponseFilterer interface for ethereum
-type ResponseFilterer struct{}
-
-// NewResponseFilterer creates a new Filterer satisfying the ResponseFilterer interface
-func NewResponseFilterer() *ResponseFilterer {
-	return &ResponseFilterer{}
-}
-
-// Filter is used to filter through eth data to extract and package requested data into a Payload
-func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error) {
-	if checkRange(filter.Start.Int64(), filter.End.Int64(), payload.Block.Number().Int64()) {
-		response := new(IPLDs)
-		response.TotalDifficulty = payload.TotalDifficulty
-		if err := s.filterHeaders(filter.HeaderFilter, response, payload); err != nil {
-			return nil, err
-		}
-		txHashes, err := s.filterTransactions(filter.TxFilter, response, payload)
-		if err != nil {
-			return nil, err
-		}
-		var filterTxs []common.Hash
-		if filter.ReceiptFilter.MatchTxs {
-			filterTxs = txHashes
-		}
-		if err := s.filerReceipts(filter.ReceiptFilter, response, payload, filterTxs); err != nil {
-			return nil, err
-		}
-		if err := s.filterStateAndStorage(filter.StateFilter, filter.StorageFilter, response, payload); err != nil {
-			return nil, err
-		}
-		response.BlockNumber = payload.Block.Number()
-		return response, nil
-	}
-	return nil, nil
-}
-
-func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IPLDs, payload eth.ConvertedPayload) error {
-	if !headerFilter.Off {
-		headerRLP, err := rlp.EncodeToBytes(payload.Block.Header())
-		if err != nil {
-			return err
-		}
-		cid, err := ipld.RawdataToCid(ipld.MEthHeader, headerRLP, multihash.KECCAK_256)
-		if err != nil {
-			return err
-		}
-		response.Header = ipfs.BlockModel{
-			Data: headerRLP,
-			CID:  cid.String(),
-		}
-		if headerFilter.Uncles {
-			response.Uncles = make([]ipfs.BlockModel, len(payload.Block.Body().Uncles))
-			for i, uncle := range payload.Block.Body().Uncles {
-				uncleRlp, err := rlp.EncodeToBytes(uncle)
-				if err != nil {
-					return err
-				}
-				cid, err := ipld.RawdataToCid(ipld.MEthHeader, uncleRlp, multihash.KECCAK_256)
-				if err != nil {
-					return err
-				}
-				response.Uncles[i] = ipfs.BlockModel{
-					Data: uncleRlp,
-					CID:  cid.String(),
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func checkRange(start, end, actual int64) bool {
-	if (end <= 0 || end >= actual) && start <= actual {
-		return true
-	}
-	return false
-}
-
-func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload eth.ConvertedPayload) ([]common.Hash, error) {
-	var trxHashes []common.Hash
-	if !trxFilter.Off {
-		trxLen := len(payload.Block.Body().Transactions)
-		trxHashes = make([]common.Hash, 0, trxLen)
-		response.Transactions = make([]ipfs.BlockModel, 0, trxLen)
-		for i, trx := range payload.Block.Body().Transactions {
-			// TODO: check if want corresponding receipt and if we do we must include this transaction
-			if checkTransactionAddrs(trxFilter.Src, trxFilter.Dst, payload.TxMetaData[i].Src, payload.TxMetaData[i].Dst) {
-				trxBuffer := new(bytes.Buffer)
-				if err := trx.EncodeRLP(trxBuffer); err != nil {
-					return nil, err
-				}
-				data := trxBuffer.Bytes()
-				cid, err := ipld.RawdataToCid(ipld.MEthTx, data, multihash.KECCAK_256)
-				if err != nil {
-					return nil, err
-				}
-				response.Transactions = append(response.Transactions, ipfs.BlockModel{
-					Data: data,
-					CID:  cid.String(),
-				})
-				trxHashes = append(trxHashes, trx.Hash())
-			}
-		}
-	}
-	return trxHashes, nil
-}
-
-// checkTransactionAddrs returns true if either the transaction src and dst are one of the wanted src and dst addresses
-func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst string) bool {
-	// If we aren't filtering for any addresses, every transaction is a go
-	if len(wantedDst) == 0 && len(wantedSrc) == 0 {
-		return true
-	}
-	for _, src := range wantedSrc {
-		if src == actualSrc {
-			return true
-		}
-	}
-	for _, dst := range wantedDst {
-		if dst == actualDst {
-			return true
-		}
-	}
-	return false
-}
-
-func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload eth.ConvertedPayload, trxHashes []common.Hash) error {
-	if !receiptFilter.Off {
-		response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
-		for i, receipt := range payload.Receipts {
-			// topics is always length 4
-			topics := [][]string{payload.ReceiptMetaData[i].Topic0s, payload.ReceiptMetaData[i].Topic1s, payload.ReceiptMetaData[i].Topic2s, payload.ReceiptMetaData[i].Topic3s}
-			if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.LogAddresses, payload.ReceiptMetaData[i].LogContracts, trxHashes) {
-				receiptBuffer := new(bytes.Buffer)
-				if err := receipt.EncodeRLP(receiptBuffer); err != nil {
-					return err
-				}
-				data := receiptBuffer.Bytes()
-				cid, err := ipld.RawdataToCid(ipld.MEthTxReceipt, data, multihash.KECCAK_256)
-				if err != nil {
-					return err
-				}
-				response.Receipts = append(response.Receipts, ipfs.BlockModel{
-					Data: data,
-					CID:  cid.String(),
-				})
-			}
-		}
-	}
-	return nil
-}
-
-func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics [][]string, wantedAddresses []string, actualAddresses []string, wantedTrxHashes []common.Hash) bool {
-	// If we aren't filtering for any topics, contracts, or corresponding trxs then all receipts are a go
-	if len(wantedTopics) == 0 && len(wantedAddresses) == 0 && len(wantedTrxHashes) == 0 {
-		return true
-	}
-	// Keep receipts that are from watched txs
-	for _, wantedTrxHash := range wantedTrxHashes {
-		if bytes.Equal(wantedTrxHash.Bytes(), rct.TxHash.Bytes()) {
-			return true
-		}
-	}
-	// If there are no wanted contract addresses, we keep all receipts that match the topic filter
-	if len(wantedAddresses) == 0 {
-		if match := filterMatch(wantedTopics, actualTopics); match == true {
-			return true
-		}
-	}
-	// If there are wanted contract addresses to filter on
-	for _, wantedAddr := range wantedAddresses {
-		// and this is an address of interest
-		for _, actualAddr := range actualAddresses {
-			if wantedAddr == actualAddr {
-				// we keep the receipt if it matches on the topic filter
-				if match := filterMatch(wantedTopics, actualTopics); match == true {
-					return true
-				}
-			}
-		}
-	}
-	return false
-}
-
-// filterMatch returns true if the actualTopics conform to the wantedTopics filter
-func filterMatch(wantedTopics, actualTopics [][]string) bool {
-	// actualTopics should always be length 4, but the members can be nil slices
-	matches := 0
-	for i, actualTopicSet := range actualTopics {
-		if i < len(wantedTopics) && len(wantedTopics[i]) > 0 {
-			// If we have topics in this filter slot, count as a match if one of the topics matches
-			matches += slicesShareString(actualTopicSet, wantedTopics[i])
-		} else {
-			// Filter slot is either empty or doesn't exist => not matching any topics at this slot => counts as a match
-			matches++
-		}
-	}
-	if matches == 4 {
-		return true
-	}
-	return false
-}
-
-// returns 1 if the two slices have a string in common, 0 if they do not
-func slicesShareString(slice1, slice2 []string) int {
-	for _, str1 := range slice1 {
-		for _, str2 := range slice2 {
-			if str1 == str2 {
-				return 1
-			}
-		}
-	}
-	return 0
-}
-
-// filterStateAndStorage filters state and storage nodes into the response according to the provided filters
-func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload eth.ConvertedPayload) error {
-	response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
-	response.StorageNodes = make([]StorageNode, 0)
-	stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses))
-	for i, addr := range stateFilter.Addresses {
-		stateAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
-	}
-	storageAddressFilters := make([]common.Hash, len(storageFilter.Addresses))
-	for i, addr := range storageFilter.Addresses {
-		storageAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
-	}
-	storageKeyFilters := make([]common.Hash, len(storageFilter.StorageKeys))
-	for i, store := range storageFilter.StorageKeys {
-		storageKeyFilters[i] = common.HexToHash(store)
-	}
-	for _, stateNode := range payload.StateNodes {
-		if !stateFilter.Off && checkNodeKeys(stateAddressFilters, stateNode.LeafKey) {
-			if stateNode.Type == statediff.Leaf || stateFilter.IntermediateNodes {
-				cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
-				if err != nil {
-					return err
-				}
-				response.StateNodes = append(response.StateNodes, StateNode{
-					StateLeafKey: stateNode.LeafKey,
-					Path:         stateNode.Path,
-					IPLD: ipfs.BlockModel{
-						Data: stateNode.Value,
-						CID:  cid.String(),
-					},
-					Type: stateNode.Type,
-				})
-			}
-		}
-		if !storageFilter.Off && checkNodeKeys(storageAddressFilters, stateNode.LeafKey) {
-			for _, storageNode := range payload.StorageNodes[common.Bytes2Hex(stateNode.Path)] {
-				if checkNodeKeys(storageKeyFilters, storageNode.LeafKey) {
-					cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
-					if err != nil {
-						return err
-					}
-					response.StorageNodes = append(response.StorageNodes, StorageNode{
-						StateLeafKey:   stateNode.LeafKey,
-						StorageLeafKey: storageNode.LeafKey,
-						IPLD: ipfs.BlockModel{
-							Data: storageNode.Value,
-							CID:  cid.String(),
-						},
-						Type: storageNode.Type,
-						Path: storageNode.Path,
-					})
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
-	// If we aren't filtering for any specific keys, all nodes are a go
-	if len(wantedKeys) == 0 {
-		return true
-	}
-	for _, key := range wantedKeys {
-		if bytes.Equal(key.Bytes(), actualKey.Bytes()) {
-			return true
-		}
-	}
-	return false
-}

pkg/eth/filterer_test.go

@@ -1,200 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	"bytes"
-
-	"github.com/ethereum/go-ethereum/statediff"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-var (
-	filterer *eth.ResponseFilterer
-)
-
-var _ = Describe("Filterer", func() {
-	Describe("FilterResponse", func() {
-		BeforeEach(func() {
-			filterer = eth.NewResponseFilterer()
-		})
-
-		It("Transcribes all the data from the IPLDPayload into the StreamPayload if given an open filter", func() {
-			iplds, err := filterer.Filter(openFilter, test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds).ToNot(BeNil())
-			Expect(iplds.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds.Header).To(Equal(test_helpers.MockIPLDs.Header))
-			var expectedEmptyUncles []ipfs.BlockModel
-			Expect(iplds.Uncles).To(Equal(expectedEmptyUncles))
-			Expect(len(iplds.Transactions)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.MockTransactions.GetRlp(2))).To(BeTrue())
-			Expect(len(iplds.Receipts)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.MockReceipts.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.MockReceipts.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.MockReceipts.GetRlp(2))).To(BeTrue())
-			Expect(len(iplds.StateNodes)).To(Equal(2))
-			for _, stateNode := range iplds.StateNodes {
-				Expect(stateNode.Type).To(Equal(statediff.Leaf))
-				if bytes.Equal(stateNode.StateLeafKey.Bytes(), test_helpers.AccountLeafKey) {
-					Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
-						Data: test_helpers.State2IPLD.RawData(),
-						CID:  test_helpers.State2IPLD.Cid().String(),
-					}))
-				}
-				if bytes.Equal(stateNode.StateLeafKey.Bytes(), test_helpers.ContractLeafKey) {
-					Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
-						Data: test_helpers.State1IPLD.RawData(),
-						CID:  test_helpers.State1IPLD.Cid().String(),
-					}))
-				}
-			}
-			Expect(iplds.StorageNodes).To(Equal(test_helpers.MockIPLDs.StorageNodes))
-		})
-
-		It("Applies filters from the provided config.Subscription", func() {
-			iplds1, err := filterer.Filter(rctAddressFilter, test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds1).ToNot(BeNil())
-			Expect(iplds1.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds1.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds1.Uncles)).To(Equal(0))
-			Expect(len(iplds1.Transactions)).To(Equal(0))
-			Expect(len(iplds1.StorageNodes)).To(Equal(0))
-			Expect(len(iplds1.StateNodes)).To(Equal(0))
-			Expect(len(iplds1.Receipts)).To(Equal(1))
-			Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
-			}))
-
-			iplds2, err := filterer.Filter(rctTopicsFilter, test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds2).ToNot(BeNil())
-			Expect(iplds2.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds2.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds2.Uncles)).To(Equal(0))
-			Expect(len(iplds2.Transactions)).To(Equal(0))
-			Expect(len(iplds2.StorageNodes)).To(Equal(0))
-			Expect(len(iplds2.StateNodes)).To(Equal(0))
-			Expect(len(iplds2.Receipts)).To(Equal(1))
-			Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
-			}))
-
-			iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds3).ToNot(BeNil())
-			Expect(iplds3.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds3.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds3.Uncles)).To(Equal(0))
-			Expect(len(iplds3.Transactions)).To(Equal(0))
-			Expect(len(iplds3.StorageNodes)).To(Equal(0))
-			Expect(len(iplds3.StateNodes)).To(Equal(0))
-			Expect(len(iplds3.Receipts)).To(Equal(1))
-			Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct1IPLD.RawData(),
-				CID:  test_helpers.Rct1IPLD.Cid().String(),
-			}))
-
-			iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds4).ToNot(BeNil())
-			Expect(iplds4.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds4.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds4.Uncles)).To(Equal(0))
-			Expect(len(iplds4.Transactions)).To(Equal(0))
-			Expect(len(iplds4.StorageNodes)).To(Equal(0))
-			Expect(len(iplds4.StateNodes)).To(Equal(0))
-			Expect(len(iplds4.Receipts)).To(Equal(1))
-			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct2IPLD.RawData(),
-				CID:  test_helpers.Rct2IPLD.Cid().String(),
-			}))
-
-			iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds5).ToNot(BeNil())
-			Expect(iplds5.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds5.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds5.Uncles)).To(Equal(0))
-			Expect(len(iplds5.Transactions)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(2))).To(BeTrue())
-			Expect(len(iplds5.StorageNodes)).To(Equal(0))
-			Expect(len(iplds5.StateNodes)).To(Equal(0))
-			Expect(len(iplds5.Receipts)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.MockReceipts.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.MockReceipts.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.MockReceipts.GetRlp(2))).To(BeTrue())
-
-			iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds6).ToNot(BeNil())
-			Expect(iplds6.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds6.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds6.Uncles)).To(Equal(0))
-			Expect(len(iplds6.Transactions)).To(Equal(1))
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(len(iplds6.StorageNodes)).To(Equal(0))
-			Expect(len(iplds6.StateNodes)).To(Equal(0))
-			Expect(len(iplds6.Receipts)).To(Equal(1))
-			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.Rct2IPLD.RawData(),
-				CID:  test_helpers.Rct2IPLD.Cid().String(),
-			}))
-
-			iplds7, err := filterer.Filter(stateFilter, test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds7).ToNot(BeNil())
-			Expect(iplds7.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds7.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds7.Uncles)).To(Equal(0))
-			Expect(len(iplds7.Transactions)).To(Equal(0))
-			Expect(len(iplds7.StorageNodes)).To(Equal(0))
-			Expect(len(iplds7.Receipts)).To(Equal(0))
-			Expect(len(iplds7.StateNodes)).To(Equal(1))
-			Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(test_helpers.AccountLeafKey))
-			Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
-				Data: test_helpers.State2IPLD.RawData(),
-				CID:  test_helpers.State2IPLD.Cid().String(),
-			}))
-
-			iplds8, err := filterer.Filter(rctTopicsAndAddressFilterFail, test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds8).ToNot(BeNil())
-			Expect(iplds8.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds8.Header).To(Equal(ipfs.BlockModel{}))
-			Expect(len(iplds8.Uncles)).To(Equal(0))
-			Expect(len(iplds8.Transactions)).To(Equal(0))
-			Expect(len(iplds8.StorageNodes)).To(Equal(0))
-			Expect(len(iplds8.StateNodes)).To(Equal(0))
-			Expect(len(iplds8.Receipts)).To(Equal(0))
-		})
-	})
-})

pkg/eth/helpers.go

@@ -17,39 +17,10 @@
 package eth
 
 import (
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/statediff"
+	"time"
 )
 
-func ResolveToNodeType(nodeType int) statediff.NodeType {
-	switch nodeType {
-	case 0:
-		return statediff.Branch
-	case 1:
-		return statediff.Extension
-	case 2:
-		return statediff.Leaf
-	case 3:
-		return statediff.Removed
-	default:
-		return statediff.Unknown
-	}
-}
-
-// ChainConfig returns the appropriate ethereum chain config for the provided chain id
-func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
-	switch chainID {
-	case 1:
-		return params.MainnetChainConfig, nil
-	case 3:
-		return params.TestnetChainConfig, nil // Ropsten
-	case 4:
-		return params.RinkebyChainConfig, nil
-	case 5:
-		return params.GoerliChainConfig, nil
-	default:
-		return nil, fmt.Errorf("chain config for chainid %d not available", chainID)
-	}
+// Timestamp in milliseconds
+func makeTimestamp() int64 {
+	return time.Now().UnixNano() / int64(time.Millisecond)
 }

pkg/eth/interfaces.go

@@ -1,47 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"context"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/bloombits"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/event"
-	"github.com/ethereum/go-ethereum/rpc"
-)
-
-// FilterBackend is the geth interface we need to satisfy to use their filters
-type FilterBackend interface {
-	ChainDb() ethdb.Database
-	HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error)
-	HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error)
-	GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
-	GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error)
-
-	SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
-	SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
-	SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
-	SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
-	SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
-
-	BloomStatus() (uint64, uint64)
-	ServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
-}

pkg/eth/ipld_fetcher.go

@@ -1,219 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"errors"
-	"fmt"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/jmoiron/sqlx"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-// Fetcher interface for substituting mocks in tests
-type Fetcher interface {
-	Fetch(cids CIDWrapper) (*IPLDs, error)
-}
-
-// IPLDFetcher satisfies the IPLDFetcher interface for ethereum
-// It interfaces directly with PG-IPFS
-type IPLDFetcher struct {
-	db *postgres.DB
-}
-
-// NewIPLDFetcher creates a pointer to a new IPLDFetcher
-func NewIPLDFetcher(db *postgres.DB) *IPLDFetcher {
-	return &IPLDFetcher{
-		db: db,
-	}
-}
-
-// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
-func (f *IPLDFetcher) Fetch(cids CIDWrapper) (*IPLDs, error) {
-	log.Debug("fetching iplds")
-	iplds := new(IPLDs)
-	var ok bool
-	iplds.TotalDifficulty, ok = new(big.Int).SetString(cids.Header.TotalDifficulty, 10)
-	if !ok {
-		return nil, errors.New("eth fetcher: unable to set total difficulty")
-	}
-	iplds.BlockNumber = cids.BlockNumber
-
-	tx, err := f.db.Beginx()
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		if p := recover(); p != nil {
-			shared.Rollback(tx)
-			panic(p)
-		} else if err != nil {
-			shared.Rollback(tx)
-		} else {
-			err = tx.Commit()
-		}
-	}()
-
-	iplds.Header, err = f.FetchHeader(tx, cids.Header)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: header fetching error: %s", err.Error())
-	}
-	iplds.Uncles, err = f.FetchUncles(tx, cids.Uncles)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: uncle fetching error: %s", err.Error())
-	}
-	iplds.Transactions, err = f.FetchTrxs(tx, cids.Transactions)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: transaction fetching error: %s", err.Error())
-	}
-	iplds.Receipts, err = f.FetchRcts(tx, cids.Receipts)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: receipt fetching error: %s", err.Error())
-	}
-	iplds.StateNodes, err = f.FetchState(tx, cids.StateNodes)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: state fetching error: %s", err.Error())
-	}
-	iplds.StorageNodes, err = f.FetchStorage(tx, cids.StorageNodes)
-	if err != nil {
-		return nil, fmt.Errorf("eth pg fetcher: storage fetching error: %s", err.Error())
-	}
-	return iplds, err
-}
-
-// FetchHeaders fetches headers
-func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c eth.HeaderModel) (ipfs.BlockModel, error) {
-	log.Debug("fetching header ipld")
-	headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-	if err != nil {
-		return ipfs.BlockModel{}, err
-	}
-	return ipfs.BlockModel{
-		Data: headerBytes,
-		CID:  c.CID,
-	}, nil
-}
-
-// FetchUncles fetches uncles
-func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []eth.UncleModel) ([]ipfs.BlockModel, error) {
-	log.Debug("fetching uncle iplds")
-	uncleIPLDs := make([]ipfs.BlockModel, len(cids))
-	for i, c := range cids {
-		uncleBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		uncleIPLDs[i] = ipfs.BlockModel{
-			Data: uncleBytes,
-			CID:  c.CID,
-		}
-	}
-	return uncleIPLDs, nil
-}
-
-// FetchTrxs fetches transactions
-func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []eth.TxModel) ([]ipfs.BlockModel, error) {
-	log.Debug("fetching transaction iplds")
-	trxIPLDs := make([]ipfs.BlockModel, len(cids))
-	for i, c := range cids {
-		txBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		trxIPLDs[i] = ipfs.BlockModel{
-			Data: txBytes,
-			CID:  c.CID,
-		}
-	}
-	return trxIPLDs, nil
-}
-
-// FetchRcts fetches receipts
-func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []eth.ReceiptModel) ([]ipfs.BlockModel, error) {
-	log.Debug("fetching receipt iplds")
-	rctIPLDs := make([]ipfs.BlockModel, len(cids))
-	for i, c := range cids {
-		rctBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		rctIPLDs[i] = ipfs.BlockModel{
-			Data: rctBytes,
-			CID:  c.CID,
-		}
-	}
-	return rctIPLDs, nil
-}
-
-// FetchState fetches state nodes
-func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]StateNode, error) {
-	log.Debug("fetching state iplds")
-	stateNodes := make([]StateNode, 0, len(cids))
-	for _, stateNode := range cids {
-		if stateNode.CID == "" {
-			continue
-		}
-		stateBytes, err := shared.FetchIPLDByMhKey(tx, stateNode.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		stateNodes = append(stateNodes, StateNode{
-			IPLD: ipfs.BlockModel{
-				Data: stateBytes,
-				CID:  stateNode.CID,
-			},
-			StateLeafKey: common.HexToHash(stateNode.StateKey),
-			Type:         ResolveToNodeType(stateNode.NodeType),
-			Path:         stateNode.Path,
-		})
-	}
-	return stateNodes, nil
-}
-
-// FetchStorage fetches storage nodes
-func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateKeyModel) ([]StorageNode, error) {
-	log.Debug("fetching storage iplds")
-	storageNodes := make([]StorageNode, 0, len(cids))
-	for _, storageNode := range cids {
-		if storageNode.CID == "" || storageNode.StateKey == "" {
-			continue
-		}
-		storageBytes, err := shared.FetchIPLDByMhKey(tx, storageNode.MhKey)
-		if err != nil {
-			return nil, err
-		}
-		storageNodes = append(storageNodes, StorageNode{
-			IPLD: ipfs.BlockModel{
-				Data: storageBytes,
-				CID:  storageNode.CID,
-			},
-			StateLeafKey:   common.HexToHash(storageNode.StateKey),
-			StorageLeafKey: common.HexToHash(storageNode.StorageKey),
-			Type:           ResolveToNodeType(storageNode.NodeType),
-			Path:           storageNode.Path,
-		})
-	}
-	return storageNodes, nil
-}

pkg/eth/ipld_fetcher_test.go

@@ -1,65 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth_test
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-var _ = Describe("IPLDFetcher", func() {
-	var (
-		db            *postgres.DB
-		pubAndIndexer *eth2.IPLDPublisher
-		fetcher       *eth.IPLDFetcher
-	)
-	Describe("Fetch", func() {
-		BeforeEach(func() {
-			var err error
-			db, err = shared.SetupDB()
-			Expect(err).ToNot(HaveOccurred())
-			pubAndIndexer = eth2.NewIPLDPublisher(db)
-			err = pubAndIndexer.Publish(test_helpers.MockConvertedPayload)
-			Expect(err).ToNot(HaveOccurred())
-			fetcher = eth.NewIPLDFetcher(db)
-		})
-		AfterEach(func() {
-			eth.TearDownDB(db)
-		})
-
-		It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {
-			iplds, err := fetcher.Fetch(*test_helpers.MockCIDWrapper)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(iplds).ToNot(BeNil())
-			Expect(iplds.TotalDifficulty).To(Equal(test_helpers.MockConvertedPayload.TotalDifficulty))
-			Expect(iplds.BlockNumber).To(Equal(test_helpers.MockConvertedPayload.Block.Number()))
-			Expect(iplds.Header).To(Equal(test_helpers.MockIPLDs.Header))
-			Expect(len(iplds.Uncles)).To(Equal(0))
-			Expect(iplds.Transactions).To(Equal(test_helpers.MockIPLDs.Transactions))
-			Expect(iplds.Receipts).To(Equal(test_helpers.MockIPLDs.Receipts))
-			Expect(iplds.StateNodes).To(Equal(test_helpers.MockIPLDs.StateNodes))
-			Expect(iplds.StorageNodes).To(Equal(test_helpers.MockIPLDs.StorageNodes))
-		})
-	})
-})

pkg/eth/ipld_retriever.go

@@ -1,467 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/lib/pq"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-)
-
-const (
-	RetrieveHeadersByHashesPgStr = `SELECT cid, data
-								FROM eth.header_cids 
-									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
-								WHERE block_hash = ANY($1::VARCHAR(66)[])`
-	RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data
-								FROM eth.header_cids 
-									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
-								WHERE block_number = $1`
-	RetrieveHeaderByHashPgStr = `SELECT cid, data
-								FROM eth.header_cids 
-									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
-								WHERE block_hash = $1`
-	RetrieveUnclesByHashesPgStr = `SELECT cid, data
-								FROM eth.uncle_cids
-									INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
-								WHERE block_hash = ANY($1::VARCHAR(66)[])`
-	RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data
-										FROM eth.uncle_cids
-											INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id)
-											INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
-										WHERE block_hash = $1`
-	RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data
-										FROM eth.uncle_cids
-											INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id)
-											INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
-										WHERE block_number = $1`
-	RetrieveUncleByHashPgStr = `SELECT cid, data
-								FROM eth.uncle_cids
-									INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
-								WHERE block_hash = $1`
-	RetrieveTransactionsByHashesPgStr = `SELECT cid, data
-									FROM eth.transaction_cids
-										INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
-									WHERE tx_hash = ANY($1::VARCHAR(66)[])`
-	RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data
-											FROM eth.transaction_cids
-												INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-												INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
-											WHERE block_hash = $1`
-	RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data
-											FROM eth.transaction_cids
-												INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-												INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
-											WHERE block_number = $1`
-	RetrieveTransactionByHashPgStr = `SELECT cid, data
-									FROM eth.transaction_cids
-										INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
-									WHERE tx_hash = $1`
-	RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data
-									FROM eth.receipt_cids
-										INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
-										INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
-									WHERE tx_hash = ANY($1::VARCHAR(66)[])`
-	RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data
-										FROM eth.receipt_cids
-											INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
-											INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-											INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
-										WHERE block_hash = $1`
-	RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data
-										FROM eth.receipt_cids
-											INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
-											INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
-											INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
-										WHERE block_number = $1`
-	RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data
-									FROM eth.receipt_cids
-										INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
-										INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
-									WHERE tx_hash = $1`
-	RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid,
-														data,
-														was_state_removed(state_path, block_number, $2) AS removed
-												FROM eth.state_cids
-													INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-													INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key)
-												WHERE state_leaf_key = $1
-												AND block_number <= (SELECT block_number
-																	FROM eth.header_cids
-																	WHERE block_hash = $2)
-												AND header_cids.id = (SELECT canonical_header_id(block_number))
-												ORDER BY block_number DESC
-												LIMIT 1`
-	RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid,
-														  data,
-														  was_state_removed(state_path, block_number, (SELECT block_hash
-																									  FROM eth.header_cids
-																									  WHERE block_number = $2
-																									  LIMIT 1)) AS removed
-													FROM eth.state_cids
-														INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-														INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key)
-													WHERE state_leaf_key = $1
-													AND block_number <= $2
-													ORDER BY block_number DESC
-													LIMIT 1`
-	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid,
-																			data,
-																			was_storage_removed(storage_path, block_number, (SELECT block_hash
-																															FROM eth.header_cids
-																															WHERE block_number = $3
-																															LIMIT 1)) AS removed
-																	FROM eth.storage_cids
-																		INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
-																		INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-																		INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key)
-																	WHERE state_leaf_key = $1
-																	AND storage_leaf_key = $2
-																	AND block_number <= $3
-																	ORDER BY block_number DESC
-																	LIMIT 1`
-	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid,
-																		  data,
-																		  was_storage_removed(storage_path, block_number, $3) AS removed
-																	FROM eth.storage_cids
-																		INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
-																		INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
-																		INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key)
-																	WHERE state_leaf_key = $1
-																	AND storage_leaf_key = $2
-																	AND block_number <= (SELECT block_number
-																						FROM eth.header_cids
-																						WHERE block_hash = $3)
-																	AND header_cids.id = (SELECT canonical_header_id(block_number))
-																	ORDER BY block_number DESC
-																	LIMIT 1`
-)
-
-type ipldResult struct {
-	CID  string `db:"cid"`
-	Data []byte `db:"data"`
-}
-type IPLDRetriever struct {
-	db *postgres.DB
-}
-
-func NewIPLDRetriever(db *postgres.DB) *IPLDRetriever {
-	return &IPLDRetriever{
-		db: db,
-	}
-}
-
-// RetrieveHeadersByHashes returns the cids and rlp bytes for the headers corresponding to the provided block hashes
-func (r *IPLDRetriever) RetrieveHeadersByHashes(hashes []common.Hash) ([]string, [][]byte, error) {
-	headerResults := make([]ipldResult, 0)
-	hashStrs := make([]string, len(hashes))
-	for i, hash := range hashes {
-		hashStrs[i] = hash.Hex()
-	}
-	if err := r.db.Select(&headerResults, RetrieveHeadersByHashesPgStr, pq.Array(hashStrs)); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(headerResults))
-	headers := make([][]byte, len(headerResults))
-	for i, res := range headerResults {
-		cids[i] = res.CID
-		headers[i] = res.Data
-	}
-	return cids, headers, nil
-}
-
-// RetrieveHeadersByBlockNumber returns the cids and rlp bytes for the headers corresponding to the provided block number
-// This can return more than one result since there can be more than one header (non-canonical headers)
-func (r *IPLDRetriever) RetrieveHeadersByBlockNumber(number uint64) ([]string, [][]byte, error) {
-	headerResults := make([]ipldResult, 0)
-	if err := r.db.Select(&headerResults, RetrieveHeadersByBlockNumberPgStr, number); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(headerResults))
-	headers := make([][]byte, len(headerResults))
-	for i, res := range headerResults {
-		cids[i] = res.CID
-		headers[i] = res.Data
-	}
-	return cids, headers, nil
-}
-
-// RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash
-func (r *IPLDRetriever) RetrieveHeaderByHash(hash common.Hash) (string, []byte, error) {
-	headerResult := new(ipldResult)
-	return headerResult.CID, headerResult.Data, r.db.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
-}
-
-// RetrieveUnclesByHashes returns the cids and rlp bytes for the uncles corresponding to the provided uncle hashes
-func (r *IPLDRetriever) RetrieveUnclesByHashes(hashes []common.Hash) ([]string, [][]byte, error) {
-	uncleResults := make([]ipldResult, 0)
-	hashStrs := make([]string, len(hashes))
-	for i, hash := range hashes {
-		hashStrs[i] = hash.Hex()
-	}
-	if err := r.db.Select(&uncleResults, RetrieveUnclesByHashesPgStr, pq.Array(hashStrs)); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(uncleResults))
-	uncles := make([][]byte, len(uncleResults))
-	for i, res := range uncleResults {
-		cids[i] = res.CID
-		uncles[i] = res.Data
-	}
-	return cids, uncles, nil
-}
-
-// RetrieveUnclesByBlockHash returns the cids and rlp bytes for the uncles corresponding to the provided block hash (of non-omner root block)
-func (r *IPLDRetriever) RetrieveUnclesByBlockHash(hash common.Hash) ([]string, [][]byte, error) {
-	uncleResults := make([]ipldResult, 0)
-	if err := r.db.Select(&uncleResults, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(uncleResults))
-	uncles := make([][]byte, len(uncleResults))
-	for i, res := range uncleResults {
-		cids[i] = res.CID
-		uncles[i] = res.Data
-	}
-	return cids, uncles, nil
-}
-
-// RetrieveUnclesByBlockNumber returns the cids and rlp bytes for the uncles corresponding to the provided block number (of non-omner root block)
-func (r *IPLDRetriever) RetrieveUnclesByBlockNumber(number uint64) ([]string, [][]byte, error) {
-	uncleResults := make([]ipldResult, 0)
-	if err := r.db.Select(&uncleResults, RetrieveUnclesByBlockNumberPgStr, number); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(uncleResults))
-	uncles := make([][]byte, len(uncleResults))
-	for i, res := range uncleResults {
-		cids[i] = res.CID
-		uncles[i] = res.Data
-	}
-	return cids, uncles, nil
-}
-
-// RetrieveUncleByHash returns the cid and rlp bytes for the uncle corresponding to the provided uncle hash
-func (r *IPLDRetriever) RetrieveUncleByHash(hash common.Hash) (string, []byte, error) {
-	uncleResult := new(ipldResult)
-	return uncleResult.CID, uncleResult.Data, r.db.Get(uncleResult, RetrieveUncleByHashPgStr, hash.Hex())
-}
-
-// RetrieveTransactionsByHashes returns the cids and rlp bytes for the transactions corresponding to the provided tx hashes
-func (r *IPLDRetriever) RetrieveTransactionsByHashes(hashes []common.Hash) ([]string, [][]byte, error) {
-	txResults := make([]ipldResult, 0)
-	hashStrs := make([]string, len(hashes))
-	for i, hash := range hashes {
-		hashStrs[i] = hash.Hex()
-	}
-	if err := r.db.Select(&txResults, RetrieveTransactionsByHashesPgStr, pq.Array(hashStrs)); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(txResults))
-	txs := make([][]byte, len(txResults))
-	for i, res := range txResults {
-		cids[i] = res.CID
-		txs[i] = res.Data
-	}
-	return cids, txs, nil
-}
-
-// RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash
-func (r *IPLDRetriever) RetrieveTransactionsByBlockHash(hash common.Hash) ([]string, [][]byte, error) {
-	txResults := make([]ipldResult, 0)
-	if err := r.db.Select(&txResults, RetrieveTransactionsByBlockHashPgStr, hash.Hex()); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(txResults))
-	txs := make([][]byte, len(txResults))
-	for i, res := range txResults {
-		cids[i] = res.CID
-		txs[i] = res.Data
-	}
-	return cids, txs, nil
-}
-
-// RetrieveTransactionsByBlockNumber returns the cids and rlp bytes for the transactions corresponding to the provided block number
-func (r *IPLDRetriever) RetrieveTransactionsByBlockNumber(number uint64) ([]string, [][]byte, error) {
-	txResults := make([]ipldResult, 0)
-	if err := r.db.Select(&txResults, RetrieveTransactionsByBlockNumberPgStr, number); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(txResults))
-	txs := make([][]byte, len(txResults))
-	for i, res := range txResults {
-		cids[i] = res.CID
-		txs[i] = res.Data
-	}
-	return cids, txs, nil
-}
-
-// RetrieveTransactionByTxHash returns the cid and rlp bytes for the transaction corresponding to the provided tx hash
-func (r *IPLDRetriever) RetrieveTransactionByTxHash(hash common.Hash) (string, []byte, error) {
-	txResult := new(ipldResult)
-	return txResult.CID, txResult.Data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex())
-}
-
-// RetrieveReceiptsByTxHashes returns the cids and rlp bytes for the receipts corresponding to the provided tx hashes
-func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]string, [][]byte, error) {
-	rctResults := make([]ipldResult, 0)
-	hashStrs := make([]string, len(hashes))
-	for i, hash := range hashes {
-		hashStrs[i] = hash.Hex()
-	}
-	if err := r.db.Select(&rctResults, RetrieveReceiptsByTxHashesPgStr, pq.Array(hashStrs)); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(rctResults))
-	rcts := make([][]byte, len(rctResults))
-	for i, res := range rctResults {
-		cids[i] = res.CID
-		rcts[i] = res.Data
-	}
-	return cids, rcts, nil
-}
-
-// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash
-func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, [][]byte, error) {
-	rctResults := make([]ipldResult, 0)
-	if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(rctResults))
-	rcts := make([][]byte, len(rctResults))
-	for i, res := range rctResults {
-		cids[i] = res.CID
-		rcts[i] = res.Data
-	}
-	return cids, rcts, nil
-}
-
-// RetrieveReceiptsByBlockNumber returns the cids and rlp bytes for the receipts corresponding to the provided block hash
-func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string, [][]byte, error) {
-	rctResults := make([]ipldResult, 0)
-	if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockNumberPgStr, number); err != nil {
-		return nil, nil, err
-	}
-	cids := make([]string, len(rctResults))
-	rcts := make([][]byte, len(rctResults))
-	for i, res := range rctResults {
-		cids[i] = res.CID
-		rcts[i] = res.Data
-	}
-	return cids, rcts, nil
-}
-
-// RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash
-func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, error) {
-	rctResult := new(ipldResult)
-	return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex())
-}
-
-type nodeInfo struct {
-	CID     string `db:"cid"`
-	Data    []byte `db:"data"`
-	Removed bool   `db:"removed"`
-}
-
-// RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
-// TODO: ensure this handles deleted accounts appropriately
-func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) {
-	accountResult := new(nodeInfo)
-	leafKey := crypto.Keccak256Hash(address.Bytes())
-	if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil {
-		return "", nil, err
-	}
-	if accountResult.Removed {
-		return "", []byte{}, nil
-	}
-	var i []interface{}
-	if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
-		return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
-	}
-	if len(i) != 2 {
-		return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements")
-	}
-	return accountResult.CID, i[1].([]byte), nil
-}
-
-// RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number
-// This can return a non-canonical account
-func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) (string, []byte, error) {
-	accountResult := new(nodeInfo)
-	leafKey := crypto.Keccak256Hash(address.Bytes())
-	if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil {
-		return "", nil, err
-	}
-	if accountResult.Removed {
-		return "", []byte{}, nil
-	}
-	var i []interface{}
-	if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
-		return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
-	}
-	if len(i) != 2 {
-		return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements")
-	}
-	return accountResult.CID, i[1].([]byte), nil
-}
-
-// RetrieveStorageAtByAddressAndStorageKeyAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block hash
-func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(address common.Address, storageLeafKey, hash common.Hash) (string, []byte, error) {
-	storageResult := new(nodeInfo)
-	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
-	if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), hash.Hex()); err != nil {
-		return "", nil, err
-	}
-	if storageResult.Removed {
-		return "", []byte{}, nil
-	}
-	var i []interface{}
-	if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {
-		err = fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())
-		return "", nil, err
-	}
-	if len(i) != 2 {
-		return "", nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements")
-	}
-	return storageResult.CID, i[1].([]byte), nil
-}
-
-// RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block number
-// This can retrun a non-canonical value
-func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(address common.Address, storageLeafKey common.Hash, number uint64) (string, []byte, error) {
-	storageResult := new(nodeInfo)
-	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
-	if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), number); err != nil {
-		return "", nil, err
-	}
-	if storageResult.Removed {
-		return "", []byte{}, nil
-	}
-	var i []interface{}
-	if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {
-		return "", nil, fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())
-	}
-	if len(i) != 2 {
-		return "", nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements")
-	}
-	return storageResult.CID, i[1].([]byte), nil
-}

pkg/eth/node_types.go

@@ -0,0 +1,102 @@
+package eth
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/rlp"
+
+	"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
+)
+
+// NodeType for explicitly setting type of node
+type NodeType string
+
+const (
+	Unknown   NodeType = "Unknown"
+	Branch    NodeType = "Branch"
+	Extension NodeType = "Extension"
+	Leaf      NodeType = "Leaf"
+	Removed   NodeType = "Removed" // used to represent paths which have been emptied
+)
+
+func (n NodeType) Int() int {
+	switch n {
+	case Branch:
+		return 0
+	case Extension:
+		return 1
+	case Leaf:
+		return 2
+	case Removed:
+		return 3
+	default:
+		return -1
+	}
+}
+
+// CheckKeyType checks what type of key we have
+func CheckKeyType(elements []interface{}) (NodeType, error) {
+	if len(elements) > 2 {
+		return Branch, nil
+	}
+	if len(elements) < 2 {
+		return Unknown, fmt.Errorf("node cannot be less than two elements in length")
+	}
+	switch elements[0].([]byte)[0] / 16 {
+	case '\x00':
+		return Extension, nil
+	case '\x01':
+		return Extension, nil
+	case '\x02':
+		return Leaf, nil
+	case '\x03':
+		return Leaf, nil
+	default:
+		return Unknown, fmt.Errorf("unknown hex prefix")
+	}
+}
+
+// StateNode holds the data for a single state diff node
+type StateNode struct {
+	NodeType     NodeType      `json:"nodeType"        gencodec:"required"`
+	Path         []byte        `json:"path"            gencodec:"required"`
+	NodeValue    []byte        `json:"value"           gencodec:"required"`
+	StorageNodes []StorageNode `json:"storage"`
+	LeafKey      []byte        `json:"leafKey"`
+}
+
+// StorageNode holds the data for a single storage diff node
+type StorageNode struct {
+	NodeType  NodeType `json:"nodeType"        gencodec:"required"`
+	Path      []byte   `json:"path"            gencodec:"required"`
+	NodeValue []byte   `json:"value"           gencodec:"required"`
+	LeafKey   []byte   `json:"leafKey"`
+}
+
+func ResolveNode(path []byte, node []byte, trieDB *trie.Database) (StateNode, []interface{}, error) {
+	var nodeElements []interface{}
+	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return StateNode{}, nil, err
+	}
+	ty, err := CheckKeyType(nodeElements)
+	if err != nil {
+		return StateNode{}, nil, err
+	}
+
+	nodePath := make([]byte, len(path))
+	copy(nodePath, path)
+	return StateNode{
+		NodeType:  ty,
+		Path:      nodePath,
+		NodeValue: node,
+	}, nodeElements, nil
+}
+
+// ResolveNodeIt return the state diff node pointed by the iterator.
+func ResolveNodeIt(it trie.NodeIterator, trieDB *trie.Database) (StateNode, []interface{}, error) {
+	node, err := it.NodeBlob(), it.Error()
+	if err != nil {
+		return StateNode{}, nil, err
+	}
+	return ResolveNode(it.Path(), node, trieDB)
+}

pkg/eth/retriever.go

@@ -0,0 +1,555 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"bytes"
+	"fmt"
+	"math/big"
+	"strconv"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+)
+
+// Retriever is used for fetching
+type Retriever struct {
+	db     *sqlx.DB
+	gormDB *gorm.DB
+}
+
+type IPLDModelRecord struct {
+	models.IPLDModel
+}
+
+// TableName overrides the table name used by IPLD
+func (IPLDModelRecord) TableName() string {
+	return "ipld.blocks"
+}
+
+type HeaderCIDRecord struct {
+	CID             string `gorm:"column:cid"`
+	BlockHash       string `gorm:"primaryKey"`
+	BlockNumber     string `gorm:"primaryKey"`
+	ParentHash      string
+	Timestamp       uint64
+	StateRoot       string
+	TotalDifficulty string `gorm:"column:td"`
+	TxRoot          string
+	RctRoot         string `gorm:"column:receipt_root"`
+	UnclesHash      string
+	Bloom           []byte
+
+	// gorm doesn't check if foreign key exists in database.
+	// It is required to eager load relations using preload.
+	TransactionCIDs []TransactionCIDRecord `gorm:"foreignKey:HeaderID,BlockNumber;references:BlockHash,BlockNumber"`
+	IPLD            IPLDModelRecord        `gorm:"foreignKey:CID,BlockNumber;references:Key,BlockNumber"`
+}
+
+// TableName overrides the table name used by HeaderCIDRecord
+func (HeaderCIDRecord) TableName() string {
+	return "eth.header_cids"
+}
+
+type TransactionCIDRecord struct {
+	CID         string `gorm:"column:cid"`
+	TxHash      string `gorm:"primaryKey"`
+	BlockNumber string `gorm:"primaryKey"`
+	HeaderID    string `gorm:"column:header_id"`
+	Index       int64
+	Src         string
+	Dst         string
+	IPLD        IPLDModelRecord `gorm:"foreignKey:CID,BlockNumber;references:Key,BlockNumber"`
+}
+
+type StateAccountRecord struct {
+	Nonce    uint64 `db:"nonce"`
+	Balance  string `db:"balance"`
+	Root     string `db:"storage_root"`
+	CodeHash []byte `db:"code_hash"`
+	Removed  bool   `db:"removed"`
+}
+
+// TableName overrides the table name used by TransactionCIDRecord
+func (TransactionCIDRecord) TableName() string {
+	return "eth.transaction_cids"
+}
+
+// NewRetriever returns a pointer to a new Retriever which supports the Retriever interface
+func NewRetriever(db *sqlx.DB) *Retriever {
+	gormDB, err := gorm.Open(postgres.New(postgres.Config{
+		Conn: db,
+	}), &gorm.Config{})
+
+	if err != nil {
+		log.Error(err)
+		return nil
+	}
+
+	return &Retriever{
+		db:     db,
+		gormDB: gormDB,
+	}
+}
+
+// RetrieveFirstBlockNumber is used to retrieve the first block number in the db
+func (r *Retriever) RetrieveFirstBlockNumber() (int64, error) {
+	var blockNumber int64
+	err := r.db.Get(&blockNumber, "SELECT block_number FROM ipld.blocks ORDER BY block_number ASC LIMIT 1")
+	return blockNumber, err
+}
+
+// RetrieveLastBlockNumber is used to retrieve the latest block number in the db
+func (r *Retriever) RetrieveLastBlockNumber() (int64, error) {
+	var blockNumber int64
+	err := r.db.Get(&blockNumber, "SELECT block_number FROM ipld.blocks ORDER BY block_number DESC LIMIT 1")
+	return blockNumber, err
+}
+
+func topicFilterCondition(id *int, topics [][]string, args []interface{}, pgStr string, first bool) (string, []interface{}) {
+	for i, topicSet := range topics {
+		if len(topicSet) == 0 {
+			continue
+		}
+
+		if !first {
+			pgStr += " AND"
+		} else {
+			first = false
+		}
+		pgStr += fmt.Sprintf(` eth.log_cids.topic%d = ANY ($%d)`, i, *id)
+		args = append(args, pq.Array(topicSet))
+		*id++
+	}
+	return pgStr, args
+}
+
+func logFilterCondition(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter) (string, []interface{}) {
+	if len(rctFilter.LogAddresses) > 0 {
+		pgStr += fmt.Sprintf(` AND eth.log_cids.address = ANY ($%d)`, *id)
+		args = append(args, pq.Array(rctFilter.LogAddresses))
+		*id++
+	}
+
+	// Filter on topics if there are any
+	if hasTopics(rctFilter.Topics) {
+		pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
+	}
+
+	return pgStr, args
+}
+
+func receiptFilterConditions(id *int, pgStr string, args []interface{}, rctFilter ReceiptFilter, txHashes []string) (string, []interface{}) {
+	rctCond := " AND (receipt_cids.tx_id = ANY ( "
+	logQuery := "SELECT rct_id FROM eth.log_cids WHERE"
+	if len(rctFilter.LogAddresses) > 0 {
+		// Filter on log contract addresses if there are any
+		pgStr += fmt.Sprintf(`%s %s eth.log_cids.address = ANY ($%d)`, rctCond, logQuery, *id)
+		args = append(args, pq.Array(rctFilter.LogAddresses))
+		*id++
+
+		// Filter on topics if there are any
+		if hasTopics(rctFilter.Topics) {
+			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, false)
+		}
+
+		pgStr += ")"
+
+		// Filter on txHashes if there are any, and we are matching txs
+		if rctFilter.MatchTxs && len(txHashes) > 0 {
+			pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
+			args = append(args, pq.Array(txHashes))
+		}
+		pgStr += ")"
+	} else { // If there are no contract addresses to filter on
+		// Filter on topics if there are any
+		if hasTopics(rctFilter.Topics) {
+			pgStr += rctCond + logQuery
+			pgStr, args = topicFilterCondition(id, rctFilter.Topics, args, pgStr, true)
+			pgStr += ")"
+			// Filter on txHashes if there are any, and we are matching txs
+			if rctFilter.MatchTxs && len(txHashes) > 0 {
+				pgStr += fmt.Sprintf(` OR receipt_cids.tx_id = ANY($%d)`, *id)
+				args = append(args, pq.Array(txHashes))
+			}
+			pgStr += ")"
+		} else if rctFilter.MatchTxs && len(txHashes) > 0 {
+			// If there are no contract addresses or topics to filter on,
+			// Filter on txHashes if there are any, and we are matching txs
+			pgStr += fmt.Sprintf(` AND receipt_cids.tx_id = ANY($%d)`, *id)
+			args = append(args, pq.Array(txHashes))
+		}
+	}
+
+	return pgStr, args
+}
+
+// RetrieveFilteredGQLLogs retrieves and returns all the log CIDs provided blockHash that conform to the provided
+// filter parameters.
+func (r *Retriever) RetrieveFilteredGQLLogs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockHash *common.Hash, blockNumber *big.Int) ([]LogResult, error) {
+	log.Debug("retrieving log cids for receipt ids with block hash", blockHash.String())
+	args := make([]interface{}, 0, 4)
+	id := 1
+	pgStr := RetrieveFilteredGQLLogs
+	args = append(args, blockHash.String())
+	id++
+
+	if blockNumber != nil {
+		pgStr += ` AND receipt_cids.block_number = $2`
+		id++
+		args = append(args, blockNumber.Int64())
+	}
+
+	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
+	pgStr += ` ORDER BY log_cids.block_number, log_cids.index`
+
+	logs := make([]LogResult, 0)
+	err := tx.Select(&logs, pgStr, args...)
+	if err != nil {
+		return nil, err
+	}
+
+	return logs, nil
+}
+
+// RetrieveFilteredLogsForBlock retrieves and returns all the log CIDs for the block that conform to the provided filter parameters.
+func (r *Retriever) RetrieveFilteredLogsForBlock(db *sqlx.DB, rctFilter ReceiptFilter, blockHash *common.Hash) ([]LogResult, error) {
+	return r.retrieveFilteredLogs(db, rctFilter, -1, -1, blockHash)
+}
+
+// RetrieveFilteredLogsForBlockRange retrieves and returns all the log CIDs for the blocks in the range that conform
+// to the provided filter parameters.
+func (r *Retriever) RetrieveFilteredLogsForBlockRange(db *sqlx.DB, rctFilter ReceiptFilter, startBlockNumber int64, stopBlockNumber int64) ([]LogResult, error) {
+	return r.retrieveFilteredLogs(db, rctFilter, startBlockNumber, stopBlockNumber, nil)
+}
+
+// retrieveFilteredLogs retrieves all the log CIDs either for a single block (by hash) or range of blocks (by number) which
+// conform to the provided filter parameters.
+func (r *Retriever) retrieveFilteredLogs(db *sqlx.DB, rctFilter ReceiptFilter, startBlockNumber int64, stopBlockNumber int64, blockHash *common.Hash) ([]LogResult, error) {
+	log.Debug("retrieving log cids for receipt ids")
+	args := make([]interface{}, 0, 4)
+	var pgStr string
+	id := 1
+	if blockHash != nil {
+		pgStr = RetrieveFilteredLogsSingle
+		args = append(args, blockHash.String())
+		id++
+	} else {
+		pgStr = RetrieveFilteredLogsRange
+		args = append(args, startBlockNumber)
+		id++
+		args = append(args, stopBlockNumber)
+		id++
+	}
+
+	pgStr, args = logFilterCondition(&id, pgStr, args, rctFilter)
+	pgStr += ` ORDER BY eth.log_cids.block_number, eth.log_cids.index`
+
+	logs := make([]LogResult, 0)
+	err := db.Select(&logs, pgStr, args...)
+	if err != nil {
+		return nil, err
+	}
+	// decode logs and extract original contract Data
+	for i, log := range logs {
+		var buf []interface{}
+		r := bytes.NewReader(log.LogLeafData)
+		if err := rlp.Decode(r, &buf); err != nil {
+			return nil, err
+		}
+		logs[i].Data = buf[2].([]byte)
+	}
+
+	return logs, nil
+}
+
+func hasTopics(topics [][]string) bool {
+	for _, topicSet := range topics {
+		if len(topicSet) > 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// RetrieveBlockNumberByHash returns the block number for the given block hash
+func (r *Retriever) RetrieveBlockNumberByHash(tx *sqlx.Tx, blockHash common.Hash) (uint64, error) {
+	log.Debug("retrieving block number for block hash ", blockHash.String())
+	pgStr := `SELECT CAST(block_number as TEXT) FROM eth.header_cids WHERE block_hash = $1`
+	var blockNumberStr string
+	if err := tx.Get(&blockNumberStr, pgStr, blockHash.String()); err != nil {
+		return 0, err
+	}
+	return strconv.ParseUint(blockNumberStr, 10, 64)
+}
+
+// RetrieveHeaderAndTxCIDsByBlockNumber retrieves header CIDs and their associated tx CIDs by block number
+func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockNumber(blockNumber int64) ([]HeaderCIDRecord, error) {
+	log.Debug("retrieving header cids and tx cids for block number ", blockNumber)
+
+	var headerCIDs []HeaderCIDRecord
+
+	// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
+	// Will use join for TransactionCIDs once preload for 1:N is supported.
+	err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
+		return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
+	}).Joins("IPLD").Find(&headerCIDs, "header_cids.block_number = ?", blockNumber).Error
+
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return nil, err
+	}
+
+	return headerCIDs, nil
+}
+
+// RetrieveHeaderAndTxCIDsByBlockHash retrieves header CID and their associated tx CIDs by block hash (and optionally block number)
+func (r *Retriever) RetrieveHeaderAndTxCIDsByBlockHash(blockHash common.Hash, blockNumber *big.Int) (HeaderCIDRecord, error) {
+	log.Debug("retrieving header cid and tx cids for block hash ", blockHash.String())
+
+	var headerCIDs []HeaderCIDRecord
+
+	conditions := map[string]interface{}{"block_hash": blockHash.String()}
+	if blockNumber != nil {
+		conditions["header_cids.block_number"] = blockNumber.Int64()
+	}
+
+	// https://github.com/go-gorm/gorm/issues/4083#issuecomment-778883283
+	// Will use join for TransactionCIDs once preload for 1:N is supported.
+	err := r.gormDB.Preload("TransactionCIDs", func(tx *gorm.DB) *gorm.DB {
+		return tx.Select("cid", "tx_hash", "index", "src", "dst", "header_id", "block_number")
+	}).Joins("IPLD").Find(&headerCIDs, conditions).Error
+
+	if err != nil {
+		log.Error("header cid retrieval error")
+		return HeaderCIDRecord{}, err
+	}
+
+	if len(headerCIDs) == 0 {
+		return HeaderCIDRecord{}, errHeaderHashNotFound
+	} else if len(headerCIDs) > 1 {
+		return HeaderCIDRecord{}, errMultipleHeadersForHash
+	}
+
+	return headerCIDs[0], nil
+}
+
+// RetrieveTxCIDByHash returns the tx for the given tx hash (and optionally block number)
+func (r *Retriever) RetrieveTxCIDByHash(txHash string, blockNumber *big.Int) (TransactionCIDRecord, error) {
+	log.Debug("retrieving tx cid for tx hash ", txHash)
+
+	var txCIDs []TransactionCIDRecord
+
+	var err error
+	if blockNumber != nil {
+		err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number)) AND transaction_cids.block_number = ?", txHash, blockNumber.Int64()).Error
+	} else {
+		err = r.gormDB.Joins("IPLD").Find(&txCIDs, "tx_hash = ? AND transaction_cids.header_id = (SELECT canonical_header_hash(transaction_cids.block_number))", txHash).Error
+	}
+	if err != nil {
+		log.Error("tx retrieval error")
+		return TransactionCIDRecord{}, err
+	}
+
+	if len(txCIDs) == 0 {
+		return TransactionCIDRecord{}, errTxHashNotFound
+	} else if len(txCIDs) > 1 {
+		// a transaction can be part of a only one canonical block
+		return TransactionCIDRecord{}, errTxHashInMultipleBlocks
+	}
+
+	return txCIDs[0], nil
+}
+
+var EmptyNodeValue = make([]byte, common.HashLength)
+
+// RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash
+func (r *Retriever) RetrieveHeaderByHash(hash common.Hash) (string, []byte, error) {
+	headerResult := new(ipldResult)
+	return headerResult.CID, headerResult.Data, r.db.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
+}
+
+// RetrieveHeaderByHash2 returns the cid and rlp bytes for the header corresponding to the provided block hash
+// using a sqlx.Tx
+func (r *Retriever) RetrieveHeaderByHash2(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) {
+	headerResult := new(ipldResult)
+	return headerResult.CID, headerResult.Data, tx.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
+}
+
+// RetrieveUncles returns the cid and rlp bytes for the uncle list corresponding to the provided block hash, number (of non-omner root block)
+func (r *Retriever) RetrieveUncles(tx *sqlx.Tx, hash common.Hash, number uint64) (string, []byte, error) {
+	uncleResult := new(ipldResult)
+	if err := tx.Get(uncleResult, RetrieveUnclesPgStr, hash.Hex(), number); err != nil {
+		return "", nil, err
+	}
+	return uncleResult.CID, uncleResult.Data, nil
+}
+
+// RetrieveUnclesByBlockHash returns the cid and rlp bytes for the uncle list corresponding to the provided block hash (of non-omner root block)
+func (r *Retriever) RetrieveUnclesByBlockHash(tx *sqlx.Tx, hash common.Hash) (string, []byte, error) {
+	uncleResult := new(ipldResult)
+	if err := tx.Get(uncleResult, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil {
+		return "", nil, err
+	}
+	return uncleResult.CID, uncleResult.Data, nil
+}
+
+// RetrieveTransactions returns the cids and rlp bytes for the transactions corresponding to the provided block hash, number
+func (r *Retriever) RetrieveTransactions(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, error) {
+	txResults := make([]ipldResult, 0)
+	if err := tx.Select(&txResults, RetrieveTransactionsPgStr, hash.Hex(), number); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(txResults))
+	txs := make([][]byte, len(txResults))
+	for i, res := range txResults {
+		cids[i] = res.CID
+		txs[i] = res.Data
+	}
+	return cids, txs, nil
+}
+
+// RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash
+func (r *Retriever) RetrieveTransactionsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, error) {
+	txResults := make([]ipldResult, 0)
+	if err := tx.Select(&txResults, RetrieveTransactionsByBlockHashPgStr, hash.Hex()); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(txResults))
+	txs := make([][]byte, len(txResults))
+	for i, res := range txResults {
+		cids[i] = res.CID
+		txs[i] = res.Data
+	}
+	return cids, txs, nil
+}
+
+// DecodeLeafNode decodes the leaf node data
+func DecodeLeafNode(node []byte) ([]byte, error) {
+	var nodeElements []interface{}
+	if err := rlp.DecodeBytes(node, &nodeElements); err != nil {
+		return nil, err
+	}
+	ok, err := IsLeaf(nodeElements)
+	if err != nil {
+		return nil, err
+	}
+	if !ok {
+		return nil, fmt.Errorf("expected leaf node but found %v", nodeElements)
+	}
+	return nodeElements[1].([]byte), nil
+}
+
+// RetrieveReceipts returns the cids and rlp bytes for the receipts corresponding to the provided block hash, number.
+// cid returned corresponds to the leaf node data which contains the receipt.
+func (r *Retriever) RetrieveReceipts(tx *sqlx.Tx, hash common.Hash, number uint64) ([]string, [][]byte, []common.Hash, error) {
+	rctResults := make([]ipldResult, 0)
+	if err := tx.Select(&rctResults, RetrieveReceiptsPgStr, hash.Hex(), number); err != nil {
+		return nil, nil, nil, err
+	}
+	cids := make([]string, len(rctResults))
+	rcts := make([][]byte, len(rctResults))
+	txs := make([]common.Hash, len(rctResults))
+
+	for i, res := range rctResults {
+		cids[i] = res.CID
+		rcts[i] = res.Data
+		txs[i] = common.HexToHash(res.TxHash)
+	}
+
+	return cids, rcts, txs, nil
+}
+
+// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash.
+// cid returned corresponds to the leaf node data which contains the receipt.
+func (r *Retriever) RetrieveReceiptsByBlockHash(tx *sqlx.Tx, hash common.Hash) ([]string, [][]byte, []common.Hash, error) {
+	rctResults := make([]ipldResult, 0)
+	if err := tx.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
+		return nil, nil, nil, err
+	}
+	cids := make([]string, len(rctResults))
+	rcts := make([][]byte, len(rctResults))
+	txs := make([]common.Hash, len(rctResults))
+
+	for i, res := range rctResults {
+		cids[i] = res.CID
+		nodeVal, err := DecodeLeafNode(res.Data)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		rcts[i] = nodeVal
+		txs[i] = common.HexToHash(res.TxHash)
+	}
+
+	return cids, rcts, txs, nil
+}
+
+// RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
+// TODO: ensure this handles deleted accounts appropriately
+func (r *Retriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (StateAccountRecord, error) {
+	var accountResult StateAccountRecord
+	leafKey := crypto.Keccak256Hash(address.Bytes())
+	if err := r.db.Get(&accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil {
+		return StateAccountRecord{}, err
+	}
+
+	if accountResult.Removed {
+		return StateAccountRecord{}, nil
+	}
+	return accountResult, nil
+}
+
+// RetrieveStorageAtByAddressAndStorageSlotAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage slot, and block hash
+func (r *Retriever) RetrieveStorageAtByAddressAndStorageSlotAndBlockHash(address common.Address, key, hash common.Hash) ([]byte, error) {
+	var storageResult nodeInfo
+	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
+	storageHash := crypto.Keccak256Hash(key.Bytes())
+	if err := r.db.Get(&storageResult,
+		RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr,
+		stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil {
+		return nil, err
+	}
+	if storageResult.StateLeafRemoved || storageResult.Removed {
+		return EmptyNodeValue, nil
+	}
+	return storageResult.Value, nil
+}
+
+// RetrieveStorageAndRLP returns the cid and rlp bytes for the storage value corresponding to the
+// provided address, storage slot, and block hash
+func (r *Retriever) RetrieveStorageAndRLP(address common.Address, key, hash common.Hash) (string, []byte, error) {
+	var storageResult nodeInfo
+	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
+	storageHash := crypto.Keccak256Hash(key.Bytes())
+	if err := r.db.Get(&storageResult,
+		RetrieveStorageAndRLPByAddressHashAndLeafKeyAndBlockHashPgStr,
+		stateLeafKey.Hex(), storageHash.Hex(), hash.Hex()); err != nil {
+		return "", nil, err
+	}
+	if storageResult.StateLeafRemoved || storageResult.Removed {
+		return "", EmptyNodeValue, nil
+	}
+	return storageResult.CID, storageResult.Data, nil
+}

pkg/eth/retriever_test.go

@@ -0,0 +1,173 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
+)
+
+var _ = Describe("Retriever", func() {
+	var (
+		db          *sqlx.DB
+		diffIndexer interfaces.StateDiffIndexer
+		retriever   *eth.Retriever
+		ctx         = context.Background()
+	)
+	BeforeEach(func() {
+		db = shared.SetupDB()
+		diffIndexer = shared.SetupTestStateDiffIndexer(ctx, params.TestChainConfig, test_helpers.Genesis.Hash())
+
+		retriever = eth.NewRetriever(db)
+	})
+	AfterEach(func() {
+		shared.TearDownDB(db)
+		db.Close()
+	})
+
+	It("Retrieve", func() {
+		tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+		Expect(err).ToNot(HaveOccurred())
+		for _, node := range test_helpers.MockStateNodes {
+			err = diffIndexer.PushStateNode(tx, node, test_helpers.MockBlock.Hash().String())
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		err = tx.Submit(err)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Describe("RetrieveFirstBlockNumber", func() {
+		It("Throws an error if there are no blocks in the database", func() {
+			_, err := retriever.RetrieveFirstBlockNumber()
+			Expect(err).To(HaveOccurred())
+		})
+		It("Gets the number of the first block that has data in the database", func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveFirstBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1)))
+		})
+
+		It("Gets the number of the first block that has data in the database", func() {
+			payload := test_helpers.MockConvertedPayload
+			payload.Block = newMockBlock(1010101)
+			tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveFirstBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1010101)))
+		})
+
+		It("Gets the number of the first block that has data in the database", func() {
+			payload1 := test_helpers.MockConvertedPayload
+			payload1.Block = newMockBlock(1010101)
+			payload2 := payload1
+			payload2.Block = newMockBlock(5)
+			tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
+			tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveFirstBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(5)))
+		})
+	})
+
+	Describe("RetrieveLastBlockNumber", func() {
+		It("Throws an error if there are no blocks in the database", func() {
+			_, err := retriever.RetrieveLastBlockNumber()
+			Expect(err).To(HaveOccurred())
+		})
+		It("Gets the number of the latest block that has data in the database", func() {
+			tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveLastBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1)))
+		})
+
+		It("Gets the number of the latest block that has data in the database", func() {
+			payload := test_helpers.MockConvertedPayload
+			payload.Block = newMockBlock(1010101)
+			tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveLastBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1010101)))
+		})
+
+		It("Gets the number of the latest block that has data in the database", func() {
+			payload1 := test_helpers.MockConvertedPayload
+			payload1.Block = newMockBlock(1010101)
+			payload2 := payload1
+			payload2.Block = newMockBlock(5)
+			tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
+			tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty())
+			Expect(err).ToNot(HaveOccurred())
+			err = tx.Submit(err)
+			Expect(err).ToNot(HaveOccurred())
+
+			num, err := retriever.RetrieveLastBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1010101)))
+		})
+	})
+})
+
+func newMockBlock(blockNumber uint64) *types.Block {
+	header := test_helpers.MockHeader
+	header.Number.SetUint64(blockNumber)
+	return types.NewBlock(&test_helpers.MockHeader, test_helpers.MockTransactions, nil, test_helpers.MockReceipts, trie.NewEmpty(nil))
+}

pkg/eth/sql.go

@@ -0,0 +1,277 @@
+package eth
+
+const (
+	RetrieveHeaderByHashPgStr = `
+SELECT header_cids.cid,
+	blocks.data
+FROM ipld.blocks,
+	eth.header_cids
+WHERE header_cids.block_hash = $1
+	AND header_cids.canonical
+	AND blocks.key = header_cids.cid
+	AND blocks.block_number = header_cids.block_number
+LIMIT 1
+`
+	RetrieveUnclesPgStr = `SELECT uncle_cids.cid, data
+										FROM eth.uncle_cids
+											INNER JOIN eth.header_cids ON (
+												uncle_cids.header_id = header_cids.block_hash
+												AND uncle_cids.block_number = header_cids.block_number
+											)
+											INNER JOIN ipld.blocks ON (
+												uncle_cids.cid = blocks.key
+												AND uncle_cids.block_number = blocks.block_number
+											)
+										WHERE header_cids.block_hash = $1
+										AND header_cids.block_number = $2
+										ORDER BY uncle_cids.parent_hash
+										LIMIT 1`
+	RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data
+										FROM eth.uncle_cids
+											INNER JOIN eth.header_cids ON (
+												uncle_cids.header_id = header_cids.block_hash
+												AND uncle_cids.block_number = header_cids.block_number
+											)
+											INNER JOIN ipld.blocks ON (
+												uncle_cids.cid = blocks.key
+												AND uncle_cids.block_number = blocks.block_number
+											)
+										WHERE header_cids.block_hash = $1
+										ORDER BY uncle_cids.parent_hash
+										LIMIT 1`
+	RetrieveTransactionsPgStr = `
+SELECT transaction_cids.cid,
+	blocks.data
+FROM eth.transaction_cids,
+	eth.header_cids,
+	ipld.blocks
+WHERE header_cids.block_hash = $1
+    AND header_cids.block_number = $2
+	AND header_cids.canonical
+	AND transaction_cids.block_number = header_cids.block_number
+	AND transaction_cids.header_id = header_cids.block_hash
+	AND blocks.block_number = header_cids.block_number
+	AND blocks.key = transaction_cids.cid
+ORDER BY eth.transaction_cids.index ASC
+`
+	RetrieveTransactionsByBlockHashPgStr = `
+SELECT transaction_cids.cid,
+	blocks.data
+FROM eth.transaction_cids,
+	eth.header_cids,
+	ipld.blocks
+WHERE header_cids.block_hash = $1
+	AND header_cids.canonical
+	AND transaction_cids.block_number = header_cids.block_number
+	AND transaction_cids.header_id = header_cids.block_hash
+	AND blocks.block_number = header_cids.block_number
+	AND blocks.key = transaction_cids.cid
+ORDER BY eth.transaction_cids.index ASC
+`
+	RetrieveReceiptsPgStr = `
+SELECT receipt_cids.cid,
+	blocks.data,
+	eth.transaction_cids.tx_hash
+FROM eth.receipt_cids,
+	eth.transaction_cids,
+	eth.header_cids,
+	ipld.blocks
+WHERE header_cids.block_hash = $1
+	AND header_cids.block_number = $2
+	AND header_cids.canonical
+	AND receipt_cids.block_number = header_cids.block_number
+	AND receipt_cids.header_id = header_cids.block_hash
+	AND receipt_cids.TX_ID = transaction_cids.TX_HASH
+	AND transaction_cids.block_number = header_cids.block_number
+	AND transaction_cids.header_id = header_cids.block_hash
+	AND blocks.block_number = header_cids.block_number
+	AND blocks.key = receipt_cids.cid
+ORDER BY eth.transaction_cids.index ASC
+`
+	RetrieveReceiptsByBlockHashPgStr = `
+SELECT receipt_cids.cid,
+	blocks.data,
+	eth.transaction_cids.tx_hash
+FROM eth.receipt_cids,
+	eth.transaction_cids,
+	eth.header_cids,
+	ipld.blocks
+WHERE header_cids.block_hash = $1
+	AND header_cids.canonical
+	AND receipt_cids.block_number = header_cids.block_number
+	AND receipt_cids.header_id = header_cids.block_hash
+	AND receipt_cids.TX_ID = transaction_cids.TX_HASH
+	AND transaction_cids.block_number = header_cids.block_number
+	AND transaction_cids.header_id = header_cids.block_hash
+	AND blocks.block_number = header_cids.block_number
+	AND blocks.key = receipt_cids.cid
+ORDER BY eth.transaction_cids.index ASC
+`
+	RetrieveAccountByLeafKeyAndBlockHashPgStr = `
+SELECT state_cids.nonce,
+	state_cids.balance,
+	state_cids.storage_root,
+	state_cids.code_hash,
+	state_cids.removed
+FROM eth.state_cids,
+	eth.header_cids
+WHERE state_cids.state_leaf_key = $1
+	AND state_cids.block_number <=
+		(SELECT block_number
+			FROM eth.header_cids
+			WHERE block_hash = $2
+			LIMIT 1)
+	AND header_cids.canonical
+	AND state_cids.header_id = header_cids.block_hash
+	AND state_cids.block_number = header_cids.block_number
+ORDER BY state_cids.block_number DESC
+LIMIT 1
+`
+	RetrieveFilteredGQLLogs = `SELECT CAST(eth.log_cids.block_number as TEXT), eth.log_cids.header_id as block_hash,
+			eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id, eth.log_cids.address,
+			eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
+			data, eth.receipt_cids.cid AS rct_cid, eth.receipt_cids.post_status, eth.receipt_cids.tx_id AS tx_hash
+				FROM eth.log_cids, eth.receipt_cids, ipld.blocks
+				WHERE eth.log_cids.rct_id = receipt_cids.tx_id
+				AND eth.log_cids.header_id = receipt_cids.header_id
+				AND eth.log_cids.block_number = receipt_cids.block_number
+				AND log_cids.cid = blocks.key
+				AND log_cids.block_number = blocks.block_number
+				AND receipt_cids.header_id = $1`
+	RetrieveFilteredLogsRange = `SELECT CAST(eth.log_cids.block_number as TEXT), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id,
+			eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
+			eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index,
+			ipld.blocks.data, eth.receipt_cids.cid AS rct_cid, eth.receipt_cids.post_status, log_cids.header_id AS block_hash
+							FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, ipld.blocks
+							WHERE eth.log_cids.block_number >= $1 AND eth.log_cids.block_number <= $2
+							AND eth.log_cids.header_id IN (SELECT block_hash from eth.header_cids where eth.header_cids.block_number >= $1 AND eth.header_cids.block_number <= $2 and eth.header_cids.canonical)
+							AND eth.transaction_cids.block_number = eth.log_cids.block_number
+							AND eth.transaction_cids.header_id = eth.log_cids.header_id
+							AND eth.receipt_cids.block_number = eth.log_cids.block_number
+							AND eth.receipt_cids.header_id = eth.log_cids.header_id
+							AND eth.receipt_cids.tx_id = eth.log_cids.rct_id
+							AND eth.receipt_cids.tx_id = eth.transaction_cids.tx_hash
+							AND ipld.blocks.block_number = eth.log_cids.block_number
+							AND ipld.blocks.key = eth.log_cids.cid`
+
+	RetrieveFilteredLogsSingle = `SELECT CAST(eth.log_cids.block_number as TEXT), eth.log_cids.cid, eth.log_cids.index, eth.log_cids.rct_id,
+			eth.log_cids.address, eth.log_cids.topic0, eth.log_cids.topic1, eth.log_cids.topic2, eth.log_cids.topic3,
+			eth.transaction_cids.tx_hash, eth.transaction_cids.index as txn_index,
+			ipld.blocks.data, eth.receipt_cids.cid AS rct_cid, eth.receipt_cids.post_status, log_cids.header_id AS block_hash
+							FROM eth.log_cids, eth.receipt_cids, eth.transaction_cids, ipld.blocks
+							WHERE eth.log_cids.header_id = $1
+							AND eth.transaction_cids.block_number = eth.log_cids.block_number
+							AND eth.transaction_cids.header_id = eth.log_cids.header_id
+							AND eth.receipt_cids.block_number = eth.log_cids.block_number
+							AND eth.receipt_cids.header_id = eth.log_cids.header_id
+							AND eth.receipt_cids.tx_id = eth.log_cids.rct_id
+							AND eth.receipt_cids.tx_id = eth.transaction_cids.tx_hash
+							AND ipld.blocks.block_number = eth.log_cids.block_number
+							AND ipld.blocks.key = eth.log_cids.cid`
+	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `
+SELECT storage_cids.cid,
+       storage_cids.val,
+       storage_cids.block_number,
+       storage_cids.removed,
+       was_state_leaf_removed_by_number(storage_cids.state_leaf_key, storage_cids.block_number) AS state_leaf_removed
+FROM eth.storage_cids, eth.header_cids
+WHERE header_cids.block_number <= (SELECT block_number from eth.header_cids where block_hash = $3 LIMIT 1)
+    AND header_cids.canonical
+    AND storage_cids.block_number = header_cids.block_number
+    AND storage_cids.header_id = header_cids.block_hash
+    AND storage_cids.storage_leaf_key = $2
+    AND storage_cids.state_leaf_key = $1
+ORDER BY storage_cids.block_number DESC LIMIT 1
+`
+	RetrieveStorageAndRLPByAddressHashAndLeafKeyAndBlockHashPgStr = `
+SELECT storage_cids.cid,
+       storage_cids.val,
+       storage_cids.block_number,
+       storage_cids.removed,
+       was_state_leaf_removed_by_number(storage_cids.state_leaf_key, storage_cids.block_number) AS state_leaf_removed,
+       blocks.data
+FROM eth.storage_cids, eth.header_cids, ipld.blocks
+WHERE header_cids.block_number <= (SELECT block_number from eth.header_cids where block_hash = $3 LIMIT 1)
+    AND header_cids.canonical
+    AND storage_cids.block_number = header_cids.block_number
+    AND storage_cids.header_id = header_cids.block_hash
+    AND storage_cids.storage_leaf_key = $2
+    AND storage_cids.state_leaf_key = $1
+    AND blocks.key = storage_cids.cid
+    AND blocks.block_number = storage_cids.block_number
+ORDER BY storage_cids.block_number DESC LIMIT 1
+`
+	RetrieveCanonicalBlockHashByNumber = `SELECT block_hash FROM eth.header_cids WHERE block_number = $1 AND canonical`
+	RetrieveCanonicalHeaderByNumber    = `
+SELECT header_cids.cid,
+	blocks.data
+FROM ipld.blocks,
+	eth.header_cids
+WHERE header_cids.block_number = $1
+	AND header_cids.canonical
+	AND blocks.key = header_cids.cid
+	AND blocks.block_number = header_cids.block_number
+LIMIT 1
+`
+	RetrieveCanonicalHeaderAndHashByNumber = `
+SELECT blocks.data,
+       header_cids.block_hash
+FROM ipld.blocks,
+	eth.header_cids
+WHERE header_cids.block_number = $1
+	AND header_cids.canonical
+	AND blocks.key = header_cids.cid
+	AND blocks.block_number = header_cids.block_number
+LIMIT 1
+`
+	RetrieveTD = `SELECT CAST(td as TEXT) FROM eth.header_cids
+			WHERE header_cids.block_hash = $1`
+	RetrieveRPCTransaction = `
+SELECT blocks.data,
+	transaction_cids.header_id,
+	transaction_cids.block_number,
+	transaction_cids.index
+FROM eth.transaction_cids,
+    ipld.blocks,
+	eth.header_cids
+WHERE transaction_cids.TX_HASH = $1
+	AND header_cids.block_hash = transaction_cids.header_id
+	AND header_cids.block_number = transaction_cids.block_number
+	AND header_cids.canonical
+        AND blocks.key = transaction_cids.cid
+	AND blocks.block_number = transaction_cids.block_number
+`
+	RetrieveCodeHashByLeafKeyAndBlockHash = `
+SELECT state_cids.code_hash
+FROM eth.state_cids,
+	eth.header_cids
+WHERE
+state_cids.state_leaf_key = $1
+	AND state_cids.block_number <=
+		(SELECT block_number
+			FROM eth.header_cids
+			WHERE block_hash = $2
+			LIMIT 1)
+	AND header_cids.canonical
+	AND state_cids.header_id = header_cids.block_hash
+	AND state_cids.block_number = header_cids.block_number
+ORDER BY state_cids.block_number DESC
+LIMIT 1
+`
+	RetrieveCodeByKey = `SELECT data FROM ipld.blocks WHERE key = $1`
+)
+
+type ipldResult struct {
+	CID    string `db:"cid"`
+	Data   []byte `db:"data"`
+	TxHash string `db:"tx_hash"`
+}
+
+type nodeInfo struct {
+	CID              string `db:"cid"`
+	Value            []byte `db:"val"`
+	BlockNumber      string `db:"block_number"`
+	Data             []byte `db:"data"`
+	Removed          bool   `db:"removed"`
+	StateLeafRemoved bool   `db:"state_leaf_removed"`
+}

pkg/eth/state_test/eth_suite_test.go

@@ -0,0 +1,29 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_state_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestETHSuite(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ipld-eth-server/pkg/eth/state_test")
+}

pkg/eth/state_test/helper_test.go

@@ -0,0 +1,13 @@
+package eth_state_test
+
+import (
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	. "github.com/onsi/gomega"
+)
+
+func CheckGetSliceResponse(sliceResponse eth.GetSliceResponse, expectedResponse eth.GetSliceResponse) {
+	Expect(sliceResponse.SliceID).To(Equal(expectedResponse.SliceID))
+	Expect(sliceResponse.TrieNodes).To(Equal(expectedResponse.TrieNodes))
+	Expect(sliceResponse.Leaves).To(Equal(expectedResponse.Leaves))
+	Expect(sliceResponse.MetaData.NodeStats).To(Equal(expectedResponse.MetaData.NodeStats))
+}

pkg/eth/state_test/state_test.go

@@ -0,0 +1,833 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_state_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"io/ioutil"
+	"math/big"
+	// "github.com/ethereum/go-ethereum/statediff/test_helpers"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
+)
+
+var (
+	parsedABI     abi.ABI
+	randomAddress = common.HexToAddress("0x9F4203bd7a11aCB94882050E6f1C3ab14BBaD3D9")
+	randomHash    = crypto.Keccak256Hash(randomAddress.Bytes())
+	number        = rpc.BlockNumber(test_helpers.BlockNumber.Int64())
+
+	block1StateRoot    = common.HexToHash("0xa1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21")
+	rootDataHashBlock1 = "a1f614839ebdd58677df2c9d66a3e0acc9462acc49fad6006d0b6e5d2b98ed21"
+	rootDataBlock1     = "f871a0577652b625b77bdb5bf77bc43f3125cad7464d679d1575565277d3611b8053e780808080a0fe889f10e5db8f2c2bf355928152a17f6e3bb99a9241ac6d84c77e6264509c798080808080808080a011db0cda34a896dabeb6839bb06a38f49514cfa486435984eb013b7df9ee85c58080"
+
+	block5StateRoot    = common.HexToHash("0x572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43")
+	rootDataHashBlock5 = "572ef3b6b3d5164ed9d83341073f13af4d60a3aab38989b6c03917544f186a43"
+	rootDataBlock5     = "f8b1a0408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f280808080a0b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91a0180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e48080808080a0422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac995180a02d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba358080"
+
+	account1DataHash     = "180d554b171f6acf8295e376266df2311f68975d74c02753b85707d308f703e4"
+	account1Data         = "f869a03114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45b846f8440180a04bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0a0ba79854f3dbf6505fdbb085888e25fae8fa97288c5ce8fcd39aa589290d9a659"
+	account1StateLeafKey = "0x6114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"
+	account1Code         = "608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032"
+	account2DataHash     = "2d264f591aa3fa9df3cbeea190a4fd8d5483ddfb1b85603b2a006d179f79ba35"
+	account2Data         = "f871a03926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2b84ef84c02881bc16d674ec82710a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+	account3DataHash     = "408dd81f6cd5c614f91ecd9faa01d5feba936e0314ba04f99c74069ba819e0f2"
+	account3Data         = "f86da030bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2ab84af848058405f5b608a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+	account4DataHash     = "422c7cc4fa407603f0879a0ecaa809682ce98dbef30551a34bcce09fa3ac9951"
+	account4Data         = "f871a03957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45b84ef84c80883782dace9d9003e8a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+	account5DataHash     = "b356351d60bc9894cf1f1d6cb68c815f0131d50f1da83c4023a09ec855cfff91"
+	account5Data         = "f871a03380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312ab84ef84c80883782dace9d900000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
+
+	contractStorageRootBlock5 = common.HexToHash("0x4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0")
+	storageRootDataHashBlock5 = "4bd45c41d863f1bcf5da53364387fcdd64f77924d388a4df47e64132273fb4c0"
+	storageRootDataBlock5     = "f838a120290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7"
+
+	contractStorageRootBlock4 = common.HexToHash("0x64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68")
+	storageRootDataHashBlock4 = "64ad893aa7937d05983daa8b7d221acdf1c116433f29dcd1ea69f16fa96fce68"
+	storageRootDataBlock4     = "f8518080a08e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d558080808080808080a07b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af8080808080"
+
+	storageNode1DataHash = "7b58866e3801680bea90c82a80eb08889ececef107b8b504ae1d1a1e1e17b7af"
+	storageNode1Data     = "e2a0310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf609"
+	storageNode2DataHash = "8e8ada45207a7d2f19dd6f0ee4955cec64fa5ebef29568b5c449a4c4dd361d55"
+	storageNode2Data     = "f7a0390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e5639594703c4b2bd70c169f5717101caee543299fc946c7"
+)
+
+func init() {
+	// load abi
+	abiBytes, err := ioutil.ReadFile("../test_helpers/abi.json")
+	if err != nil {
+		panic(err)
+	}
+	parsedABI, err = abi.JSON(bytes.NewReader(abiBytes))
+	if err != nil {
+		panic(err)
+	}
+}
+
+const chainLength = 5
+
+var (
+	blocks                  []*types.Block
+	receipts                []types.Receipts
+	chain                   *core.BlockChain
+	db                      *sqlx.DB
+	api                     *eth.PublicEthAPI
+	backend                 *eth.Backend
+	chainConfig             = &*params.TestChainConfig
+	mockTD                  = big.NewInt(1337)
+	expectedCanonicalHeader map[string]interface{}
+	ctx                     = context.Background()
+)
+
+var _ = BeforeSuite(func() {
+	chainConfig.LondonBlock = big.NewInt(100)
+
+	// db and type initializations
+	var err error
+	db = shared.SetupDB()
+
+	backend, err = eth.NewEthBackend(db, &eth.Config{
+		ChainConfig: chainConfig,
+		VMConfig:    vm.Config{},
+		RPCGasCap:   big.NewInt(10000000000), // Max gas capacity for a rpc call.
+		GroupCacheConfig: &shared.GroupCacheConfig{
+			StateDB: shared.GroupConfig{
+				Name:                   "eth_state_test",
+				CacheSizeInMB:          8,
+				CacheExpiryInMins:      60,
+				LogStatsIntervalInSecs: 0,
+			},
+		},
+	})
+	Expect(err).ToNot(HaveOccurred())
+	api, _ = eth.NewPublicEthAPI(backend, nil, eth.APIConfig{StateDiffTimeout: shared.DefaultStateDiffTimeout})
+
+	// make the test blockchain (and state)
+	blocks, receipts, chain = test_helpers.MakeChain(chainLength, test_helpers.Genesis, test_helpers.TestChainGen, chainConfig)
+
+	transformer := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
+	params := statediff.Params{}
+	canonicalHeader := blocks[1].Header()
+	expectedCanonicalHeader = map[string]interface{}{
+		"number":           (*hexutil.Big)(canonicalHeader.Number),
+		"hash":             canonicalHeader.Hash(),
+		"parentHash":       canonicalHeader.ParentHash,
+		"nonce":            canonicalHeader.Nonce,
+		"mixHash":          canonicalHeader.MixDigest,
+		"sha3Uncles":       canonicalHeader.UncleHash,
+		"logsBloom":        canonicalHeader.Bloom,
+		"stateRoot":        canonicalHeader.Root,
+		"miner":            canonicalHeader.Coinbase,
+		"difficulty":       (*hexutil.Big)(canonicalHeader.Difficulty),
+		"extraData":        hexutil.Bytes([]byte{}),
+		"size":             hexutil.Uint64(canonicalHeader.Size()),
+		"gasLimit":         hexutil.Uint64(canonicalHeader.GasLimit),
+		"gasUsed":          hexutil.Uint64(canonicalHeader.GasUsed),
+		"timestamp":        hexutil.Uint64(canonicalHeader.Time),
+		"transactionsRoot": canonicalHeader.TxHash,
+		"receiptsRoot":     canonicalHeader.ReceiptHash,
+		"totalDifficulty":  (*hexutil.Big)(mockTD),
+	}
+
+	// Insert some non-canonical data into the database so that we test our ability to discern canonicity
+	// NOTE: Nan-canonical blocks must come first, because the statediffer will assume the most recent block it is
+	// provided at a certain height is canonical.  This is true inside geth, but not necessarily inside this test.
+	indexAndPublisher := shared.SetupTestStateDiffIndexer(ctx, chainConfig, test_helpers.Genesis.Hash())
+
+	tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+	Expect(err).ToNot(HaveOccurred())
+
+	err = tx.Submit(err)
+	Expect(err).ToNot(HaveOccurred())
+
+	// The non-canonical header has a child
+	tx, err = indexAndPublisher.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
+	Expect(err).ToNot(HaveOccurred())
+
+	err = tx.Submit(err)
+	Expect(err).ToNot(HaveOccurred())
+
+	// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
+	builder := statediff.NewBuilder(chain.StateCache())
+	for i, block := range blocks {
+		var args statediff.Args
+		var rcts types.Receipts
+		if i == 0 {
+			args = statediff.Args{
+				OldStateRoot: common.Hash{},
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+		} else {
+			args = statediff.Args{
+				OldStateRoot: blocks[i-1].Root(),
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+			rcts = receipts[i-1]
+		}
+		diff, err := builder.BuildStateDiffObject(args, params)
+		Expect(err).ToNot(HaveOccurred())
+		tx, err := transformer.PushBlock(block, rcts, mockTD)
+		Expect(err).ToNot(HaveOccurred())
+
+		for _, node := range diff.Nodes {
+			err = transformer.PushStateNode(tx, node, block.Hash().String())
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		for _, ipld := range diff.IPLDs {
+			err = transformer.PushIPLD(tx, ipld)
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		err = tx.Submit(err)
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+})
+
+var _ = AfterSuite(func() {
+	shared.TearDownDB(db)
+	chain.Stop()
+})
+
+var _ = Describe("eth state reading tests", func() {
+
+	Describe("eth_call", func() {
+		It("Applies call args (tx data) on top of state, returning the result (e.g. a Getter method call)", func() {
+			data, err := parsedABI.Pack("data")
+			Expect(err).ToNot(HaveOccurred())
+			bdata := hexutil.Bytes(data)
+			callArgs := eth.CallArgs{
+				To:   &test_helpers.ContractAddr,
+				Data: &bdata,
+			}
+
+			// Before contract deployment, returns nil
+			res, err := api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(0), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(BeNil())
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(1), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(BeNil())
+
+			// After deployment
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(2), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes := hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(3), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(4), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(5), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000"))
+			Expect(res).To(Equal(expectedRes))
+		})
+	})
+
+	Describe("eth_getBalance", func() {
+		var (
+			expectedContractBalance   = (*hexutil.Big)(common.Big0)
+			expectedBankBalanceBlock0 = (*hexutil.Big)(test_helpers.TestBankFunds)
+
+			expectedAcct1BalanceBlock1 = (*hexutil.Big)(big.NewInt(10000))
+			expectedBankBalanceBlock1  = (*hexutil.Big)(new(big.Int).Sub(test_helpers.TestBankFunds, big.NewInt(10000)))
+
+			expectedAcct2BalanceBlock2 = (*hexutil.Big)(big.NewInt(1000))
+			expectedBankBalanceBlock2  = (*hexutil.Big)(new(big.Int).Sub(expectedBankBalanceBlock1.ToInt(), big.NewInt(1000)))
+
+			expectedAcct2BalanceBlock3 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock2.ToInt(), test_helpers.MiningReward))
+
+			expectedAcct2BalanceBlock4 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock3.ToInt(), test_helpers.MiningReward))
+
+			expectedAcct1BalanceBlock5 = (*hexutil.Big)(new(big.Int).Add(expectedAcct1BalanceBlock1.ToInt(), test_helpers.MiningReward))
+		)
+
+		It("Retrieves account balance by block number", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock3))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock5))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+		})
+		It("Retrieves account balance by block hash", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			_, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock2))
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock3))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock5))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+		})
+		It("Returns 0 if account balance not found by block number", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+		})
+		It("Returns 0 if account balance not found by block hash", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal((*hexutil.Big)(common.Big0)))
+		})
+	})
+
+	Describe("eth_getCode", func() {
+		It("Retrieves the code for the provided contract address at the block with the provided number", func() {
+			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+
+			code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+		})
+		It("Retrieves the code for the provided contract address at the block with the provided hash", func() {
+			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+
+			code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+		})
+		It("Returns `nil` for an account it cannot find the code for", func() {
+			code, err := api.GetCode(ctx, randomAddress, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(BeEmpty())
+		})
+		It("Returns `nil`  for a contract that doesn't exist at this height", func() {
+			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(BeEmpty())
+		})
+	})
+
+	Describe("eth_getStorageAt", func() {
+		It("Returns empty slice if it tries to access a contract which does not exist", func() {
+			storage, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+
+			storage, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+		})
+		It("Returns empty slice if it tries to access a contract slot which does not exist", func() {
+			storage, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, randomHash.Hex(), rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(storage).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+		})
+		It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() {
+			// After deployment
+			val, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes := hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(hexutil.Bytes(eth.EmptyNodeValue)))
+		})
+		It("Throws an error for a non-existing block hash", func() {
+			_, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithHash(randomHash, true))
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError("header for hash not found"))
+		})
+		It("Throws an error for a non-existing block number", func() {
+			_, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.IndexOne, rpc.BlockNumberOrHashWithNumber(chainLength+1))
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError("header not found"))
+		})
+	})
+
+	Describe("eth_getHeaderByNumber", func() {
+		It("Finds the canonical header based on the header's weight relative to others at the provided height", func() {
+			header, err := api.GetHeaderByNumber(ctx, number)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(header).To(Equal(expectedCanonicalHeader))
+		})
+	})
+
+	Describe("eth_getSlice", func() {
+		It("Retrieves the state slice for root path", func() {
+			path := "0x"
+			depth := 3
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "1",
+						"02-total-trie-nodes":    "6",
+						"03-leaves":              "5",
+						"04-smart-contracts":     "1",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						rootDataHashBlock5: rootDataBlock5,
+					},
+					Slice: map[string]string{
+						account1DataHash: account1Data,
+						account2DataHash: account2Data,
+						account3DataHash: account3Data,
+						account4DataHash: account4Data,
+						account5DataHash: account5Data,
+					},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{
+					account1StateLeafKey: {
+						StorageRoot: contractStorageRootBlock5.Hex(),
+						EVMCode:     account1Code,
+					},
+				},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the state slice for root path with 0 depth", func() {
+			path := "0x"
+			depth := 0
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "0",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						rootDataHashBlock5: rootDataBlock5,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the state slice for a path to an account", func() {
+			path := "0x06"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block5StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block5StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "2",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "2",
+						"03-leaves":              "1",
+						"04-smart-contracts":     "1",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{
+						rootDataHashBlock5: rootDataBlock5,
+					},
+					Head: map[string]string{
+						account1DataHash: account1Data,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{
+					account1StateLeafKey: {
+						StorageRoot: contractStorageRootBlock5.Hex(),
+						EVMCode:     account1Code,
+					},
+				},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the state slice for a path to a non-existing account", func() {
+			path := "0x06"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, block1StateRoot, false)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, block1StateRoot.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "0",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{
+						rootDataHashBlock1: rootDataBlock1,
+					},
+					Head:  map[string]string{},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+
+		It("Retrieves the storage slice for root path", func() {
+			path := "0x"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "1",
+						"02-total-trie-nodes":    "3",
+						"03-leaves":              "2",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						storageRootDataHashBlock4: storageRootDataBlock4,
+					},
+					Slice: map[string]string{
+						storageNode1DataHash: storageNode1Data,
+						storageNode2DataHash: storageNode2Data,
+					},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the storage slice for root path with 0 depth", func() {
+			path := "0x"
+			depth := 0
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "0",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						storageRootDataHashBlock4: storageRootDataBlock4,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the storage slice for root path with deleted nodes", func() {
+			path := "0x"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock5, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock5.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "1",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "1",
+						"03-leaves":              "1",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{},
+					Head: map[string]string{
+						storageRootDataHashBlock5: storageRootDataBlock5,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+		It("Retrieves the storage slice for a path to a storage node", func() {
+			path := "0x0b"
+			depth := 2
+			sliceResponse, err := api.GetSlice(ctx, path, depth, contractStorageRootBlock4, true)
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedResponse := eth.GetSliceResponse{
+				SliceID: fmt.Sprintf("%s-%d-%s", path, depth, contractStorageRootBlock4.String()),
+				MetaData: eth.GetSliceResponseMetadata{
+					NodeStats: map[string]string{
+						"00-stem-and-head-nodes": "2",
+						"01-max-depth":           "0",
+						"02-total-trie-nodes":    "2",
+						"03-leaves":              "1",
+						"04-smart-contracts":     "0",
+					},
+				},
+				TrieNodes: eth.GetSliceResponseTrieNodes{
+					Stem: map[string]string{
+						storageRootDataHashBlock4: storageRootDataBlock4,
+					},
+					Head: map[string]string{
+						storageNode1DataHash: storageNode1Data,
+					},
+					Slice: map[string]string{},
+				},
+				Leaves: map[string]eth.GetSliceResponseAccount{},
+			}
+
+			CheckGetSliceResponse(*sliceResponse, expectedResponse)
+		})
+	})
+})

pkg/eth/subscription_config.go

@@ -16,38 +16,6 @@
 
 package eth
 
-import (
-	"math/big"
-
-	"github.com/spf13/viper"
-)
-
-// SubscriptionSettings config is used by a subscriber to specify what eth data to stream from the watcher
-type SubscriptionSettings struct {
-	BackFill      bool
-	BackFillOnly  bool
-	Start         *big.Int
-	End           *big.Int // set to 0 or a negative value to have no ending block
-	HeaderFilter  HeaderFilter
-	TxFilter      TxFilter
-	ReceiptFilter ReceiptFilter
-	StateFilter   StateFilter
-	StorageFilter StorageFilter
-}
-
-// HeaderFilter contains filter settings for headers
-type HeaderFilter struct {
-	Off    bool
-	Uncles bool
-}
-
-// TxFilter contains filter settings for txs
-type TxFilter struct {
-	Off bool
-	Src []string
-	Dst []string
-}
-
 // ReceiptFilter contains filter settings for receipts
 type ReceiptFilter struct {
 	Off bool
@@ -56,70 +24,3 @@ type ReceiptFilter struct {
 	LogAddresses []string // receipt contains logs from the provided addresses
 	Topics       [][]string
 }
-
-// StateFilter contains filter settings for state
-type StateFilter struct {
-	Off               bool
-	Addresses         []string // is converted to state key by taking its keccak256 hash
-	IntermediateNodes bool
-}
-
-// StorageFilter contains filter settings for storage
-type StorageFilter struct {
-	Off               bool
-	Addresses         []string
-	StorageKeys       []string // need to be the hashs key themselves not slot position
-	IntermediateNodes bool
-}
-
-// Init is used to initialize a EthSubscription struct with env variables
-func NewEthSubscriptionConfig() (*SubscriptionSettings, error) {
-	sc := new(SubscriptionSettings)
-	// Below default to false, which means we do not backfill by default
-	sc.BackFill = viper.GetBool("watcher.ethSubscription.historicalData")
-	sc.BackFillOnly = viper.GetBool("watcher.ethSubscription.historicalDataOnly")
-	// Below default to 0
-	// 0 start means we start at the beginning and 0 end means we continue indefinitely
-	sc.Start = big.NewInt(viper.GetInt64("watcher.ethSubscription.startingBlock"))
-	sc.End = big.NewInt(viper.GetInt64("watcher.ethSubscription.endingBlock"))
-	// Below default to false, which means we get all headers and no uncles by default
-	sc.HeaderFilter = HeaderFilter{
-		Off:    viper.GetBool("watcher.ethSubscription.headerFilter.off"),
-		Uncles: viper.GetBool("watcher.ethSubscription.headerFilter.uncles"),
-	}
-	// Below defaults to false and two slices of length 0
-	// Which means we get all transactions by default
-	sc.TxFilter = TxFilter{
-		Off: viper.GetBool("watcher.ethSubscription.txFilter.off"),
-		Src: viper.GetStringSlice("watcher.ethSubscription.txFilter.src"),
-		Dst: viper.GetStringSlice("watcher.ethSubscription.txFilter.dst"),
-	}
-	// By default all of the topic slices will be empty => match on any/all topics
-	topics := make([][]string, 4)
-	topics[0] = viper.GetStringSlice("watcher.ethSubscription.receiptFilter.topic0s")
-	topics[1] = viper.GetStringSlice("watcher.ethSubscription.receiptFilter.topic1s")
-	topics[2] = viper.GetStringSlice("watcher.ethSubscription.receiptFilter.topic2s")
-	topics[3] = viper.GetStringSlice("watcher.ethSubscription.receiptFilter.topic3s")
-	sc.ReceiptFilter = ReceiptFilter{
-		Off:          viper.GetBool("watcher.ethSubscription.receiptFilter.off"),
-		MatchTxs:     viper.GetBool("watcher.ethSubscription.receiptFilter.matchTxs"),
-		LogAddresses: viper.GetStringSlice("watcher.ethSubscription.receiptFilter.contracts"),
-		Topics:       topics,
-	}
-	// Below defaults to two false, and a slice of length 0
-	// Which means we get all state leafs by default, but no intermediate nodes
-	sc.StateFilter = StateFilter{
-		Off:               viper.GetBool("watcher.ethSubscription.stateFilter.off"),
-		IntermediateNodes: viper.GetBool("watcher.ethSubscription.stateFilter.intermediateNodes"),
-		Addresses:         viper.GetStringSlice("watcher.ethSubscription.stateFilter.addresses"),
-	}
-	// Below defaults to two false, and two slices of length 0
-	// Which means we get all storage leafs by default, but no intermediate nodes
-	sc.StorageFilter = StorageFilter{
-		Off:               viper.GetBool("watcher.ethSubscription.storageFilter.off"),
-		IntermediateNodes: viper.GetBool("watcher.ethSubscription.storageFilter.intermediateNodes"),
-		Addresses:         viper.GetStringSlice("watcher.ethSubscription.storageFilter.addresses"),
-		StorageKeys:       viper.GetStringSlice("watcher.ethSubscription.storageFilter.storageKeys"),
-	}
-	return sc, nil
-}

pkg/eth/test_helpers.go

@@ -1,66 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-)
-
-// TearDownDB is used to tear down the watcher dbs after tests
-func TearDownDB(db *postgres.DB) {
-	tx, err := db.Beginx()
-	Expect(err).NotTo(HaveOccurred())
-
-	_, err = tx.Exec(`DELETE FROM eth.header_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.transaction_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.receipt_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.state_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM eth.storage_cids`)
-	Expect(err).NotTo(HaveOccurred())
-	_, err = tx.Exec(`DELETE FROM blocks`)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = tx.Commit()
-	Expect(err).NotTo(HaveOccurred())
-}
-
-// TxModelsContainsCID used to check if a list of TxModels contains a specific cid string
-func TxModelsContainsCID(txs []eth.TxModel, cid string) bool {
-	for _, tx := range txs {
-		if tx.CID == cid {
-			return true
-		}
-	}
-	return false
-}
-
-// ListContainsBytes used to check if a list of byte arrays contains a particular byte array
-func ReceiptModelsContainsCID(rcts []eth.ReceiptModel, cid string) bool {
-	for _, rct := range rcts {
-		if rct.CID == cid {
-			return true
-		}
-	}
-	return false
-}

pkg/eth/test_helpers/chain_indexer.go

@@ -0,0 +1,82 @@
+package test_helpers
+
+import (
+	"context"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
+)
+
+type IndexChainParams struct {
+	Blocks      []*types.Block
+	Receipts    []types.Receipts
+	StateCache  state.Database
+	ChainConfig *params.ChainConfig
+
+	StateDiffParams statediff.Params
+	TotalDifficulty *big.Int
+	// Whether to skip indexing state nodes (state_cids, storage_cids)
+	SkipStateNodes bool
+	// Whether to skip indexing IPLD blocks
+	SkipIPLDs bool
+}
+
+func IndexChain(params IndexChainParams) error {
+	indexer := shared.SetupTestStateDiffIndexer(context.Background(), params.ChainConfig, Genesis.Hash())
+	builder := statediff.NewBuilder(params.StateCache)
+	// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
+	for i, block := range params.Blocks {
+		var args statediff.Args
+		var rcts types.Receipts
+		if i == 0 {
+			args = statediff.Args{
+				OldStateRoot: common.Hash{},
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+		} else {
+			args = statediff.Args{
+				OldStateRoot: params.Blocks[i-1].Root(),
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+			rcts = params.Receipts[i-1]
+		}
+
+		diff, err := builder.BuildStateDiffObject(args, params.StateDiffParams)
+		if err != nil {
+			return err
+		}
+		tx, err := indexer.PushBlock(block, rcts, params.TotalDifficulty)
+		if err != nil {
+			return err
+		}
+
+		if !params.SkipStateNodes {
+			for _, node := range diff.Nodes {
+				if err = indexer.PushStateNode(tx, node, block.Hash().String()); err != nil {
+					return err
+				}
+			}
+		}
+		if !params.SkipIPLDs {
+			for _, ipld := range diff.IPLDs {
+				if err := indexer.PushIPLD(tx, ipld); err != nil {
+					return err
+				}
+			}
+		}
+		if err = tx.Submit(err); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/eth/test_helpers/chain_maker.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/test_helpers"
 )
 
 // Test variables
@@ -35,7 +36,8 @@ var (
 	TestBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 	TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
 	TestBankFunds   = big.NewInt(100000000)
-	Genesis         = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds)
+
+	Genesis = test_helpers.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds, big.NewInt(params.InitialBaseFee), params.MaxGasLimit)
 
 	Account1Key, _       = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
 	Account2Key, _       = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
@@ -60,10 +62,12 @@ data function sig: 73d4a13a
 
 // MakeChain creates a chain of n blocks starting at and including parent.
 // the returned hash chain is ordered head->parent.
-func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, []types.Receipts, *core.BlockChain) {
-	config := params.TestChainConfig
+func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen), config *params.ChainConfig) ([]*types.Block, []types.Receipts, *core.BlockChain) {
 	blocks, receipts := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
-	chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil)
+	chain, err := core.NewBlockChain(Testdb, nil, nil, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
+	if err != nil {
+		panic(err)
+	}
 	return append([]*types.Block{parent}, blocks...), receipts, chain
 }
 

pkg/eth/test_helpers/test_data.go

@@ -22,25 +22,19 @@ import (
 	"crypto/rand"
 	"math/big"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
-
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/statediff"
-	"github.com/ethereum/go-ethereum/statediff/testhelpers"
-	"github.com/ipfs/go-block-format"
-	"github.com/multiformats/go-multihash"
-	log "github.com/sirupsen/logrus"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ipfs/go-cid"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
-
-	eth2 "github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
 )
 
 // Test variables
@@ -56,7 +50,7 @@ var (
 		Difficulty:  big.NewInt(5000000),
 		Extra:       []byte{},
 	}
-	MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts()
+	MockTransactions, MockReceipts, SenderAddr = createLegacyTransactionsAndReceipts()
 	MockUncles                                 = []*types.Header{
 		{
 			Time:        1,
@@ -66,6 +60,7 @@ var (
 			ReceiptHash: common.HexToHash("0x1"),
 			Difficulty:  big.NewInt(500001),
 			Extra:       []byte{},
+			ParentHash:  Genesis.Hash(),
 		},
 		{
 			Time:        2,
@@ -75,12 +70,11 @@ var (
 			ReceiptHash: common.HexToHash("0x2"),
 			Difficulty:  big.NewInt(500002),
 			Extra:       []byte{},
+			ParentHash:  Genesis.Hash(),
 		},
 	}
-	ReceiptsRlp, _   = rlp.EncodeToBytes(MockReceipts)
-	MockBlock        = types.NewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts)
-	MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header())
-	MockChildHeader  = types.Header{
+	MockBlock       = createNewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, trie.NewEmpty(nil))
+	MockChildHeader = types.Header{
 		Time:        0,
 		Number:      new(big.Int).Add(BlockNumber, common.Big1),
 		Root:        common.HexToHash("0x0"),
@@ -90,181 +84,121 @@ var (
 		Extra:       []byte{},
 		ParentHash:  MockBlock.Header().Hash(),
 	}
-	MockChild            = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts)
-	MockChildRlp, _      = rlp.EncodeToBytes(MockChild.Header())
-	Address              = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
-	AnotherAddress       = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
-	ContractAddress      = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
-	ContractHash         = crypto.Keccak256Hash(ContractAddress.Bytes()).String()
-	MockContractByteCode = []byte{0, 1, 2, 3, 4, 5}
-	mockTopic11          = common.HexToHash("0x04")
-	mockTopic12          = common.HexToHash("0x06")
-	mockTopic21          = common.HexToHash("0x05")
-	mockTopic22          = common.HexToHash("0x07")
-	MockLog1             = &types.Log{
-		Address: Address,
-		Topics:  []common.Hash{mockTopic11, mockTopic12},
-		Data:    []byte{},
+	MockChild       = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts, trie.NewEmpty(nil))
+	Address         = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
+	AnotherAddress  = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	AnotherAddress1 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476594")
+	AnotherAddress2 = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476596")
+	ContractAddress = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
+	mockTopic11     = common.HexToHash("0x04")
+	mockTopic12     = common.HexToHash("0x06")
+	mockTopic21     = common.HexToHash("0x05")
+	mockTopic22     = common.HexToHash("0x07")
+	mockTopic31     = common.HexToHash("0x08")
+	mockTopic41     = common.HexToHash("0x09")
+	mockTopic42     = common.HexToHash("0x0a")
+	mockTopic43     = common.HexToHash("0x0b")
+	mockTopic51     = common.HexToHash("0x0c")
+	mockTopic61     = common.HexToHash("0x0d")
+	MockLog1        = &types.Log{
+		Address:     Address,
+		Topics:      []common.Hash{mockTopic11, mockTopic12},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     0,
+		Index:       0,
 	}
 	MockLog2 = &types.Log{
-		Address: AnotherAddress,
-		Topics:  []common.Hash{mockTopic21, mockTopic22},
-		Data:    []byte{},
+		Address:     AnotherAddress,
+		Topics:      []common.Hash{mockTopic21, mockTopic22},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     1,
+		Index:       1,
 	}
-	HeaderCID, _  = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
-	HeaderMhKey   = shared.MultihashKeyFromCID(HeaderCID)
-	Trx1CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(0), multihash.KECCAK_256)
-	Trx1MhKey     = shared.MultihashKeyFromCID(Trx1CID)
-	Trx2CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
-	Trx2MhKey     = shared.MultihashKeyFromCID(Trx2CID)
-	Trx3CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(2), multihash.KECCAK_256)
-	Trx3MhKey     = shared.MultihashKeyFromCID(Trx3CID)
-	Rct1CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
-	Rct1MhKey     = shared.MultihashKeyFromCID(Rct1CID)
-	Rct2CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
-	Rct2MhKey     = shared.MultihashKeyFromCID(Rct2CID)
-	Rct3CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(2), multihash.KECCAK_256)
-	Rct3MhKey     = shared.MultihashKeyFromCID(Rct3CID)
-	State1CID, _  = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
-	State1MhKey   = shared.MultihashKeyFromCID(State1CID)
-	State2CID, _  = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
-	State2MhKey   = shared.MultihashKeyFromCID(State2CID)
-	StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
-	StorageMhKey  = shared.MultihashKeyFromCID(StorageCID)
-	MockTrxMeta   = []eth.TxModel{
+	MockLog3 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic31},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     2,
+		Index:       2,
+	}
+
+	MockLog4 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic41, mockTopic42, mockTopic43},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     2,
+		Index:       3,
+	}
+	MockLog5 = &types.Log{
+		Address:     AnotherAddress1,
+		Topics:      []common.Hash{mockTopic51},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     2,
+		Index:       4,
+	}
+	MockLog6 = &types.Log{
+		Address:     AnotherAddress2,
+		Topics:      []common.Hash{mockTopic61},
+		Data:        []byte{},
+		BlockNumber: BlockNumber.Uint64(),
+		TxIndex:     3,
+		Index:       5,
+	}
+
+	rctCIDs, _  = getReceiptCIDs(MockReceipts)
+	Rct1CID     = rctCIDs[0]
+	Rct4CID     = rctCIDs[3]
+	MockTrxMeta = []models.TxModel{
 		{
 			CID:    "", // This is empty until we go to publish to ipfs
-			MhKey:  "",
 			Src:    SenderAddr.Hex(),
 			Dst:    Address.String(),
 			Index:  0,
 			TxHash: MockTransactions[0].Hash().String(),
-			Data:   []byte{},
 		},
 		{
 			CID:    "",
-			MhKey:  "",
 			Src:    SenderAddr.Hex(),
 			Dst:    AnotherAddress.String(),
 			Index:  1,
 			TxHash: MockTransactions[1].Hash().String(),
-			Data:   []byte{},
 		},
 		{
 			CID:    "",
-			MhKey:  "",
 			Src:    SenderAddr.Hex(),
 			Dst:    "",
 			Index:  2,
 			TxHash: MockTransactions[2].Hash().String(),
-			Data:   MockContractByteCode,
-		},
-	}
-	MockTrxMetaPostPublsh = []eth.TxModel{
-		{
-			CID:    Trx1CID.String(), // This is empty until we go to publish to ipfs
-			MhKey:  Trx1MhKey,
-			Src:    SenderAddr.Hex(),
-			Dst:    Address.String(),
-			Index:  0,
-			TxHash: MockTransactions[0].Hash().String(),
-			Data:   []byte{},
 		},
 		{
-			CID:    Trx2CID.String(),
-			MhKey:  Trx2MhKey,
-			Src:    SenderAddr.Hex(),
-			Dst:    AnotherAddress.String(),
-			Index:  1,
-			TxHash: MockTransactions[1].Hash().String(),
-			Data:   []byte{},
-		},
-		{
-			CID:    Trx3CID.String(),
-			MhKey:  Trx3MhKey,
+			CID:    "",
 			Src:    SenderAddr.Hex(),
 			Dst:    "",
-			Index:  2,
-			TxHash: MockTransactions[2].Hash().String(),
-			Data:   MockContractByteCode,
+			Index:  3,
+			TxHash: MockTransactions[3].Hash().String(),
 		},
 	}
-	MockRctMeta = []eth.ReceiptModel{
+	MockRctMeta = []models.ReceiptModel{
 		{
-			CID:   "",
-			MhKey: "",
-			Topic0s: []string{
-				mockTopic11.String(),
-			},
-			Topic1s: []string{
-				mockTopic12.String(),
-			},
-			Contract:     "",
-			ContractHash: "",
-			LogContracts: []string{
-				Address.String(),
-			},
+			CID:      "",
+			Contract: "",
 		},
 		{
-			CID:   "",
-			MhKey: "",
-			Topic0s: []string{
-				mockTopic21.String(),
-			},
-			Topic1s: []string{
-				mockTopic22.String(),
-			},
-			Contract:     "",
-			ContractHash: "",
-			LogContracts: []string{
-				AnotherAddress.String(),
-			},
+			CID:      "",
+			Contract: "",
 		},
 		{
-			CID:          "",
-			MhKey:        "",
-			Contract:     ContractAddress.String(),
-			ContractHash: ContractHash,
-			LogContracts: []string{},
-		},
-	}
-	MockRctMetaPostPublish = []eth.ReceiptModel{
-		{
-			CID:   Rct1CID.String(),
-			MhKey: Rct1MhKey,
-			Topic0s: []string{
-				mockTopic11.String(),
-			},
-			Topic1s: []string{
-				mockTopic12.String(),
-			},
-			Contract:     "",
-			ContractHash: "",
-			LogContracts: []string{
-				Address.String(),
-			},
+			CID:      "",
+			Contract: ContractAddress.String(),
 		},
 		{
-			CID:   Rct2CID.String(),
-			MhKey: Rct2MhKey,
-			Topic0s: []string{
-				mockTopic21.String(),
-			},
-			Topic1s: []string{
-				mockTopic22.String(),
-			},
-			Contract:     "",
-			ContractHash: "",
-			LogContracts: []string{
-				AnotherAddress.String(),
-			},
-		},
-		{
-			CID:          Rct3CID.String(),
-			MhKey:        Rct3MhKey,
-			Contract:     ContractAddress.String(),
-			ContractHash: ContractHash,
-			LogContracts: []string{},
+			CID:      "",
+			Contract: "",
 		},
 	}
 
@@ -273,26 +207,25 @@ var (
 	StorageLeafKey     = crypto.Keccak256Hash(storageLocation[:]).Bytes()
 	StorageValue       = crypto.Keccak256([]byte{1, 2, 3, 4, 5})
 	StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
-	StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		StoragePartialPath,
 		StorageValue,
 	})
 
-	nonce1             = uint64(1)
-	ContractRoot       = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
-	ContractCodeHash   = crypto.Keccak256Hash(MockContractByteCode)
-	contractPath       = common.Bytes2Hex([]byte{'\x06'})
-	ContractLeafKey    = testhelpers.AddressToLeafKey(ContractAddress)
-	ContractAccount, _ = rlp.EncodeToBytes(state.Account{
-		Nonce:    nonce1,
+	ContractRoot    = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
+	contractPath    = common.Bytes2Hex([]byte{'\x06'})
+	ContractLeafKey = crypto.Keccak256(ContractAddress[:])
+	ContractAccount = types.StateAccount{
+		Nonce:    uint64(1),
 		Balance:  big.NewInt(0),
-		CodeHash: ContractCodeHash.Bytes(),
+		CodeHash: CodeHash.Bytes(),
 		Root:     common.HexToHash(ContractRoot),
-	})
-	ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
-	ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	}
+	ContractAccountRLP, _ = rlp.EncodeToBytes(&ContractAccount)
+	ContractPartialPath   = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
+	ContractLeafNode, _   = rlp.EncodeToBytes(&[]interface{}{
 		ContractPartialPath,
-		ContractAccount,
+		ContractAccountRLP,
 	})
 
 	nonce0          = uint64(0)
@@ -300,56 +233,49 @@ var (
 	AccountRoot     = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
 	AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
 	AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
-	AccountLeafKey  = testhelpers.Account2LeafKey
-	Account, _      = rlp.EncodeToBytes(state.Account{
+	AccountLeafKey  = crypto.Keccak256(AccountAddresss[:])
+	Account         = types.StateAccount{
 		Nonce:    nonce0,
 		Balance:  AccountBalance,
 		CodeHash: AccountCodeHash.Bytes(),
 		Root:     common.HexToHash(AccountRoot),
-	})
+	}
+	AccountRLP, _      = rlp.EncodeToBytes(&Account)
 	AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
-	AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	AccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		AccountPartialPath,
-		Account,
+		AccountRLP,
 	})
 
-	MockStateNodes = []eth.TrieNode{
+	MockStateNodes = []sdtypes.StateLeafNode{
 		{
-			LeafKey: common.BytesToHash(ContractLeafKey),
-			Path:    []byte{'\x06'},
-			Value:   ContractLeafNode,
-			Type:    statediff.Leaf,
+			AccountWrapper: sdtypes.AccountWrapper{
+				Account: &ContractAccount,
+				LeafKey: ContractLeafKey,
+				CID:     ipld.Keccak256ToCid(ipld.MEthStateTrie, crypto.Keccak256(ContractLeafNode)).String(),
+			},
+			StorageDiff: []sdtypes.StorageLeafNode{
+				{
+					LeafKey: StorageLeafKey,
+					Value:   StorageValue,
+					CID:     ipld.Keccak256ToCid(ipld.MEthStorageTrie, crypto.Keccak256(StorageLeafNode)).String(),
+				},
+			},
 		},
 		{
-			LeafKey: common.BytesToHash(AccountLeafKey),
-			Path:    []byte{'\x0c'},
-			Value:   AccountLeafNode,
-			Type:    statediff.Leaf,
+			AccountWrapper: sdtypes.AccountWrapper{
+				Account: &Account,
+				LeafKey: AccountLeafKey,
+				CID:     ipld.Keccak256ToCid(ipld.MEthStateTrie, crypto.Keccak256(AccountLeafNode)).String(),
+			},
 		},
 	}
-	MockStateMetaPostPublish = []eth.StateNodeModel{
-		{
-			CID:      State1CID.String(),
-			MhKey:    State1MhKey,
-			Path:     []byte{'\x06'},
-			NodeType: 2,
-			StateKey: common.BytesToHash(ContractLeafKey).Hex(),
-		},
-		{
-			CID:      State2CID.String(),
-			MhKey:    State2MhKey,
-			Path:     []byte{'\x0c'},
-			NodeType: 2,
-			StateKey: common.BytesToHash(AccountLeafKey).Hex(),
-		},
-	}
-	MockStorageNodes = map[string][]eth.TrieNode{
+	MockStorageNodes = map[string][]sdtypes.StorageLeafNode{
 		contractPath: {
 			{
-				LeafKey: common.BytesToHash(StorageLeafKey),
-				Value:   StorageLeafNode,
-				Type:    statediff.Leaf,
-				Path:    []byte{},
+				LeafKey: StorageLeafKey,
+				Value:   StorageValue,
+				CID:     ipld.Keccak256ToCid(ipld.MEthStorageTrie, crypto.Keccak256(StorageLeafNode)).String(),
 			},
 		},
 	}
@@ -363,136 +289,108 @@ var (
 		StorageNodes:    MockStorageNodes,
 		StateNodes:      MockStateNodes,
 	}
-	MockConvertedPayloadForChild = eth.ConvertedPayload{
-		TotalDifficulty: MockChild.Difficulty(),
-		Block:           MockChild,
-		Receipts:        MockReceipts,
-		TxMetaData:      MockTrxMeta,
-		ReceiptMetaData: MockRctMeta,
-		StorageNodes:    MockStorageNodes,
-		StateNodes:      MockStateNodes,
+
+	LondonBlockNum   = new(big.Int).Add(BlockNumber, big.NewInt(2))
+	MockLondonHeader = types.Header{
+		Time:       0,
+		Number:     LondonBlockNum,
+		Root:       common.HexToHash("0x00"),
+		Difficulty: big.NewInt(5000000),
+		Extra:      []byte{},
+		BaseFee:    big.NewInt(params.InitialBaseFee),
 	}
 
-	MockCIDWrapper = &eth2.CIDWrapper{
-		BlockNumber: new(big.Int).Set(BlockNumber),
-		Header: eth.HeaderModel{
-			BlockNumber:     "1",
-			BlockHash:       MockBlock.Hash().String(),
-			ParentHash:      "0x0000000000000000000000000000000000000000000000000000000000000000",
-			CID:             HeaderCID.String(),
-			MhKey:           HeaderMhKey,
-			TotalDifficulty: MockBlock.Difficulty().String(),
-			Reward:          "5312500000000000000",
-			StateRoot:       MockBlock.Root().String(),
-			RctRoot:         MockBlock.ReceiptHash().String(),
-			TxRoot:          MockBlock.TxHash().String(),
-			UncleRoot:       MockBlock.UncleHash().String(),
-			Bloom:           MockBlock.Bloom().Bytes(),
-			Timestamp:       MockBlock.Time(),
-			TimesValidated:  1,
+	MockLondonTransactions, MockLondonReceipts, _ = createDynamicTransactionsAndReceipts(LondonBlockNum)
+	MockLondonUncles                              = []*types.Header{
+		{
+			Time:        1,
+			Number:      new(big.Int).Add(BlockNumber, big.NewInt(1)),
+			ParentHash:  common.HexToHash("0x2"),
+			Root:        common.HexToHash("0x1"),
+			TxHash:      common.HexToHash("0x1"),
+			ReceiptHash: common.HexToHash("0x1"),
+			Difficulty:  big.NewInt(500001),
+			Extra:       []byte{},
 		},
-		Transactions: MockTrxMetaPostPublsh,
-		Receipts:     MockRctMetaPostPublish,
-		Uncles:       []eth.UncleModel{},
-		StateNodes:   MockStateMetaPostPublish,
-		StorageNodes: []eth.StorageNodeWithStateKeyModel{
-			{
-				Path:       []byte{},
-				CID:        StorageCID.String(),
-				MhKey:      StorageMhKey,
-				NodeType:   2,
-				StateKey:   common.BytesToHash(ContractLeafKey).Hex(),
-				StorageKey: common.BytesToHash(StorageLeafKey).Hex(),
-			},
-		},
-	}
-
-	HeaderIPLD, _  = blocks.NewBlockWithCid(MockHeaderRlp, HeaderCID)
-	Trx1IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(0), Trx1CID)
-	Trx2IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), Trx2CID)
-	Trx3IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(2), Trx3CID)
-	Rct1IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), Rct1CID)
-	Rct2IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), Rct2CID)
-	Rct3IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(2), Rct3CID)
-	State1IPLD, _  = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
-	State2IPLD, _  = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
-	StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
-
-	MockIPLDs = eth2.IPLDs{
-		BlockNumber: new(big.Int).Set(BlockNumber),
-		Header: ipfs.BlockModel{
-			Data: HeaderIPLD.RawData(),
-			CID:  HeaderIPLD.Cid().String(),
-		},
-		Transactions: []ipfs.BlockModel{
-			{
-				Data: Trx1IPLD.RawData(),
-				CID:  Trx1IPLD.Cid().String(),
-			},
-			{
-				Data: Trx2IPLD.RawData(),
-				CID:  Trx2IPLD.Cid().String(),
-			},
-			{
-				Data: Trx3IPLD.RawData(),
-				CID:  Trx3IPLD.Cid().String(),
-			},
-		},
-		Receipts: []ipfs.BlockModel{
-			{
-				Data: Rct1IPLD.RawData(),
-				CID:  Rct1IPLD.Cid().String(),
-			},
-			{
-				Data: Rct2IPLD.RawData(),
-				CID:  Rct2IPLD.Cid().String(),
-			},
-			{
-				Data: Rct3IPLD.RawData(),
-				CID:  Rct3IPLD.Cid().String(),
-			},
-		},
-		StateNodes: []eth2.StateNode{
-			{
-				StateLeafKey: common.BytesToHash(ContractLeafKey),
-				Type:         statediff.Leaf,
-				IPLD: ipfs.BlockModel{
-					Data: State1IPLD.RawData(),
-					CID:  State1IPLD.Cid().String(),
-				},
-				Path: []byte{'\x06'},
-			},
-			{
-				StateLeafKey: common.BytesToHash(AccountLeafKey),
-				Type:         statediff.Leaf,
-				IPLD: ipfs.BlockModel{
-					Data: State2IPLD.RawData(),
-					CID:  State2IPLD.Cid().String(),
-				},
-				Path: []byte{'\x0c'},
-			},
-		},
-		StorageNodes: []eth2.StorageNode{
-			{
-				StateLeafKey:   common.BytesToHash(ContractLeafKey),
-				StorageLeafKey: common.BytesToHash(StorageLeafKey),
-				Type:           statediff.Leaf,
-				IPLD: ipfs.BlockModel{
-					Data: StorageIPLD.RawData(),
-					CID:  StorageIPLD.Cid().String(),
-				},
-				Path: []byte{},
-			},
+		{
+			Time:        2,
+			Number:      new(big.Int).Add(BlockNumber, big.NewInt(1)),
+			ParentHash:  common.HexToHash("0x1"),
+			Root:        common.HexToHash("0x2"),
+			TxHash:      common.HexToHash("0x2"),
+			ReceiptHash: common.HexToHash("0x2"),
+			Difficulty:  big.NewInt(500002),
+			Extra:       []byte{},
 		},
 	}
+	MockLondonBlock = createNewBlock(&MockLondonHeader, MockLondonTransactions, MockLondonUncles, MockLondonReceipts, trie.NewEmpty(nil))
 )
 
-// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
-func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
+func createNewBlock(header *types.Header, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, hasher types.TrieHasher) *types.Block {
+	block := types.NewBlock(header, txs, uncles, receipts, hasher)
+	bHash := block.Hash()
+	for _, r := range receipts {
+		for _, l := range r.Logs {
+			l.BlockHash = bHash
+		}
+	}
+	return block
+}
+
+// createDynamicTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createDynamicTransactionsAndReceipts(blockNumber *big.Int) (types.Transactions, types.Receipts, common.Address) {
+	// make transactions
+	config := *params.TestChainConfig
+	config.LondonBlock = blockNumber
+	trx1 := types.NewTx(&types.DynamicFeeTx{
+		ChainID:   config.ChainID,
+		Nonce:     1,
+		GasTipCap: big.NewInt(50),
+		GasFeeCap: big.NewInt(100),
+		Gas:       50,
+		To:        &Address,
+		Value:     big.NewInt(1000),
+		Data:      []byte{},
+	})
+
+	transactionSigner := types.MakeSigner(&config, blockNumber)
+	mockCurve := elliptic.P256()
+	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+	signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	senderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	// make receipts
+	// TODO: Change the receipt type to DynamicFeeTxType once this PR is merged.
+	// https://github.com/ethereum/go-ethereum/pull/22806
+	mockReceipt1 := &types.Receipt{
+		Type:              types.DynamicFeeTxType,
+		PostState:         common.HexToHash("0x0").Bytes(),
+		Status:            types.ReceiptStatusSuccessful,
+		CumulativeGasUsed: 50,
+		Logs:              []*types.Log{},
+		TxHash:            signedTrx1.Hash(),
+	}
+
+	return types.Transactions{signedTrx1}, types.Receipts{mockReceipt1}, senderAddr
+}
+
+// createLegacyTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createLegacyTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
 	// make transactions
 	trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
 	trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
-	trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode)
+	trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), ContractCode)
+	trx4 := types.NewTransaction(3, AnotherAddress1, big.NewInt(2000), 100, big.NewInt(200), []byte{})
 	transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber))
 	mockCurve := elliptic.P256()
 	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
@@ -511,19 +409,54 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common
 	if err != nil {
 		log.Fatal(err)
 	}
+	signedTrx4, err := types.SignTx(trx4, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
 	SenderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
 	if err != nil {
 		log.Fatal(err)
 	}
 	// make receipts
-	mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
+	mockReceipt1 := types.NewReceipt(nil, false, 50)
+
+	hash1 := signedTrx1.Hash()
+	MockLog1.TxHash = hash1
+
 	mockReceipt1.Logs = []*types.Log{MockLog1}
-	mockReceipt1.TxHash = signedTrx1.Hash()
+	mockReceipt1.TxHash = hash1
+	mockReceipt1.GasUsed = mockReceipt1.CumulativeGasUsed
+
 	mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
+	hash2 := signedTrx2.Hash()
+	MockLog2.TxHash = hash2
+
 	mockReceipt2.Logs = []*types.Log{MockLog2}
-	mockReceipt2.TxHash = signedTrx2.Hash()
-	mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75)
-	mockReceipt3.Logs = []*types.Log{}
+	mockReceipt2.TxHash = hash2
+	mockReceipt2.GasUsed = mockReceipt2.CumulativeGasUsed - mockReceipt1.CumulativeGasUsed
+
+	mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 175)
+	mockReceipt3.Logs = []*types.Log{MockLog3, MockLog4, MockLog5}
 	mockReceipt3.TxHash = signedTrx3.Hash()
-	return types.Transactions{signedTrx1, signedTrx2, signedTrx3}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3}, SenderAddr
+	mockReceipt3.GasUsed = mockReceipt3.CumulativeGasUsed - mockReceipt2.CumulativeGasUsed
+
+	// Receipt with failed status.
+	mockReceipt4 := types.NewReceipt(nil, true, 250)
+	mockReceipt4.Logs = []*types.Log{MockLog6}
+	mockReceipt4.TxHash = signedTrx4.Hash()
+	mockReceipt4.GasUsed = mockReceipt4.CumulativeGasUsed - mockReceipt3.CumulativeGasUsed
+
+	return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4}, SenderAddr
+}
+
+func getReceiptCIDs(rcts []*types.Receipt) ([]cid.Cid, error) {
+	cids := make([]cid.Cid, len(rcts))
+	for i, rct := range rcts {
+		ethRct, err := ipld.NewReceipt(rct)
+		if err != nil {
+			return nil, err
+		}
+		cids[i] = ethRct.Cid()
+	}
+	return cids, nil
 }

pkg/eth/transaction_arguments.go

@@ -0,0 +1,49 @@
+package eth
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+// TransactionArgs represents the arguments to construct a new transaction
+// or a message call.
+type TransactionArgs struct {
+	From                 *common.Address `json:"from"`
+	To                   *common.Address `json:"to"`
+	Gas                  *hexutil.Uint64 `json:"gas"`
+	GasPrice             *hexutil.Big    `json:"gasPrice"`
+	MaxFeePerGas         *hexutil.Big    `json:"maxFeePerGas"`
+	MaxPriorityFeePerGas *hexutil.Big    `json:"maxPriorityFeePerGas"`
+	Value                *hexutil.Big    `json:"value"`
+	Nonce                *hexutil.Uint64 `json:"nonce"`
+
+	// We accept "data" and "input" for backwards-compatibility reasons.
+	// "input" is the newer name and should be preferred by clients.
+	// Issue detail: https://github.com/ethereum/go-ethereum/issues/15628
+	Data  *hexutil.Bytes `json:"data"`
+	Input *hexutil.Bytes `json:"input"`
+
+	// Introduced by AccessListTxType transaction.
+	AccessList *types.AccessList `json:"accessList,omitempty"`
+	ChainID    *hexutil.Big      `json:"chainId,omitempty"`
+}
+
+// from retrieves the transaction sender address.
+func (args *TransactionArgs) from() common.Address {
+	if args.From == nil {
+		return common.Address{}
+	}
+	return *args.From
+}
+
+// data retrieves the transaction calldata. Input field is preferred.
+func (args *TransactionArgs) data() []byte {
+	if args.Input != nil {
+		return *args.Input
+	}
+	if args.Data != nil {
+		return *args.Data
+	}
+	return nil
+}

pkg/eth/types.go

@@ -17,32 +17,42 @@
 package eth
 
 import (
+	"errors"
+	"fmt"
 	"math/big"
+	"strconv"
 
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/statediff"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	"github.com/ethereum/go-ethereum/statediff/indexer/models"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 )
 
 // RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
 type RPCTransaction struct {
-	BlockHash        *common.Hash    `json:"blockHash"`
-	BlockNumber      *hexutil.Big    `json:"blockNumber"`
-	From             common.Address  `json:"from"`
-	Gas              hexutil.Uint64  `json:"gas"`
-	GasPrice         *hexutil.Big    `json:"gasPrice"`
-	Hash             common.Hash     `json:"hash"`
-	Input            hexutil.Bytes   `json:"input"`
-	Nonce            hexutil.Uint64  `json:"nonce"`
-	To               *common.Address `json:"to"`
-	TransactionIndex *hexutil.Uint64 `json:"transactionIndex"`
-	Value            *hexutil.Big    `json:"value"`
-	V                *hexutil.Big    `json:"v"`
-	R                *hexutil.Big    `json:"r"`
-	S                *hexutil.Big    `json:"s"`
+	BlockHash        *common.Hash      `json:"blockHash"`
+	BlockNumber      *hexutil.Big      `json:"blockNumber"`
+	From             common.Address    `json:"from"`
+	Gas              hexutil.Uint64    `json:"gas"`
+	GasPrice         *hexutil.Big      `json:"gasPrice"`
+	GasFeeCap        *hexutil.Big      `json:"maxFeePerGas,omitempty"`
+	GasTipCap        *hexutil.Big      `json:"maxPriorityFeePerGas,omitempty"`
+	Hash             common.Hash       `json:"hash"`
+	Input            hexutil.Bytes     `json:"input"`
+	Nonce            hexutil.Uint64    `json:"nonce"`
+	To               *common.Address   `json:"to"`
+	TransactionIndex *hexutil.Uint64   `json:"transactionIndex"`
+	Value            *hexutil.Big      `json:"value"`
+	Type             hexutil.Uint64    `json:"type"`
+	Accesses         *types.AccessList `json:"accessList,omitempty"`
+	ChainID          *hexutil.Big      `json:"chainId,omitempty"`
+	V                *hexutil.Big      `json:"v"`
+	R                *hexutil.Big      `json:"r"`
+	S                *hexutil.Big      `json:"s"`
 }
 
 // RPCReceipt represents a receipt that will serialize to the RPC representation of a receipt
@@ -82,65 +92,209 @@ type StorageResult struct {
 
 // CallArgs represents the arguments for a call.
 type CallArgs struct {
-	From     *common.Address `json:"from"`
-	To       *common.Address `json:"to"`
-	Gas      *hexutil.Uint64 `json:"gas"`
-	GasPrice *hexutil.Big    `json:"gasPrice"`
-	Value    *hexutil.Big    `json:"value"`
-	Data     *hexutil.Bytes  `json:"data"`
+	From                 *common.Address   `json:"from"`
+	To                   *common.Address   `json:"to"`
+	Gas                  *hexutil.Uint64   `json:"gas"`
+	GasPrice             *hexutil.Big      `json:"gasPrice"`
+	MaxFeePerGas         *hexutil.Big      `json:"maxFeePerGas"`
+	MaxPriorityFeePerGas *hexutil.Big      `json:"maxPriorityFeePerGas"`
+	Value                *hexutil.Big      `json:"value"`
+	Data                 *hexutil.Bytes    `json:"data"`
+	AccessList           *types.AccessList `json:"accessList,omitempty"`
+	Input                *hexutil.Bytes    `json:"input"`
 }
 
-// account indicates the overriding fields of account during the execution of
-// a message call.
-// Note, state and stateDiff can't be specified at the same time. If state is
-// set, message execution will only use the data in the given state. Otherwise
-// if statDiff is set, all diff will be applied first and then execute the call
-// message.
-type account struct {
-	Nonce     *hexutil.Uint64              `json:"nonce"`
-	Code      *hexutil.Bytes               `json:"code"`
-	Balance   **hexutil.Big                `json:"balance"`
-	State     *map[common.Hash]common.Hash `json:"state"`
-	StateDiff *map[common.Hash]common.Hash `json:"stateDiff"`
+// from retrieves the transaction sender address.
+func (arg *CallArgs) from() common.Address {
+	if arg.From == nil {
+		return common.Address{}
+	}
+	return *arg.From
 }
 
-// IPLDs is used to package raw IPLD block data fetched from IPFS and returned by the server
-// Returned by IPLDFetcher and ResponseFilterer
-type IPLDs struct {
-	BlockNumber     *big.Int
+// data retrieves the transaction calldata. Input field is preferred.
+func (arg *CallArgs) data() []byte {
+	if arg.Input != nil {
+		return *arg.Input
+	}
+	if arg.Data != nil {
+		return *arg.Data
+	}
+	return nil
+}
+
+// ToMessage converts the transaction arguments to the Message type used by the
+// core evm. This method is used in calls and traces that do not require a real
+// live transaction.
+func (arg *CallArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (*core.Message, error) {
+	// Reject invalid combinations of pre- and post-1559 fee styles
+	if arg.GasPrice != nil && (arg.MaxFeePerGas != nil || arg.MaxPriorityFeePerGas != nil) {
+		return nil, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified")
+	}
+	// Set sender address or use zero address if none specified.
+	addr := arg.from()
+
+	// Set default gas & gas price if none were set
+	gas := globalGasCap
+	if gas == 0 {
+		gas = uint64(math.MaxUint64 / 2)
+	}
+	if arg.Gas != nil {
+		gas = uint64(*arg.Gas)
+	}
+	if globalGasCap != 0 && globalGasCap < gas {
+		log.Warn("Caller gas above allowance, capping", "requested", gas, "cap", globalGasCap)
+		gas = globalGasCap
+	}
+	var (
+		gasPrice  *big.Int
+		gasFeeCap *big.Int
+		gasTipCap *big.Int
+	)
+	if baseFee == nil {
+		// If there's no basefee, then it must be a non-1559 execution
+		gasPrice = new(big.Int)
+		if arg.GasPrice != nil {
+			gasPrice = arg.GasPrice.ToInt()
+		}
+		gasFeeCap, gasTipCap = gasPrice, gasPrice
+	} else {
+		// A basefee is provided, necessitating 1559-type execution
+		if arg.GasPrice != nil {
+			// User specified the legacy gas field, convert to 1559 gas typing
+			gasPrice = arg.GasPrice.ToInt()
+			gasFeeCap, gasTipCap = gasPrice, gasPrice
+		} else {
+			// User specified 1559 gas feilds (or none), use those
+			gasFeeCap = new(big.Int)
+			if arg.MaxFeePerGas != nil {
+				gasFeeCap = arg.MaxFeePerGas.ToInt()
+			}
+			gasTipCap = new(big.Int)
+			if arg.MaxPriorityFeePerGas != nil {
+				gasTipCap = arg.MaxPriorityFeePerGas.ToInt()
+			}
+			// Backfill the legacy gasPrice for EVM execution, unless we're all zeroes
+			gasPrice = new(big.Int)
+			if gasFeeCap.BitLen() > 0 || gasTipCap.BitLen() > 0 {
+				gasPrice = math.BigMin(new(big.Int).Add(gasTipCap, baseFee), gasFeeCap)
+			}
+		}
+	}
+	value := new(big.Int)
+	if arg.Value != nil {
+		value = arg.Value.ToInt()
+	}
+	data := arg.data()
+	var accessList types.AccessList
+	if arg.AccessList != nil {
+		accessList = *arg.AccessList
+	}
+	msg := &core.Message{
+		Nonce:             0,
+		GasLimit:          gas,
+		GasPrice:          gasPrice,
+		GasFeeCap:         gasFeeCap,
+		GasTipCap:         gasTipCap,
+		To:                arg.To,
+		Value:             value,
+		Data:              data,
+		AccessList:        accessList,
+		SkipAccountChecks: true,
+		From:              addr,
+	}
+	return msg, nil
+}
+
+// ConvertedPayload is a custom type which packages raw ETH data for publishing to IPFS and filtering to subscribers
+// Returned by PayloadConverter
+// Passed to IPLDPublisher and ResponseFilterer
+type ConvertedPayload struct {
 	TotalDifficulty *big.Int
-	Header          ipfs.BlockModel
-	Uncles          []ipfs.BlockModel
-	Transactions    []ipfs.BlockModel
-	Receipts        []ipfs.BlockModel
-	StateNodes      []StateNode
-	StorageNodes    []StorageNode
+	Block           *types.Block
+	TxMetaData      []models.TxModel
+	Receipts        types.Receipts
+	ReceiptMetaData []models.ReceiptModel
+	StateNodes      []sdtypes.StateLeafNode
+	StorageNodes    map[string][]sdtypes.StorageLeafNode
 }
 
-type StateNode struct {
-	Type         statediff.NodeType
-	StateLeafKey common.Hash
-	Path         []byte
-	IPLD         ipfs.BlockModel
+// LogResult represent a log.
+type LogResult struct {
+	LeafCID     string `db:"cid"`
+	ReceiptID   string `db:"rct_id"`
+	Address     string `db:"address"`
+	Index       int64  `db:"index"`
+	Data        []byte `db:"log_data"`
+	Topic0      string `db:"topic0"`
+	Topic1      string `db:"topic1"`
+	Topic2      string `db:"topic2"`
+	Topic3      string `db:"topic3"`
+	LogLeafData []byte `db:"data"`
+	RctCID      string `db:"rct_cid"`
+	RctStatus   uint64 `db:"post_status"`
+	BlockNumber string `db:"block_number"`
+	BlockHash   string `db:"block_hash"`
+	TxnIndex    int64  `db:"txn_index"`
+	TxHash      string `db:"tx_hash"`
 }
 
-type StorageNode struct {
-	Type           statediff.NodeType
-	StateLeafKey   common.Hash
-	StorageLeafKey common.Hash
-	Path           []byte
-	IPLD           ipfs.BlockModel
+// GetSliceResponse holds response for the eth_getSlice method
+type GetSliceResponse struct {
+	SliceID   string                             `json:"sliceId"`
+	MetaData  GetSliceResponseMetadata           `json:"metadata"`
+	TrieNodes GetSliceResponseTrieNodes          `json:"trieNodes"`
+	Leaves    map[string]GetSliceResponseAccount `json:"leaves"` // key: Keccak256Hash(address) in hex (leafKey)
 }
 
-// CIDWrapper is used to direct fetching of IPLDs from IPFS
-// Returned by CIDRetriever
-// Passed to IPLDFetcher
-type CIDWrapper struct {
-	BlockNumber  *big.Int
-	Header       eth.HeaderModel
-	Uncles       []eth.UncleModel
-	Transactions []eth.TxModel
-	Receipts     []eth.ReceiptModel
-	StateNodes   []eth.StateNodeModel
-	StorageNodes []eth.StorageNodeWithStateKeyModel
+func (sr *GetSliceResponse) init(path string, depth int, root common.Hash) {
+	sr.SliceID = fmt.Sprintf("%s-%d-%s", path, depth, root.String())
+	sr.MetaData = GetSliceResponseMetadata{
+		NodeStats: make(map[string]string, 0),
+		TimeStats: make(map[string]string, 0),
+	}
+	sr.Leaves = make(map[string]GetSliceResponseAccount)
+	sr.TrieNodes = GetSliceResponseTrieNodes{
+		Stem:  make(map[string]string),
+		Head:  make(map[string]string),
+		Slice: make(map[string]string),
+	}
+}
+
+func (sr *GetSliceResponse) populateMetaData(metaData metaDataFields) {
+	sr.MetaData.NodeStats["00-stem-and-head-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head))
+	sr.MetaData.NodeStats["01-max-depth"] = strconv.Itoa(metaData.maxDepth)
+	sr.MetaData.NodeStats["02-total-trie-nodes"] = strconv.Itoa(len(sr.TrieNodes.Stem) + len(sr.TrieNodes.Head) + len(sr.TrieNodes.Slice))
+	sr.MetaData.NodeStats["03-leaves"] = strconv.Itoa(metaData.leafCount)
+	sr.MetaData.NodeStats["04-smart-contracts"] = strconv.Itoa(len(sr.Leaves))
+
+	sr.MetaData.TimeStats["00-trie-loading"] = strconv.FormatInt(metaData.trieLoadingTime, 10)
+	sr.MetaData.TimeStats["01-fetch-stem-keys"] = strconv.FormatInt(metaData.stemNodesFetchTime, 10)
+	sr.MetaData.TimeStats["02-fetch-slice-keys"] = strconv.FormatInt(metaData.sliceNodesFetchTime, 10)
+	sr.MetaData.TimeStats["03-fetch-leaves-info"] = strconv.FormatInt(metaData.leavesFetchTime, 10)
+}
+
+type GetSliceResponseMetadata struct {
+	TimeStats map[string]string `json:"timeStats"` // stem, state, storage (one by one)
+	NodeStats map[string]string `json:"nodeStats"` // total, leaves, smart contracts
+}
+
+type GetSliceResponseTrieNodes struct {
+	Stem  map[string]string `json:"stem"` // key: Keccak256Hash(data) in hex, value: trie node data in hex
+	Head  map[string]string `json:"head"`
+	Slice map[string]string `json:"sliceNodes"`
+}
+
+type GetSliceResponseAccount struct {
+	StorageRoot string `json:"storageRoot"`
+	EVMCode     string `json:"evmCode"`
+}
+
+type metaDataFields struct {
+	maxDepth            int
+	leafCount           int
+	trieLoadingTime     int64
+	stemNodesFetchTime  int64
+	sliceNodesFetchTime int64
+	leavesFetchTime     int64
 }

pkg/graphql/client.go

@@ -0,0 +1,274 @@
+package graphql
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	gqlclient "github.com/machinebox/graphql"
+)
+
+type StorageResponse struct {
+	CID       string        `json:"cid"`
+	Value     common.Hash   `json:"value"`
+	IpldBlock hexutil.Bytes `json:"ipldBlock"`
+}
+
+type GetStorageAt struct {
+	Response StorageResponse `json:"getStorageAt"`
+}
+
+type LogResponse struct {
+	Topics      []common.Hash       `json:"topics"`
+	Data        hexutil.Bytes       `json:"data"`
+	Transaction TransactionResponse `json:"transaction"`
+	ReceiptCID  string              `json:"receiptCID"`
+	Status      int32               `json:"status"`
+}
+
+type TransactionResponse struct {
+	Hash common.Hash `json:"hash"`
+}
+
+type GetLogs struct {
+	Responses []LogResponse `json:"getLogs"`
+}
+
+type IPFSBlockResponse struct {
+	Key  string `json:"key"`
+	Data string `json:"data"`
+}
+
+type EthTransactionCIDResponse struct {
+	CID        string            `json:"cid"`
+	TxHash     string            `json:"txHash"`
+	Index      int32             `json:"index"`
+	Src        string            `json:"src"`
+	Dst        string            `json:"dst"`
+	BlockByCid IPFSBlockResponse `json:"blockByCid"`
+}
+
+type EthTransactionCIDByTxHash struct {
+	Response EthTransactionCIDResponse `json:"ethTransactionCidByTxHash"`
+}
+
+type EthTransactionCIDsByHeaderIdResponse struct {
+	Nodes []EthTransactionCIDResponse `json:"nodes"`
+}
+
+type EthHeaderCIDResponse struct {
+	CID                          string                               `json:"cid"`
+	BlockNumber                  BigInt                               `json:"blockNumber"`
+	BlockHash                    string                               `json:"blockHash"`
+	ParentHash                   string                               `json:"parentHash"`
+	Timestamp                    BigInt                               `json:"timestamp"`
+	StateRoot                    string                               `json:"stateRoot"`
+	Td                           BigInt                               `json:"td"`
+	TxRoot                       string                               `json:"txRoot"`
+	ReceiptRoot                  string                               `json:"receiptRoot"`
+	UncleRoot                    string                               `json:"uncleRoot"`
+	Bloom                        string                               `json:"bloom"`
+	EthTransactionCIDsByHeaderId EthTransactionCIDsByHeaderIdResponse `json:"ethTransactionCidsByHeaderId"`
+	BlockByCid                   IPFSBlockResponse                    `json:"blockByCid"`
+}
+
+type AllEthHeaderCIDsResponse struct {
+	Nodes []EthHeaderCIDResponse `json:"nodes"`
+}
+
+type AllEthHeaderCIDs struct {
+	Response AllEthHeaderCIDsResponse `json:"allEthHeaderCids"`
+}
+
+type Client struct {
+	client *gqlclient.Client
+}
+
+func NewClient(endpoint string) *Client {
+	client := gqlclient.NewClient(endpoint)
+	return &Client{client: client}
+}
+
+func (c *Client) GetLogs(ctx context.Context, hash common.Hash, addresses []common.Address) ([]LogResponse, error) {
+	params := fmt.Sprintf(`blockHash: "%s"`, hash.String())
+
+	if addresses != nil {
+		addressStrings := make([]string, len(addresses))
+		for i, address := range addresses {
+			addressStrings[i] = fmt.Sprintf(`"%s"`, address.String())
+		}
+
+		params += fmt.Sprintf(`, addresses: [%s]`, strings.Join(addressStrings, ","))
+	}
+
+	getLogsQuery := fmt.Sprintf(`query{
+			getLogs(%s) {
+				data
+				topics
+				transaction {
+					hash
+				}
+				status
+				receiptCID
+			}
+		}`, params)
+
+	req := gqlclient.NewRequest(getLogsQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var logs GetLogs
+	err = json.Unmarshal(jsonStr, &logs)
+	if err != nil {
+		return nil, err
+	}
+	return logs.Responses, nil
+}
+
+func (c *Client) GetStorageAt(ctx context.Context, hash common.Hash, address common.Address, slot string) (*StorageResponse, error) {
+	getLogsQuery := fmt.Sprintf(`
+		query{
+			getStorageAt(blockHash: "%s", contract: "%s",slot: "%s") {
+				cid
+				value
+				ipldBlock
+			}
+		}
+	`, hash.String(), address.String(), common.HexToHash(slot))
+
+	req := gqlclient.NewRequest(getLogsQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var storageAt GetStorageAt
+	err = json.Unmarshal(jsonStr, &storageAt)
+	if err != nil {
+		return nil, err
+	}
+	return &storageAt.Response, nil
+}
+
+func (c *Client) AllEthHeaderCIDs(ctx context.Context, condition EthHeaderCIDCondition) (*AllEthHeaderCIDsResponse, error) {
+	var params string
+	if condition.BlockHash != nil {
+		params = fmt.Sprintf(`blockHash: "%s"`, *condition.BlockHash)
+	}
+	if condition.BlockNumber != nil {
+		params += fmt.Sprintf(`blockNumber: "%s"`, condition.BlockNumber.String())
+	}
+
+	getHeadersQuery := fmt.Sprintf(`
+		query{
+			allEthHeaderCids(condition: { %s }) {
+				nodes {
+					cid
+					blockNumber
+					blockHash
+					parentHash
+					timestamp
+					stateRoot
+					td
+					txRoot
+					receiptRoot
+					uncleRoot
+					bloom
+					blockByCid {
+						key
+						data
+					}
+					ethTransactionCidsByHeaderId {
+						nodes {
+							cid
+							txHash
+							index
+							src
+							dst
+						}
+					}
+				}
+			}
+		}
+	`, params)
+
+	req := gqlclient.NewRequest(getHeadersQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var allEthHeaderCIDs AllEthHeaderCIDs
+	err = json.Unmarshal(jsonStr, &allEthHeaderCIDs)
+	if err != nil {
+		return nil, err
+	}
+	return &allEthHeaderCIDs.Response, nil
+}
+
+func (c *Client) EthTransactionCIDByTxHash(ctx context.Context, txHash string) (*EthTransactionCIDResponse, error) {
+	getTxQuery := fmt.Sprintf(`
+		query{
+			ethTransactionCidByTxHash(txHash: "%s") {
+				cid
+				txHash
+				index
+				src
+				dst
+				blockByCid {
+					data
+				}
+			}
+		}
+	`, txHash)
+
+	req := gqlclient.NewRequest(getTxQuery)
+	req.Header.Set("Cache-Control", "no-cache")
+
+	var respData map[string]interface{}
+	err := c.client.Run(ctx, req, &respData)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonStr, err := json.Marshal(respData)
+	if err != nil {
+		return nil, err
+	}
+
+	var ethTxCID EthTransactionCIDByTxHash
+	err = json.Unmarshal(jsonStr, &ethTxCID)
+	if err != nil {
+		return nil, err
+	}
+	return &ethTxCID.Response, nil
+}

pkg/graphql/graphql.go

@@ -18,19 +18,24 @@
 package graphql
 
 import (
+	"bytes"
 	"context"
+	"database/sql"
 	"errors"
+	"fmt"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth/filters"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	state "github.com/cerc-io/ipld-eth-statedb/direct_by_leaf"
 )
 
 var (
@@ -46,7 +51,7 @@ type Account struct {
 
 // getState fetches the StateDB object for an account.
 func (a *Account) getState(ctx context.Context) (*state.StateDB, error) {
-	state, _, err := a.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash)
+	state, _, err := a.backend.IPLDDirectStateDBAndHeaderByNumberOrHash(ctx, a.blockNrOrHash)
 	return state, err
 }
 
@@ -91,13 +96,19 @@ type Log struct {
 	backend     *eth.Backend
 	transaction *Transaction
 	log         *types.Log
+	cid         string
+	receiptCID  string
+	ipldBlock   []byte // log leaf node IPLD block data
+	status      uint64
 }
 
-func (l *Log) Transaction(ctx context.Context) *Transaction {
+// Transaction returns transaction that generated this log entry.
+func (l *Log) Transaction(_ context.Context) *Transaction {
 	return l.transaction
 }
 
-func (l *Log) Account(ctx context.Context, args BlockNumberArgs) *Account {
+// Account returns the contract account which generated this log.
+func (l *Log) Account(_ context.Context, args BlockNumberArgs) *Account {
 	return &Account{
 		backend:       l.backend,
 		address:       l.log.Address,
@@ -105,16 +116,39 @@ func (l *Log) Account(ctx context.Context, args BlockNumberArgs) *Account {
 	}
 }
 
-func (l *Log) Index(ctx context.Context) int32 {
+// Index returns the index of this log in the block
+func (l *Log) Index(_ context.Context) int32 {
 	return int32(l.log.Index)
 }
 
-func (l *Log) Topics(ctx context.Context) []common.Hash {
+// Topics returns the list of 0-4 indexed topics for the log.
+func (l *Log) Topics(_ context.Context) []common.Hash {
 	return l.log.Topics
 }
 
-func (l *Log) Data(ctx context.Context) hexutil.Bytes {
-	return hexutil.Bytes(l.log.Data)
+// Data returns data of this log.
+func (l *Log) Data(_ context.Context) hexutil.Bytes {
+	return l.log.Data
+}
+
+// Cid returns cid of the leaf node of this log.
+func (l *Log) Cid(_ context.Context) string {
+	return l.cid
+}
+
+// IpldBlock returns IPLD block of the leaf node of this log.
+func (l *Log) IpldBlock(_ context.Context) hexutil.Bytes {
+	return l.ipldBlock
+}
+
+// Status returns the status of the receipt IPLD block this Log exists in.
+func (l *Log) Status(_ context.Context) int32 {
+	return int32(l.status)
+}
+
+// ReceiptCID returns the receipt CID of the receipt IPLD block this Log exists in.
+func (l *Log) ReceiptCID(_ context.Context) string {
+	return l.receiptCID
 }
 
 // Transaction represents an Ethereum transaction.
@@ -209,10 +243,7 @@ func (t *Transaction) From(ctx context.Context, args BlockNumberArgs) (*Account,
 	if err != nil || tx == nil {
 		return nil, err
 	}
-	var signer types.Signer = types.HomesteadSigner{}
-	if tx.Protected() {
-		signer = types.NewEIP155Signer(tx.ChainId())
-	}
+	signer := eth.SignerForTx(tx)
 	from, _ := types.Sender(signer, tx)
 
 	return &Account{
@@ -742,8 +773,8 @@ func (b *Block) Logs(ctx context.Context, args struct{ Filter BlockFilterCriteri
 		hash = header.Hash()
 	}
 	// Construct the range filter
-	filter := filters.NewBlockFilter(b.backend, hash, addresses, topics)
-
+	filterSys := filters.NewFilterSystem(b.backend, filters.Config{})
+	filter := filterSys.NewBlockFilter(hash, addresses, topics)
 	// Run the filter and return all the logs
 	return runFilter(ctx, b.backend, filter)
 }
@@ -803,16 +834,20 @@ func (b *Block) Call(ctx context.Context, args struct {
 			return nil, err
 		}
 	}
-	result, gas, failed, err := eth.DoCall(ctx, b.backend, args.Data, *b.numberOrHash, nil, 5*time.Second, b.backend.RPCGasCap())
+	result, err := eth.DoCall(ctx, b.backend, args.Data, *b.numberOrHash, nil, 5*time.Second, b.backend.RPCGasCap())
+	if err != nil {
+		return nil, err
+	}
 	status := hexutil.Uint64(1)
-	if failed {
+	if result.Failed() {
 		status = 0
 	}
+
 	return &CallResult{
-		data:    hexutil.Bytes(result),
-		gasUsed: hexutil.Uint64(gas),
+		data:    result.ReturnData,
+		gasUsed: hexutil.Uint64(result.UsedGas),
 		status:  status,
-	}, err
+	}, nil
 }
 
 // Resolver is the top-level object in the GraphQL hierarchy.
@@ -867,7 +902,11 @@ func (r *Resolver) Blocks(ctx context.Context, args struct {
 	if args.To != nil {
 		to = rpc.BlockNumber(*args.To)
 	} else {
-		to = rpc.BlockNumber(r.backend.CurrentBlock().Number().Int64())
+		block, err := r.backend.CurrentBlock()
+		if err != nil {
+			return []*Block{}, nil
+		}
+		to = rpc.BlockNumber(block.Number().Int64())
 	}
 	if to < from {
 		return []*Block{}, nil
@@ -937,6 +976,380 @@ func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria
 		topics = *args.Filter.Topics
 	}
 	// Construct the range filter
-	filter := filters.NewRangeFilter(filters.Backend(r.backend), begin, end, addresses, topics)
+	filterSys := filters.NewFilterSystem(r.backend, filters.Config{})
+	filter := filterSys.NewRangeFilter(begin, end, addresses, topics)
 	return runFilter(ctx, r.backend, filter)
 }
+
+// StorageResult represents a storage slot value. All arguments are mandatory.
+type StorageResult struct {
+	value     []byte
+	cid       string
+	ipldBlock []byte
+}
+
+func (s *StorageResult) Value(ctx context.Context) common.Hash {
+	return common.BytesToHash(s.value)
+}
+
+func (s *StorageResult) Cid(ctx context.Context) string {
+	return s.cid
+}
+
+func (s *StorageResult) IpldBlock(ctx context.Context) hexutil.Bytes {
+	return hexutil.Bytes(s.ipldBlock)
+}
+
+func (r *Resolver) GetStorageAt(ctx context.Context, args struct {
+	BlockHash common.Hash
+	Contract  common.Address
+	Slot      common.Hash
+}) (*StorageResult, error) {
+	cid, nodeRLP, err := r.backend.Retriever.RetrieveStorageAndRLP(args.Contract, args.Slot, args.BlockHash)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return &StorageResult{value: []byte{}, cid: "", ipldBlock: []byte{}}, nil
+		}
+		return nil, err
+	}
+	valueRLP, err := eth.DecodeLeafNode(nodeRLP)
+	if err != nil {
+		return nil, err
+	}
+	if bytes.Equal(valueRLP, eth.EmptyNodeValue) {
+		return &StorageResult{value: eth.EmptyNodeValue, cid: cid, ipldBlock: nodeRLP}, nil
+	}
+	var value interface{}
+	err = rlp.DecodeBytes(valueRLP, &value)
+	if err != nil {
+		return nil, err
+	}
+
+	return &StorageResult{value: value.([]byte), cid: cid, ipldBlock: nodeRLP}, nil
+}
+
+func (r *Resolver) GetLogs(ctx context.Context, args struct {
+	BlockHash   common.Hash
+	BlockNumber *BigInt
+	Addresses   *[]common.Address
+}) (*[]*Log, error) {
+	var filter eth.ReceiptFilter
+
+	if args.Addresses != nil {
+		filter.LogAddresses = make([]string, len(*args.Addresses))
+		for i, address := range *args.Addresses {
+			filter.LogAddresses[i] = address.String()
+		}
+	}
+
+	// Begin tx
+	tx, err := r.backend.DB.Beginx()
+	if err != nil {
+		return nil, err
+	}
+
+	filteredLogs, err := r.backend.Retriever.RetrieveFilteredGQLLogs(tx, filter, &args.BlockHash, args.BlockNumber.ToInt())
+	if err != nil {
+		return nil, err
+	}
+
+	if err = tx.Commit(); err != nil {
+		return nil, err
+	}
+
+	rctLog := decomposeGQLLogs(filteredLogs)
+	if err != nil {
+		return nil, err
+	}
+
+	ret := make([]*Log, 0, 10)
+	for _, l := range rctLog {
+		ret = append(ret, &Log{
+			backend:    r.backend,
+			log:        l.Log,
+			cid:        l.CID,
+			receiptCID: l.RctCID,
+			ipldBlock:  l.LogLeafData,
+			transaction: &Transaction{
+				hash: l.Log.TxHash,
+			},
+			status: l.RctStatus,
+		})
+	}
+
+	return &ret, nil
+}
+
+type logsCID struct {
+	Log         *types.Log
+	CID         string
+	RctCID      string
+	LogLeafData []byte
+	RctStatus   uint64
+}
+
+// decomposeGQLLogs return logs for graphql.
+func decomposeGQLLogs(logCIDs []eth.LogResult) []logsCID {
+	logs := make([]logsCID, len(logCIDs))
+	for i, l := range logCIDs {
+		topics := make([]common.Hash, 0)
+		if l.Topic0 != "" {
+			topics = append(topics, common.HexToHash(l.Topic0))
+		}
+		if l.Topic1 != "" {
+			topics = append(topics, common.HexToHash(l.Topic1))
+		}
+		if l.Topic2 != "" {
+			topics = append(topics, common.HexToHash(l.Topic2))
+		}
+		if l.Topic3 != "" {
+			topics = append(topics, common.HexToHash(l.Topic3))
+		}
+
+		logs[i] = logsCID{
+			Log: &types.Log{
+				Address: common.HexToAddress(l.Address),
+				Topics:  topics,
+				Data:    l.Data,
+				Index:   uint(l.Index),
+				TxHash:  common.HexToHash(l.TxHash),
+			},
+			CID:         l.LeafCID,
+			RctCID:      l.RctCID,
+			LogLeafData: l.LogLeafData,
+			RctStatus:   l.RctStatus,
+		}
+	}
+
+	return logs
+}
+
+type EthTransactionCID struct {
+	cid       string
+	txHash    string
+	index     int32
+	src       string
+	dst       string
+	ipfsBlock IPFSBlock
+}
+
+func (t EthTransactionCID) Cid(ctx context.Context) string {
+	return t.cid
+}
+
+func (t EthTransactionCID) TxHash(ctx context.Context) string {
+	return t.txHash
+}
+
+func (t EthTransactionCID) Index(ctx context.Context) int32 {
+	return t.index
+}
+
+func (t EthTransactionCID) Src(ctx context.Context) string {
+	return t.src
+}
+
+func (t EthTransactionCID) Dst(ctx context.Context) string {
+	return t.dst
+}
+
+func (t EthTransactionCID) BlockByCid(ctx context.Context) IPFSBlock {
+	return t.ipfsBlock
+}
+
+type EthTransactionCIDsConnection struct {
+	nodes []*EthTransactionCID
+}
+
+func (transactionCIDResult EthTransactionCIDsConnection) Nodes(ctx context.Context) []*EthTransactionCID {
+	return transactionCIDResult.nodes
+}
+
+type IPFSBlock struct {
+	key  string
+	data string
+}
+
+func (b IPFSBlock) Key(ctx context.Context) string {
+	return b.key
+}
+
+func (b IPFSBlock) Data(ctx context.Context) string {
+	return b.data
+}
+
+type EthHeaderCID struct {
+	cid          string
+	blockNumber  BigInt
+	blockHash    string
+	parentHash   string
+	timestamp    BigInt
+	stateRoot    string
+	td           BigInt
+	txRoot       string
+	receiptRoot  string
+	uncleRoot    string
+	bloom        string
+	transactions []*EthTransactionCID
+	ipfsBlock    IPFSBlock
+}
+
+func (h EthHeaderCID) Cid(ctx context.Context) string {
+	return h.cid
+}
+
+func (h EthHeaderCID) BlockNumber(ctx context.Context) BigInt {
+	return h.blockNumber
+}
+
+func (h EthHeaderCID) BlockHash(ctx context.Context) string {
+	return h.blockHash
+}
+
+func (h EthHeaderCID) ParentHash(ctx context.Context) string {
+	return h.parentHash
+}
+
+func (h EthHeaderCID) Timestamp(ctx context.Context) BigInt {
+	return h.timestamp
+}
+
+func (h EthHeaderCID) StateRoot(ctx context.Context) string {
+	return h.stateRoot
+}
+
+func (h EthHeaderCID) Td(ctx context.Context) BigInt {
+	return h.td
+}
+
+func (h EthHeaderCID) TxRoot(ctx context.Context) string {
+	return h.txRoot
+}
+
+func (h EthHeaderCID) ReceiptRoot(ctx context.Context) string {
+	return h.receiptRoot
+}
+
+func (h EthHeaderCID) UncleRoot(ctx context.Context) string {
+	return h.uncleRoot
+}
+
+func (h EthHeaderCID) Bloom(ctx context.Context) string {
+	return h.bloom
+}
+
+func (h EthHeaderCID) EthTransactionCidsByHeaderId(ctx context.Context) EthTransactionCIDsConnection {
+	return EthTransactionCIDsConnection{nodes: h.transactions}
+}
+
+func (h EthHeaderCID) BlockByCid(ctx context.Context) IPFSBlock {
+	return h.ipfsBlock
+}
+
+type EthHeaderCIDsConnection struct {
+	nodes []*EthHeaderCID
+}
+
+func (headerCIDResult EthHeaderCIDsConnection) Nodes(ctx context.Context) []*EthHeaderCID {
+	return headerCIDResult.nodes
+}
+
+type EthHeaderCIDCondition struct {
+	BlockNumber *BigInt
+	BlockHash   *string
+}
+
+func (r *Resolver) AllEthHeaderCids(ctx context.Context, args struct {
+	Condition *EthHeaderCIDCondition
+}) (*EthHeaderCIDsConnection, error) {
+	var headerCIDs []eth.HeaderCIDRecord
+	var err error
+	if args.Condition.BlockHash != nil {
+		headerCID, err := r.backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(common.HexToHash(*args.Condition.BlockHash), args.Condition.BlockNumber.ToInt())
+		if err != nil {
+			if !strings.Contains(err.Error(), "not found") {
+				return nil, err
+			}
+		} else {
+			headerCIDs = append(headerCIDs, headerCID)
+		}
+	} else if args.Condition.BlockNumber != nil {
+		headerCIDs, err = r.backend.Retriever.RetrieveHeaderAndTxCIDsByBlockNumber(args.Condition.BlockNumber.ToInt().Int64())
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		return nil, fmt.Errorf("provide block number or block hash")
+	}
+
+	var resultNodes []*EthHeaderCID
+	for _, headerCID := range headerCIDs {
+		var blockNumber BigInt
+		blockNumber.UnmarshalText([]byte(headerCID.BlockNumber))
+
+		var timestamp BigInt
+		timestamp.SetUint64(headerCID.Timestamp)
+
+		var td BigInt
+		td.UnmarshalText([]byte(headerCID.TotalDifficulty))
+
+		ethHeaderCIDNode := EthHeaderCID{
+			cid:         headerCID.CID,
+			blockNumber: blockNumber,
+			blockHash:   headerCID.BlockHash,
+			parentHash:  headerCID.ParentHash,
+			timestamp:   timestamp,
+			stateRoot:   headerCID.StateRoot,
+			td:          td,
+			txRoot:      headerCID.TxRoot,
+			receiptRoot: headerCID.RctRoot,
+			uncleRoot:   headerCID.UnclesHash,
+			bloom:       Bytes(headerCID.Bloom).String(),
+			ipfsBlock: IPFSBlock{
+				key:  headerCID.IPLD.Key,
+				data: Bytes(headerCID.IPLD.Data).String(),
+			},
+		}
+
+		for _, txCID := range headerCID.TransactionCIDs {
+			ethHeaderCIDNode.transactions = append(ethHeaderCIDNode.transactions, &EthTransactionCID{
+				cid:    txCID.CID,
+				txHash: txCID.TxHash,
+				index:  int32(txCID.Index),
+				src:    txCID.Src,
+				dst:    txCID.Dst,
+			})
+		}
+
+		resultNodes = append(resultNodes, &ethHeaderCIDNode)
+	}
+
+	return &EthHeaderCIDsConnection{
+		nodes: resultNodes,
+	}, nil
+}
+
+func (r *Resolver) EthTransactionCidByTxHash(ctx context.Context, args struct {
+	TxHash      string
+	BlockNumber *BigInt
+}) (*EthTransactionCID, error) {
+	// Need not check args.BlockNumber for nil as .ToInt() uses a pointer receiver and returns nil if BlockNumber is nil
+	// https://stackoverflow.com/questions/42238624/calling-a-method-on-a-nil-struct-pointer-doesnt-panic-why-not
+	txCID, err := r.backend.Retriever.RetrieveTxCIDByHash(args.TxHash, args.BlockNumber.ToInt())
+
+	if err != nil {
+		return nil, err
+	}
+
+	return &EthTransactionCID{
+		cid:    txCID.CID,
+		txHash: txCID.TxHash,
+		index:  int32(txCID.Index),
+		src:    txCID.Src,
+		dst:    txCID.Dst,
+		ipfsBlock: IPFSBlock{
+			key:  txCID.IPLD.Key,
+			data: Bytes(txCID.IPLD.Data).String(),
+		},
+	}, nil
+}

pkg/graphql/graphql_suite_test.go

@@ -17,19 +17,13 @@
 package graphql_test
 
 import (
-	"io/ioutil"
 	"testing"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"github.com/sirupsen/logrus"
 )
 
 func TestGraphQL(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "graphql test suite")
 }
-
-var _ = BeforeSuite(func() {
-	logrus.SetOutput(ioutil.Discard)
-})

pkg/graphql/graphql_test.go

@@ -17,15 +17,338 @@
 package graphql_test
 
 import (
-	. "github.com/onsi/ginkgo"
+	"context"
+	"fmt"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/jmoiron/sqlx"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"math/big"
+	"strconv"
 
-	"github.com/vulcanize/ipld-eth-server/pkg/graphql"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth/test_helpers"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/graphql"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
 )
 
-var _ = Describe("GraphQL", func() {
-	It("Builds the schema and creates a new handler", func() {
-		_, err := graphql.NewHandler(nil)
+const (
+	gqlEndPoint = "127.0.0.1:8083"
+)
+
+var (
+	randomAddr              = common.HexToAddress("0x1C3ab14BBaD3D99F4203bd7a11aCB94882050E6f")
+	randomHash              = crypto.Keccak256Hash(randomAddr.Bytes())
+	blocks                  []*types.Block
+	receipts                []types.Receipts
+	chain                   *core.BlockChain
+	db                      *sqlx.DB
+	backend                 *eth.Backend
+	graphQLServer           *graphql.Service
+	chainConfig             = &*params.TestChainConfig
+	client                  = graphql.NewClient(fmt.Sprintf("http://%s/graphql", gqlEndPoint))
+	mockTD                  = big.NewInt(1337)
+	ctx                     = context.Background()
+	nonCanonBlockHash       common.Hash
+	nonCanonContractAddress common.Address
+)
+
+var _ = BeforeSuite(func() {
+	var err error
+	db = shared.SetupDB()
+
+	backend, err = eth.NewEthBackend(db, &eth.Config{
+		ChainConfig: chainConfig,
+		VMConfig:    vm.Config{},
+		RPCGasCap:   big.NewInt(10000000000),
+		GroupCacheConfig: &shared.GroupCacheConfig{
+			StateDB: shared.GroupConfig{
+				Name:                   "graphql_test",
+				CacheSizeInMB:          8,
+				CacheExpiryInMins:      60,
+				LogStatsIntervalInSecs: 0,
+			},
+		},
+	})
+	Expect(err).ToNot(HaveOccurred())
+
+	// make the test blockchain (and state)
+	chainConfig.LondonBlock = big.NewInt(100)
+	blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen, chainConfig)
+	indexer := shared.SetupTestStateDiffIndexer(context.Background(), chainConfig, test_helpers.Genesis.Hash())
+	builder := statediff.NewBuilder(chain.StateCache())
+
+	// Insert some non-canonical data into the database so that we test our ability to discern canonicity
+	nonCanonBlockHash = test_helpers.MockBlock.Hash()
+	nonCanonContractAddress = test_helpers.ContractAddr
+
+	tx, err := indexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
+	Expect(err).ToNot(HaveOccurred())
+
+	err = tx.Submit(err)
+	Expect(err).ToNot(HaveOccurred())
+
+	// The non-canonical header has a child
+	tx, err = indexer.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
+	Expect(err).ToNot(HaveOccurred())
+
+	ipld := sdtypes.IPLD{
+		CID:     ipld.Keccak256ToCid(ipld.RawBinary, test_helpers.CodeHash.Bytes()).String(),
+		Content: test_helpers.ContractCode,
+	}
+	err = indexer.PushIPLD(tx, ipld)
+	Expect(err).ToNot(HaveOccurred())
+
+	err = tx.Submit(err)
+	Expect(err).ToNot(HaveOccurred())
+
+	// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
+	for i, block := range blocks {
+		var args statediff.Args
+		var rcts types.Receipts
+		if i == 0 {
+			args = statediff.Args{
+				OldStateRoot: common.Hash{},
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+		} else {
+			args = statediff.Args{
+				OldStateRoot: blocks[i-1].Root(),
+				NewStateRoot: block.Root(),
+				BlockNumber:  block.Number(),
+				BlockHash:    block.Hash(),
+			}
+			rcts = receipts[i-1]
+		}
+
+		tx, err := indexer.PushBlock(block, rcts, mockTD)
 		Expect(err).ToNot(HaveOccurred())
+
+		diff, err := builder.BuildStateDiffObject(args, statediff.Params{})
+		Expect(err).ToNot(HaveOccurred())
+
+		for _, node := range diff.Nodes {
+			err = indexer.PushStateNode(tx, node, block.Hash().String())
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		for _, ipld := range diff.IPLDs {
+			err = indexer.PushIPLD(tx, ipld)
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		err = tx.Submit(err)
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	graphQLServer, err = graphql.New(backend, gqlEndPoint, nil, []string{"*"}, rpc.HTTPTimeouts{})
+	Expect(err).ToNot(HaveOccurred())
+
+	err = graphQLServer.Start(nil)
+	Expect(err).ToNot(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+	err := graphQLServer.Stop()
+	Expect(err).ToNot(HaveOccurred())
+	shared.TearDownDB(db)
+	chain.Stop()
+})
+
+var _ = Describe("GraphQL", func() {
+	Describe("eth_getLogs", func() {
+		It("Retrieves logs that matches the provided blockHash and contract address", func() {
+			logs, err := client.GetLogs(ctx, nonCanonBlockHash, []common.Address{nonCanonContractAddress})
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedLogs := []graphql.LogResponse{
+				{
+					Topics:      test_helpers.MockLog1.Topics,
+					Data:        hexutil.Bytes(test_helpers.MockLog1.Data),
+					Transaction: graphql.TransactionResponse{Hash: test_helpers.MockTransactions[0].Hash()},
+					ReceiptCID:  test_helpers.Rct1CID.String(),
+					Status:      int32(test_helpers.MockReceipts[0].Status),
+				},
+			}
+
+			Expect(logs).To(Equal(expectedLogs))
+		})
+
+		It("Retrieves logs for the failed receipt status that matches the provided blockHash and another contract address", func() {
+			logs, err := client.GetLogs(ctx, nonCanonBlockHash, []common.Address{test_helpers.AnotherAddress2})
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedLogs := []graphql.LogResponse{
+				{
+					Topics:      test_helpers.MockLog6.Topics,
+					Data:        hexutil.Bytes(test_helpers.MockLog6.Data),
+					Transaction: graphql.TransactionResponse{Hash: test_helpers.MockTransactions[3].Hash()},
+					ReceiptCID:  test_helpers.Rct4CID.String(),
+					Status:      int32(test_helpers.MockReceipts[3].Status),
+				},
+			}
+
+			Expect(logs).To(Equal(expectedLogs))
+		})
+
+		It("Retrieves logs that matches the provided blockHash and multiple contract addresses", func() {
+			logs, err := client.GetLogs(ctx, nonCanonBlockHash, []common.Address{nonCanonContractAddress, test_helpers.AnotherAddress2})
+			Expect(err).ToNot(HaveOccurred())
+
+			expectedLogs := []graphql.LogResponse{
+				{
+					Topics:      test_helpers.MockLog1.Topics,
+					Data:        hexutil.Bytes(test_helpers.MockLog1.Data),
+					Transaction: graphql.TransactionResponse{Hash: test_helpers.MockTransactions[0].Hash()},
+					ReceiptCID:  test_helpers.Rct1CID.String(),
+					Status:      int32(test_helpers.MockReceipts[0].Status),
+				},
+				{
+					Topics:      test_helpers.MockLog6.Topics,
+					Data:        hexutil.Bytes(test_helpers.MockLog6.Data),
+					Transaction: graphql.TransactionResponse{Hash: test_helpers.MockTransactions[3].Hash()},
+					ReceiptCID:  test_helpers.Rct4CID.String(),
+					Status:      int32(test_helpers.MockReceipts[3].Status),
+				},
+			}
+
+			Expect(logs).To(Equal(expectedLogs))
+		})
+
+		It("Retrieves all the logs for the receipt that matches the provided blockHash and nil contract address", func() {
+			logs, err := client.GetLogs(ctx, nonCanonBlockHash, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(6))
+		})
+
+		It("Retrieves logs with random hash", func() {
+			logs, err := client.GetLogs(ctx, randomHash, []common.Address{nonCanonContractAddress})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(logs)).To(Equal(0))
+		})
+	})
+
+	Describe("eth_getStorageAt", func() {
+		It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash", func() {
+			storageRes, err := client.GetStorageAt(ctx, blocks[2].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.HexToHash("01")))
+
+			storageRes, err = client.GetStorageAt(ctx, blocks[3].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.HexToHash("03")))
+
+			storageRes, err = client.GetStorageAt(ctx, blocks[4].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.HexToHash("09")))
+		})
+
+		It("Retrieves empty data if it tries to access a contract at a blockHash which does not exist", func() {
+			storageRes, err := client.GetStorageAt(ctx, blocks[0].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.Hash{}))
+
+			storageRes, err = client.GetStorageAt(ctx, blocks[1].Hash(), nonCanonContractAddress, test_helpers.IndexOne)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.Hash{}))
+		})
+
+		It("Retrieves empty data if it tries to access a contract slot which does not exist", func() {
+			storageRes, err := client.GetStorageAt(ctx, blocks[3].Hash(), nonCanonContractAddress, randomHash.Hex())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(storageRes.Value).To(Equal(common.Hash{}))
+		})
+	})
+
+	Describe("allEthHeaderCids", func() {
+		It("Retrieves header_cids that matches the provided blockNumber", func() {
+			allEthHeaderCIDsResp, err := client.AllEthHeaderCIDs(ctx, graphql.EthHeaderCIDCondition{BlockNumber: new(graphql.BigInt).SetUint64(2)})
+			Expect(err).ToNot(HaveOccurred())
+
+			headerCIDs, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockNumber(2)
+			Expect(err).ToNot(HaveOccurred())
+
+			for idx, headerCID := range headerCIDs {
+				ethHeaderCID := allEthHeaderCIDsResp.Nodes[idx]
+
+				compareEthHeaderCID(ethHeaderCID, headerCID)
+			}
+		})
+
+		It("Retrieves header_cids that matches the provided blockHash", func() {
+			blockHash := blocks[1].Hash().String()
+			allEthHeaderCIDsResp, err := client.AllEthHeaderCIDs(ctx, graphql.EthHeaderCIDCondition{BlockHash: &blockHash})
+			Expect(err).ToNot(HaveOccurred())
+
+			headerCID, err := backend.Retriever.RetrieveHeaderAndTxCIDsByBlockHash(blocks[1].Hash(), nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(allEthHeaderCIDsResp.Nodes)).To(Equal(1))
+			ethHeaderCID := allEthHeaderCIDsResp.Nodes[0]
+			compareEthHeaderCID(ethHeaderCID, headerCID)
+		})
+	})
+
+	Describe("ethTransactionCidByTxHash", func() {
+		It("Retrieves tx_cid that matches the provided txHash", func() {
+			txHash := blocks[2].Transactions()[0].Hash().String()
+			ethTransactionCIDResp, err := client.EthTransactionCIDByTxHash(ctx, txHash)
+			Expect(err).ToNot(HaveOccurred())
+
+			txCID, err := backend.Retriever.RetrieveTxCIDByHash(txHash, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			compareEthTxCID(*ethTransactionCIDResp, txCID)
+
+			Expect(ethTransactionCIDResp.BlockByCid.Data).To(Equal(graphql.Bytes(txCID.IPLD.Data).String()))
+		})
 	})
 })
+
+func compareEthHeaderCID(ethHeaderCID graphql.EthHeaderCIDResponse, headerCID eth.HeaderCIDRecord) {
+	blockNumber, err := strconv.ParseInt(headerCID.BlockNumber, 10, 64)
+	Expect(err).ToNot(HaveOccurred())
+
+	td, err := strconv.ParseInt(headerCID.TotalDifficulty, 10, 64)
+	Expect(err).ToNot(HaveOccurred())
+
+	Expect(ethHeaderCID.CID).To(Equal(headerCID.CID))
+	Expect(ethHeaderCID.BlockNumber).To(Equal(*new(graphql.BigInt).SetUint64(uint64(blockNumber))))
+	Expect(ethHeaderCID.BlockHash).To(Equal(headerCID.BlockHash))
+	Expect(ethHeaderCID.ParentHash).To(Equal(headerCID.ParentHash))
+	Expect(ethHeaderCID.Timestamp).To(Equal(*new(graphql.BigInt).SetUint64(headerCID.Timestamp)))
+	Expect(ethHeaderCID.StateRoot).To(Equal(headerCID.StateRoot))
+	Expect(ethHeaderCID.Td).To(Equal(*new(graphql.BigInt).SetUint64(uint64(td))))
+	Expect(ethHeaderCID.TxRoot).To(Equal(headerCID.TxRoot))
+	Expect(ethHeaderCID.ReceiptRoot).To(Equal(headerCID.RctRoot))
+	Expect(ethHeaderCID.UncleRoot).To(Equal(headerCID.UnclesHash))
+	Expect(ethHeaderCID.Bloom).To(Equal(graphql.Bytes(headerCID.Bloom).String()))
+
+	for tIdx, txCID := range headerCID.TransactionCIDs {
+		ethTxCID := ethHeaderCID.EthTransactionCIDsByHeaderId.Nodes[tIdx]
+		compareEthTxCID(ethTxCID, txCID)
+	}
+
+	Expect(ethHeaderCID.BlockByCid.Data).To(Equal(graphql.Bytes(headerCID.IPLD.Data).String()))
+	Expect(ethHeaderCID.BlockByCid.Key).To(Equal(headerCID.IPLD.Key))
+}
+
+func compareEthTxCID(ethTxCID graphql.EthTransactionCIDResponse, txCID eth.TransactionCIDRecord) {
+	Expect(ethTxCID.CID).To(Equal(txCID.CID))
+	Expect(ethTxCID.TxHash).To(Equal(txCID.TxHash))
+	Expect(ethTxCID.Index).To(Equal(int32(txCID.Index)))
+	Expect(ethTxCID.Src).To(Equal(txCID.Src))
+	Expect(ethTxCID.Dst).To(Equal(txCID.Dst))
+}

pkg/graphql/schema.go

@@ -25,8 +25,7 @@ const schema string = `
     # An empty byte string is represented as '0x'. Byte strings must have an even number of hexadecimal nybbles.
     scalar Bytes
     # BigInt is a large integer. Input is accepted as either a JSON number or as a string.
-    # Strings may be either decimal or 0x-prefixed hexadecimal. Output values are all
-    # 0x-prefixed hexadecimal.
+    # Input and output strings may be either decimal or 0x-prefixed hexadecimal depending upon the resolver implementation.
     scalar BigInt
     # Long is a 64 bit unsigned integer.
     scalar Long
@@ -65,7 +64,19 @@ const schema string = `
         # Data is unindexed data for this log.
         data: Bytes!
         # Transaction is the transaction that generated this log entry.
-        transaction: Transaction!
+        transaction: Transaction
+
+        # CID for the leaf node IPLD block of the log.
+        cid: String!
+
+        # ReceiptCID for the Receipt IPLD block this Log exists in.
+        receiptCID: String!
+
+        # IPLD block data for the Log Leaf node.
+        ipldBlock: Bytes!
+
+        # Status of the Receipt IPLD block this Log exists in.
+        status: Int!
     }
 
     # Transaction is an Ethereum transaction.
@@ -126,16 +137,16 @@ const schema string = `
         # empty, results will not be filtered by address.
         addresses: [Address!]
         # Topics list restricts matches to particular event topics. Each event has a list
-      # of topics. Topics matches a prefix of that list. An empty element array matches any
-      # topic. Non-empty elements represent an alternative that matches any of the
-      # contained topics.
-      #
-      # Examples:
-      #  - [] or nil          matches any topic list
-      #  - [[A]]              matches topic A in first position
-      #  - [[], [B]]          matches any topic in first position, B in second position
-      #  - [[A], [B]]         matches topic A in first position, B in second position
-      #  - [[A, B]], [C, D]]  matches topic (A OR B) in first position, (C OR D) in second position
+        # of topics. Topics matches a prefix of that list. An empty element array matches any
+        # topic. Non-empty elements represent an alternative that matches any of the
+        # contained topics.
+        #
+        # Examples:
+        #  - [] or nil          matches any topic list
+        #  - [[A]]              matches topic A in first position
+        #  - [[], [B]]          matches any topic in first position, B in second position
+        #  - [[A], [B]]         matches topic A in first position, B in second position
+        #  - [[A, B]], [C, D]]  matches topic (A OR B) in first position, (C OR D) in second position
         topics: [[Bytes32!]!]
     }
 
@@ -246,29 +257,98 @@ const schema string = `
         # empty, results will not be filtered by address.
         addresses: [Address!]
         # Topics list restricts matches to particular event topics. Each event has a list
-      # of topics. Topics matches a prefix of that list. An empty element array matches any
-      # topic. Non-empty elements represent an alternative that matches any of the
-      # contained topics.
-      #
-      # Examples:
-      #  - [] or nil          matches any topic list
-      #  - [[A]]              matches topic A in first position
-      #  - [[], [B]]          matches any topic in first position, B in second position
-      #  - [[A], [B]]         matches topic A in first position, B in second position
-      #  - [[A, B]], [C, D]]  matches topic (A OR B) in first position, (C OR D) in second position
+        # of topics. Topics matches a prefix of that list. An empty element array matches any
+        # topic. Non-empty elements represent an alternative that matches any of the
+        # contained topics.
+        #
+        # Examples:
+        #  - [] or nil          matches any topic list
+        #  - [[A]]              matches topic A in first position
+        #  - [[], [B]]          matches any topic in first position, B in second position
+        #  - [[A], [B]]         matches topic A in first position, B in second position
+        #  - [[A, B]], [C, D]]  matches topic (A OR B) in first position, (C OR D) in second position
         topics: [[Bytes32!]!]
     }
 
+    # Storage trie value with IPLD data.
+    type StorageResult {
+        value: Bytes32!
+
+        # CID for the storage trie IPLD block.
+        cid: String!
+
+        # Storage trie IPLD block.
+        ipldBlock: Bytes!
+    }
+
+    input EthHeaderCidCondition {
+        blockNumber: BigInt
+        blockHash: String
+    }
+
+    type EthTransactionCid {
+        cid: String!
+        txHash: String!
+        index: Int!
+        src: String!
+        dst: String!
+        blockByCid: IPFSBlock!
+    }
+
+    type EthTransactionCidsConnection {
+        nodes: [EthTransactionCid]!
+    }
+
+    type IPFSBlock {
+        key: String!
+        data: String!
+    }
+
+    type EthHeaderCid {
+        cid: String!
+        blockNumber: BigInt!
+        blockHash: String!
+        parentHash: String!
+        timestamp: BigInt!
+        stateRoot: String!
+        td: BigInt!
+        txRoot: String!
+        receiptRoot: String!
+        uncleRoot: String!
+        bloom: String!
+        ethTransactionCidsByHeaderId: EthTransactionCidsConnection!
+        blockByCid: IPFSBlock!
+    }
+
+    type EthHeaderCidsConnection {
+        nodes: [EthHeaderCid]!
+    }
+
     type Query {
         # Block fetches an Ethereum block by number or by hash. If neither is
         # supplied, the most recent known block is returned.
         block(number: Long, hash: Bytes32): Block
+
         # Blocks returns all the blocks between two numbers, inclusive. If
         # to is not supplied, it defaults to the most recent known block.
         blocks(from: Long!, to: Long): [Block!]!
+
         # Transaction returns a transaction specified by its hash.
         transaction(hash: Bytes32!): Transaction
+
         # Logs returns log entries matching the provided filter.
         logs(filter: FilterCriteria!): [Log!]!
+
+        # Get storage slot by block hash and contract address.
+        getStorageAt(blockHash: Bytes32!, contract: Address!, slot: Bytes32!): StorageResult
+
+        # Get contract logs by block hash and contract address.
+        getLogs(blockHash: Bytes32!, blockNumber: BigInt, addresses: [Address!]): [Log!]
+
+        # PostGraphile alternative to get headers with transactions using block number or block hash.
+        allEthHeaderCids(condition: EthHeaderCidCondition): EthHeaderCidsConnection
+
+        # PostGraphile alternative to get transactions using transaction hash.
+        ethTransactionCidByTxHash(txHash: String!, blockNumber: BigInt): EthTransactionCid
     }
 `

pkg/graphql/service.go

@@ -21,13 +21,15 @@ import (
 	"net"
 	"net/http"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/graph-gophers/graphql-go"
 	"github.com/graph-gophers/graphql-go/relay"
-	"github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
 )
 
 // Service encapsulates a GraphQL service.
@@ -66,15 +68,20 @@ func (s *Service) Start(server *p2p.Server) error {
 	if err != nil {
 		return err
 	}
-	if s.listener, err = net.Listen("tcp", s.endpoint); err != nil {
-		return err
+
+	handler := node.NewHTTPHandlerStack(s.handler, s.cors, s.vhosts, nil)
+
+	// start http server
+	_, addr, err := node.StartHTTPEndpoint(s.endpoint, rpc.DefaultHTTPTimeouts, handler)
+	if err != nil {
+		utils.Fatalf("Could not start RPC api: %v", err)
 	}
-	go rpc.NewHTTPServer(s.cors, s.vhosts, s.timeouts, s.handler).Serve(s.listener)
-	logrus.Debugf("graphQL endpoint opened for url %s", fmt.Sprintf("http://%s", s.endpoint))
+	extapiURL := fmt.Sprintf("http://%v", addr)
+	log.Infof("GraphQL endpoint opened at %s", extapiURL)
 	return nil
 }
 
-// newHandler returns a new `http.Handler` that will answer GraphQL queries.
+// NewHandler returns a new `http.Handler` that will answer GraphQL queries.
 // It additionally exports an interactive query browser on the / endpoint.
 func NewHandler(backend *eth.Backend) (http.Handler, error) {
 	q := Resolver{backend}
@@ -98,7 +105,7 @@ func (s *Service) Stop() error {
 	if s.listener != nil {
 		s.listener.Close()
 		s.listener = nil
-		logrus.Debugf("graphQL endpoint closed for url %s", fmt.Sprintf("http://%s", s.endpoint))
+		log.Debugf("graphQL endpoint closed for url %s", fmt.Sprintf("http://%s", s.endpoint))
 	}
 	return nil
 }

pkg/graphql/types.go

@@ -0,0 +1,122 @@
+// VulcanizeDB
+// Copyright © 2022 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+import (
+	"encoding/hex"
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+// Bytes marshals as a JSON string with \x prefix.
+// The empty slice marshals as "\x".
+type Bytes []byte
+
+// MarshalText implements encoding.TextMarshaler
+func (b Bytes) MarshalText() ([]byte, error) {
+	result := make([]byte, len(b)*2+2)
+	copy(result, `\x`)
+	hex.Encode(result[2:], b)
+	return result, nil
+}
+
+// String returns the hex encoding of b.
+func (b Bytes) String() string {
+	return b.encode()
+}
+
+// Encode encodes b as a hex string with "\x" prefix.
+// This is to make the output to be the same as given by postgraphile.
+// graphql-go prepends another "\" to the output resulting in prefix "\\x".
+func (b Bytes) encode() string {
+	result := make([]byte, len(b)*2+2)
+	copy(result, `\x`)
+	hex.Encode(result[2:], b)
+	return string(result)
+}
+
+type BigInt big.Int
+
+// ToInt converts b to a big.Int.
+func (b *BigInt) ToInt() *big.Int {
+	return (*big.Int)(b)
+}
+
+// String returns value of b as a decimal string.
+func (b *BigInt) String() string {
+	return b.ToInt().String()
+}
+
+// SetUint64 sets b to x and returns x.
+func (b *BigInt) SetUint64(x uint64) *BigInt {
+	var val big.Int
+	val.SetUint64(x)
+	*b = (BigInt)(val)
+	return b
+}
+
+// MarshalText implements encoding.TextMarshaler
+func (b BigInt) MarshalText() ([]byte, error) {
+	return []byte(b.String()), nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler
+func (b *BigInt) UnmarshalText(input []byte) error {
+	raw, err := checkNumberText(input)
+	if err != nil {
+		return err
+	}
+	if len(raw) > 64 {
+		return hexutil.ErrBig256Range
+	}
+
+	var val big.Int
+	val.SetString(string(input[:]), 10)
+	*b = (BigInt)(val)
+	return nil
+}
+
+// ImplementsGraphQLType returns true if BigInt implements the provided GraphQL type.
+func (b BigInt) ImplementsGraphQLType(name string) bool { return name == "BigInt" }
+
+// UnmarshalGraphQL unmarshals the provided GraphQL query data.
+func (b *BigInt) UnmarshalGraphQL(input interface{}) error {
+	var err error
+	switch input := input.(type) {
+	case string:
+		return b.UnmarshalText([]byte(input))
+	case int32:
+		var num big.Int
+		num.SetInt64(int64(input))
+		*b = BigInt(num)
+	default:
+		err = fmt.Errorf("unexpected type %T for BigInt", input)
+	}
+	return err
+}
+
+func checkNumberText(input []byte) (raw []byte, err error) {
+	if len(input) == 0 {
+		return nil, nil // empty strings are allowed
+	}
+	if len(input) > 1 && input[0] == '0' {
+		return nil, hexutil.ErrLeadingZero
+	}
+	return input, nil
+}

pkg/log/log.go

@@ -0,0 +1,244 @@
+// Copyright © 2023 Cerc
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package log
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+)
+
+const (
+	CtxKeyApiMethod   = "api_method"
+	CtxKeyApiParams   = "api_params"
+	CtxKeyApiReqId    = "api_reqid"
+	CtxKeyBlockHash   = "block_hash"
+	CtxKeyBlockNumber = "block_num"
+	CtxKeyConn        = "conn"
+	CtxKeyDuration    = "duration"
+	CtxKeyError       = "err"
+	CtxKeyUniqId      = "uuid"
+	CtxKeyUserId      = "user_id"
+)
+
+// TODO: Allow registering arbitrary keys.
+var registeredKeys = []string{
+	CtxKeyApiMethod,
+	CtxKeyApiParams,
+	CtxKeyApiReqId,
+	CtxKeyBlockHash,
+	CtxKeyBlockNumber,
+	CtxKeyConn,
+	CtxKeyDuration,
+	CtxKeyError,
+	CtxKeyUniqId,
+	CtxKeyUserId,
+}
+
+const FatalLevel = logrus.FatalLevel
+const ErrorLevel = logrus.ErrorLevel
+const InfoLevel = logrus.InfoLevel
+const DebugLevel = logrus.DebugLevel
+const TraceLevel = logrus.TraceLevel
+
+type Entry = logrus.Entry
+type Level = logrus.Level
+
+func WithFieldsFromContext(ctx context.Context) *Entry {
+	entry := logrus.WithContext(ctx)
+
+	for _, key := range registeredKeys {
+		if value := ctx.Value(key); value != nil {
+			entry = entry.WithField(key, value)
+		}
+	}
+	return entry
+}
+
+func Fatalx(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Fatal(args...)
+}
+
+func Errorx(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Error(args...)
+}
+
+func Warnx(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Warn(args...)
+}
+
+func Infox(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Info(args...)
+}
+
+func Debugx(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Debug(args...)
+}
+
+func Tracex(ctx context.Context, args ...interface{}) {
+	WithFieldsFromContext(ctx).Trace(args...)
+}
+
+func Errorxf(ctx context.Context, format string, args ...interface{}) {
+	WithFieldsFromContext(ctx).Errorf(format, args...)
+}
+
+func Warnxf(ctx context.Context, format string, args ...interface{}) {
+	WithFieldsFromContext(ctx).Warnf(format, args...)
+}
+
+func Infoxf(ctx context.Context, format string, args ...interface{}) {
+	WithFieldsFromContext(ctx).Infof(format, args...)
+}
+
+func Debugxf(ctx context.Context, format string, args ...interface{}) {
+	WithFieldsFromContext(ctx).Debugf(format, args...)
+}
+
+func Tracexf(ctx context.Context, format string, args ...interface{}) {
+	WithFieldsFromContext(ctx).Tracef(format, args...)
+}
+
+func Fatal(args ...interface{}) {
+	logrus.Fatal(args...)
+}
+
+func Error(args ...interface{}) {
+	logrus.Error(args...)
+}
+
+func Warn(args ...interface{}) {
+	logrus.Warn(args...)
+}
+
+func Info(args ...interface{}) {
+	logrus.Info(args...)
+}
+
+func Debug(args ...interface{}) {
+	logrus.Debug(args...)
+}
+
+func Trace(args ...interface{}) {
+	logrus.Trace(args...)
+}
+
+func Fatalf(format string, args ...interface{}) {
+	logrus.Fatalf(format, args...)
+}
+
+func Errorf(format string, args ...interface{}) {
+	logrus.Errorf(format, args...)
+}
+
+func Warnf(format string, args ...interface{}) {
+	logrus.Warnf(format, args...)
+}
+
+func Infof(format string, args ...interface{}) {
+	logrus.Infof(format, args...)
+}
+
+func Debugf(format string, args ...interface{}) {
+	logrus.Debugf(format, args...)
+}
+
+func Tracef(format string, args ...interface{}) {
+	logrus.Tracef(format, args...)
+}
+
+func SetOutput(out io.Writer) {
+	logrus.SetOutput(out)
+}
+
+func SetLevel(lvl Level) {
+	logrus.SetLevel(lvl)
+}
+
+func IsLevelEnabled(lvl Level) bool {
+	return logrus.IsLevelEnabled(lvl)
+}
+
+func WithError(err error) *Entry {
+	return logrus.WithError(err)
+}
+
+func WithField(field string, value interface{}) *Entry {
+	return logrus.WithField(field, value)
+}
+
+func Init() error {
+	// Set the output.
+	logFile := viper.GetString("log.file")
+	if logFile != "" {
+		file, err := os.OpenFile(logFile,
+			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0640)
+		if err == nil {
+			Infof("Directing output to %s", logFile)
+			SetOutput(file)
+		} else {
+			SetOutput(os.Stdout)
+			Info("Failed to logrus.to file, using default stdout")
+		}
+	} else {
+		SetOutput(os.Stdout)
+	}
+
+	// Set the level.
+	lvl, err := logrus.ParseLevel(viper.GetString("log.level"))
+	if err != nil {
+		return err
+	}
+	SetLevel(lvl)
+
+	formatter := &logrus.TextFormatter{
+		FullTimestamp: true,
+	}
+	// Show file/line number only at Trace level.
+	if lvl >= TraceLevel {
+		logrus.SetReportCaller(true)
+
+		// We need to exclude this wrapper code, logrus.us itself, and the runtime from the stack to show anything useful.
+		// cf. https://github.com/sirupsen/logrus.us/pull/973
+		formatter.CallerPrettyfier = func(frame *runtime.Frame) (function string, file string) {
+			pcs := make([]uintptr, 50)
+			_ = runtime.Callers(0, pcs)
+			frames := runtime.CallersFrames(pcs)
+
+			// Filter logrus.wrapper / logrus.us / runtime frames.
+			for next, again := frames.Next(); again; next, again = frames.Next() {
+				if !strings.Contains(next.File, "sirupsen/logrus.us") &&
+					!strings.HasPrefix(next.Function, "runtime.") &&
+					!strings.Contains(next.File, "ipld-eth-server/pkg/log") {
+					return next.Function, fmt.Sprintf("%s:%d", next.File, next.Line)
+				}
+			}
+
+			// Fallback to the raw info.
+			return frame.Function, fmt.Sprintf("%s:%d", frame.File, frame.Line)
+		}
+	}
+
+	logrus.SetFormatter(formatter)
+	Info("Log level set to ", lvl.String())
+	return nil
+}

pkg/net/api.go

@@ -0,0 +1,90 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// APIName is the namespace for the watcher's eth api
+const APIName = "net"
+
+// APIVersion is the version of the watcher's eth api
+const APIVersion = "0.0.1"
+
+// PublicNetAPI is the net nampespace API
+type PublicNetAPI struct {
+	// Proxy node for forwarding cache misses
+	networkVersion uint64
+	rpc            *rpc.Client
+	ethClient      *ethclient.Client
+}
+
+// NewPublicNetAPI creates a new PublicNetAPI with the provided underlying Backend
+func NewPublicNetAPI(networkID uint64, client *rpc.Client) *PublicNetAPI {
+	var ethClient *ethclient.Client
+	if client != nil {
+		ethClient = ethclient.NewClient(client)
+	}
+	return &PublicNetAPI{
+		networkVersion: networkID,
+		rpc:            client,
+		ethClient:      ethClient,
+	}
+}
+
+// Listening returns an indication if the node is listening for network connections.
+func (pna *PublicNetAPI) Listening() bool {
+	// in this case it is actually whether or not the proxied node is listening
+	if pna.rpc != nil {
+		var listening bool
+		if err := pna.rpc.Call(&listening, "net_listening"); err == nil {
+			return listening
+		}
+	}
+	return false
+}
+
+// PeerCount returns the number of connected peers
+func (pna *PublicNetAPI) PeerCount() hexutil.Uint {
+	// in this case it is actually the peer count of the proxied node
+	if pna.rpc != nil {
+		var num hexutil.Uint
+		if err := pna.rpc.Call(&num, "net_peerCount"); err == nil {
+			return num
+		}
+	}
+	return hexutil.Uint(0)
+}
+
+// Version returns the current ethereum protocol version.
+func (pna *PublicNetAPI) Version() string {
+	if pna.networkVersion != 0 {
+		return fmt.Sprintf("%d", pna.networkVersion)
+	}
+	if pna.rpc != nil {
+		var version string
+		if err := pna.rpc.Call(&version, "net_version"); err == nil {
+			return version
+		}
+	}
+	return ""
+}

pkg/net/api_test.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/net"
+)
+
+var _ = Describe("API", func() {
+	var (
+		api *net.PublicNetAPI
+	)
+	BeforeEach(func() {
+		api = net.NewPublicNetAPI(1, nil)
+	})
+	Describe("net_listening", func() {
+		It("Retrieves whether or not the node is listening to the p2p network", func() {
+			listening := api.Listening()
+			Expect(listening).To(BeFalse())
+		})
+	})
+
+	Describe("net_version", func() {
+		It("Retrieves the network id", func() {
+			version := api.Version()
+			Expect(version).To(Equal("1"))
+		})
+	})
+	// TODO: test PeerCount with mock proxy node
+})

pkg/net/net_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package net_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestNetSuite(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "eth ipld server net suite test")
+}
+
+var _ = BeforeSuite(func() {
+	log.SetOutput(ioutil.Discard)
+})

pkg/prom/middleware.go

@@ -17,25 +17,107 @@
 package prom
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
 	"net/http"
 	"time"
 
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/google/uuid"
+
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
-// HTTPMiddleware http connection metric reader
-func HTTPMiddleware(next http.Handler) http.Handler {
-	if !metrics {
-		return next
+const (
+	jsonMethod               = "method"
+	jsonParams               = "params"
+	jsonReqId                = "id"
+	headerUserId             = "X-User-Id"
+	headerOriginalRemoteAddr = "X-Original-Remote-Addr"
+)
+
+// Peek at the request and update the Context accordingly (eg, API method, user ID, etc.)
+func preprocessRequest(r *http.Request) (*http.Request, error) {
+	// Generate a unique ID for this request.
+	uniqId, err := uuid.NewUUID()
+	if nil != err {
+		return nil, err
 	}
 
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		httpCount.Inc()
+	// Read the body so that we can peek inside.
+	body, err := io.ReadAll(r.Body)
+	if nil != err {
+		return nil, err
+	}
 
+	// Replace it with a re-readable copy.
+	r.Body = io.NopCloser(bytes.NewBuffer(body))
+
+	// All API requests should be JSON.
+	var result map[string]interface{}
+	if len(body) > 0 {
+		err = json.Unmarshal(body, &result)
+		if nil != err {
+			return nil, err
+		}
+	}
+
+	// Pull out the method name, request ID, user ID, and address info.
+	reqId := fmt.Sprintf("%g", result[jsonReqId])
+	reqMethod := fmt.Sprintf("%v", result[jsonMethod])
+	reqParams := fmt.Sprintf("%v", result[jsonParams])
+	// Truncate parameters unless trace logging is enabled.
+	if !log.IsLevelEnabled(log.TraceLevel) {
+		if len(reqParams) > 250 {
+			reqParams = reqParams[:250] + "..."
+		}
+	}
+	userId := r.Header.Get(headerUserId)
+	conn := r.Header.Get(headerOriginalRemoteAddr)
+	if len(conn) == 0 {
+		conn = r.RemoteAddr
+	}
+
+	// Add it all to the request context.
+	ctx := r.Context()
+	ctx = context.WithValue(ctx, log.CtxKeyUniqId, uniqId.String())
+	ctx = context.WithValue(ctx, log.CtxKeyApiMethod, reqMethod)
+	ctx = context.WithValue(ctx, log.CtxKeyApiParams, string(reqParams))
+	ctx = context.WithValue(ctx, log.CtxKeyApiReqId, reqId)
+	ctx = context.WithValue(ctx, log.CtxKeyUserId, userId)
+	ctx = context.WithValue(ctx, log.CtxKeyConn, conn)
+
+	return r.WithContext(ctx), nil
+}
+
+// HTTPMiddleware http connection metric reader
+func HTTPMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		start := time.Now()
+		r, err := preprocessRequest(r)
+		if nil != err {
+			log.WithError(err).Error("Error preprocessing request")
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		ctx := r.Context()
+		apiMethod := fmt.Sprintf("%s", ctx.Value(log.CtxKeyApiMethod))
+
+		if metrics {
+			httpCount.WithLabelValues(apiMethod).Inc()
+		}
+
+		log.Debugx(ctx, "START")
 		next.ServeHTTP(w, r)
 		duration := time.Now().Sub(start)
-		httpDuration.Observe(float64(duration.Seconds()))
+		log.Debugxf(context.WithValue(ctx, log.CtxKeyDuration, duration.Milliseconds()), "END")
+
+		if metrics {
+			httpDuration.WithLabelValues(apiMethod).Observe(duration.Seconds())
+		}
 	})
 }
 

pkg/prom/prom.go

@@ -33,8 +33,8 @@ const (
 var (
 	metrics bool
 
-	httpCount    prometheus.Counter
-	httpDuration prometheus.Histogram
+	httpCount    *prometheus.CounterVec
+	httpDuration *prometheus.HistogramVec
 	wsCount      prometheus.Gauge
 	ipcCount     prometheus.Gauge
 )
@@ -43,18 +43,19 @@ var (
 func Init() {
 	metrics = true
 
-	httpCount = promauto.NewCounter(prometheus.CounterOpts{
+	httpCount = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: namespace,
 		Subsystem: subsystemHTTP,
 		Name:      "count",
 		Help:      "http request count",
-	})
-	httpDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+	}, []string{"method"})
+
+	httpDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: subsystemHTTP,
 		Name:      "duration",
 		Help:      "http request duration",
-	})
+	}, []string{"method"})
 
 	wsCount = promauto.NewGauge(prometheus.GaugeOpts{
 		Namespace: namespace,

pkg/prom/serve.go

@@ -20,8 +20,8 @@ import (
 	"errors"
 	"net/http"
 
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/sirupsen/logrus"
 )
 
 var errPromHTTP = errors.New("can't start http server for prometheus")
@@ -36,7 +36,7 @@ func Serve(addr string) *http.Server {
 	}
 	go func() {
 		if err := srv.ListenAndServe(); err != nil {
-			logrus.
+			log.
 				WithError(err).
 				WithField("module", "prom").
 				WithField("addr", addr).

pkg/rpc/http.go

@@ -17,41 +17,33 @@
 package rpc
 
 import (
-	"net"
+	"fmt"
 
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
-	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-eth-server/pkg/prom"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/prom"
 )
 
 // StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules.
-func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (net.Listener, *rpc.Server, error) {
-	if bad, available := checkModuleAvailability(modules, apis); len(bad) > 0 {
-		log.Error("Unavailable modules in HTTP API list", "unavailable", bad, "available", available)
+func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (*rpc.Server, error) {
+
+	srv := rpc.NewServer()
+	err := node.RegisterApis(apis, modules, srv)
+	if err != nil {
+		utils.Fatalf("Could not register HTTP API: %w", err)
 	}
-	// Generate the whitelist based on the allowed modules
-	whitelist := make(map[string]bool)
-	for _, module := range modules {
-		whitelist[module] = true
+	handler := prom.HTTPMiddleware(node.NewHTTPHandlerStack(srv, cors, vhosts, nil))
+
+	// start http server
+	_, addr, err := node.StartHTTPEndpoint(endpoint, rpc.DefaultHTTPTimeouts, handler)
+	if err != nil {
+		utils.Fatalf("Could not start RPC api: %v", err)
 	}
-	// Register all the APIs exposed by the services
-	handler := rpc.NewServer()
-	for _, api := range apis {
-		if whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) {
-			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
-				return nil, nil, err
-			}
-			log.Debug("HTTP registered", "namespace", api.Namespace)
-		}
-	}
-	// All APIs registered, start the HTTP listener
-	var (
-		listener net.Listener
-		err      error
-	)
-	if listener, err = net.Listen("tcp", endpoint); err != nil {
-		return nil, nil, err
-	}
-	go rpc.NewHTTPServer(cors, vhosts, timeouts, prom.HTTPMiddleware(handler)).Serve(listener)
-	return listener, handler, err
+	extapiURL := fmt.Sprintf("http://%s", addr)
+	log.Infof("HTTP endpoint opened at %s", extapiURL)
+
+	return srv, err
 }

pkg/rpc/ipc.go

@@ -22,10 +22,11 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/rpc"
-	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-eth-server/pkg/prom"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/prom"
 )
 
 var (

pkg/rpc/ws.go

@@ -18,45 +18,47 @@ package rpc
 
 import (
 	"net"
+	"net/http"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
-	log "github.com/sirupsen/logrus"
-	"github.com/vulcanize/ipld-eth-server/pkg/prom"
+
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/log"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/prom"
 )
 
 // StartWSEndpoint starts a websocket endpoint.
-func StartWSEndpoint(endpoint string, apis []rpc.API, modules []string, wsOrigins []string, exposeAll bool) (net.Listener, *rpc.Server, error) {
-	if bad, available := checkModuleAvailability(modules, apis); len(bad) > 0 {
-		log.Error("Unavailable modules in WS API list", "unavailable", bad, "available", available)
-	}
-	// Generate the whitelist based on the allowed modules
-	whitelist := make(map[string]bool)
-	for _, module := range modules {
-		whitelist[module] = true
-	}
-	// Register all the APIs exposed by the services
-	handler := rpc.NewServer()
-	for _, api := range apis {
-		if exposeAll || whitelist[api.Namespace] || (len(whitelist) == 0 && api.Public) {
-			if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
-				return nil, nil, err
-			}
-			log.Debug("WebSocket registered", "service", api.Service, "namespace", api.Namespace)
-		}
-	}
+func StartWSEndpoint(endpoint string, apis []rpc.API, modules []string, wsOrigins []string) (net.Listener, *rpc.Server, error) {
 	// All APIs registered, start the HTTP listener
 	var (
 		listener net.Listener
 		err      error
 	)
+
+	// Register all the APIs exposed by the services
+	handler := rpc.NewServer()
+	err = node.RegisterApis(apis, modules, handler)
+	if err != nil {
+		utils.Fatalf("Could not register WS API: %w", err)
+	}
+
 	if listener, err = net.Listen("tcp", endpoint); err != nil {
 		return nil, nil, err
 	}
 
-	wsServer := rpc.NewWSServer(wsOrigins, handler)
+	wsServer := NewWSServer(wsOrigins, handler)
 	wsServer.Handler = prom.WSMiddleware(wsServer.Handler)
 	go wsServer.Serve(listener)
+	log.Infof("WS endpoint opened at ws://%s", endpoint)
 
 	return listener, handler, err
 
 }
+
+// NewWSServer creates a new websocket RPC server around an API provider.
+//
+// Deprecated: use prc.Server.WebsocketHandler
+func NewWSServer(allowedOrigins []string, srv *rpc.Server) *http.Server {
+	return &http.Server{Handler: srv.WebsocketHandler(allowedOrigins)}
+}

pkg/serve/api.go

@@ -17,16 +17,8 @@
 package serve
 
 import (
-	"context"
-
-	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rpc"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
-
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	v "github.com/vulcanize/ipld-eth-server/version"
+	"github.com/ethereum/go-ethereum/statediff/types"
 )
 
 // APIName is the namespace used for the state diffing service API
@@ -37,85 +29,24 @@ const APIVersion = "0.0.1"
 
 // PublicServerAPI is the public api for the watcher
 type PublicServerAPI struct {
-	w Server
+	w   Server
+	rpc *rpc.Client
 }
 
 // NewPublicServerAPI creates a new PublicServerAPI with the provided underlying Server process
-func NewPublicServerAPI(w Server) *PublicServerAPI {
+func NewPublicServerAPI(w Server, client *rpc.Client) *PublicServerAPI {
 	return &PublicServerAPI{
-		w: w,
+		w:   w,
+		rpc: client,
 	}
 }
 
-// Stream is the public method to setup a subscription that fires off IPLD payloads as they are processed
-func (api *PublicServerAPI) Stream(ctx context.Context, params eth.SubscriptionSettings) (*rpc.Subscription, error) {
-	// ensure that the RPC connection supports subscriptions
-	notifier, supported := rpc.NotifierFromContext(ctx)
-	if !supported {
-		return nil, rpc.ErrNotificationsUnsupported
+// WatchAddress makes a geth WatchAddress API call with the given operation and args
+func (api *PublicServerAPI) WatchAddress(operation types.OperationType, args []types.WatchAddressArg) error {
+	err := api.rpc.Call(nil, "statediff_watchAddress", operation, args)
+	if err != nil {
+		return err
 	}
 
-	// create subscription and start waiting for stream events
-	rpcSub := notifier.CreateSubscription()
-
-	go func() {
-		// subscribe to events from the SyncPublishScreenAndServe service
-		payloadChannel := make(chan SubscriptionPayload, PayloadChanBufferSize)
-		quitChan := make(chan bool, 1)
-		go api.w.Subscribe(rpcSub.ID, payloadChannel, quitChan, params)
-
-		// loop and await payloads and relay them to the subscriber using notifier
-		for {
-			select {
-			case packet := <-payloadChannel:
-				if err := notifier.Notify(rpcSub.ID, packet); err != nil {
-					log.Error("Failed to send watcher data packet", "err", err)
-					api.w.Unsubscribe(rpcSub.ID)
-					return
-				}
-			case <-rpcSub.Err():
-				api.w.Unsubscribe(rpcSub.ID)
-				return
-			case <-quitChan:
-				// don't need to unsubscribe from the watcher, the service does so before sending the quit signal this way
-				return
-			}
-		}
-	}()
-
-	return rpcSub, nil
-}
-
-// Chain returns the chain type that this watcher instance supports
-func (api *PublicServerAPI) Chain() shared.ChainType {
-	return shared.Ethereum
-}
-
-// Struct for holding watcher meta data
-type InfoAPI struct{}
-
-// NewInfoAPI creates a new InfoAPI
-func NewInfoAPI() *InfoAPI {
-	return &InfoAPI{}
-}
-
-// Modules returns modules supported by this api
-func (iapi *InfoAPI) Modules() map[string]string {
-	return map[string]string{
-		"vdb": "Stream",
-	}
-}
-
-// NodeInfo gathers and returns a collection of metadata for the watcher
-func (iapi *InfoAPI) NodeInfo() *p2p.NodeInfo {
-	return &p2p.NodeInfo{
-		// TODO: formalize this
-		ID:   "vulcanizeDB",
-		Name: "ipld-eth-server",
-	}
-}
-
-// Version returns the version of the watcher
-func (iapi *InfoAPI) Version() string {
-	return v.VersionWithMeta
+	return nil
 }

pkg/serve/config.go

@@ -17,50 +17,105 @@
 package serve
 
 import (
+	"errors"
 	"fmt"
 	"math/big"
 	"os"
 	"path/filepath"
-
-	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/node"
+	"github.com/jmoiron/sqlx"
 	"github.com/spf13/viper"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-	"github.com/vulcanize/ipld-eth-indexer/utils"
-	"github.com/vulcanize/ipld-eth-server/pkg/prom"
 
-	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"github.com/cerc-io/ipld-eth-server/v5/pkg/prom"
+	ethServerShared "github.com/cerc-io/ipld-eth-server/v5/pkg/shared"
 )
 
 // Env variables
 const (
-	SERVER_WS_PATH   = "SERVER_WS_PATH"
-	SERVER_IPC_PATH  = "SERVER_IPC_PATH"
-	SERVER_HTTP_PATH = "SERVER_HTTP_PATH"
+	SERVER_WS_PATH      = "SERVER_WS_PATH"
+	SERVER_IPC_PATH     = "SERVER_IPC_PATH"
+	SERVER_HTTP_PATH    = "SERVER_HTTP_PATH"
+	SERVER_GRAPHQL_PATH = "SERVER_GRAPHQL_PATH"
 
 	SERVER_MAX_IDLE_CONNECTIONS = "SERVER_MAX_IDLE_CONNECTIONS"
 	SERVER_MAX_OPEN_CONNECTIONS = "SERVER_MAX_OPEN_CONNECTIONS"
 	SERVER_MAX_CONN_LIFETIME    = "SERVER_MAX_CONN_LIFETIME"
 
-	ETH_DEFAULT_SENDER_ADDR = "ETH_DEFAULT_SENDER_ADDR"
-	ETH_RPC_GAS_CAP         = "ETH_RPC_GAS_CAP"
+	ETH_DEFAULT_SENDER_ADDR    = "ETH_DEFAULT_SENDER_ADDR"
+	ETH_RPC_GAS_CAP            = "ETH_RPC_GAS_CAP"
+	ETH_CHAIN_CONFIG           = "ETH_CHAIN_CONFIG"
+	ETH_SUPPORTS_STATEDIFF     = "ETH_SUPPORTS_STATEDIFF"
+	ETH_STATEDIFF_TIMEOUT      = "ETH_STATEDIFF_TIMEOUT"
+	ETH_FORWARD_ETH_CALLS      = "ETH_FORWARD_ETH_CALLS"
+	ETH_FORWARD_GET_STORAGE_AT = "ETH_FORWARD_GET_STORAGE_AT"
+	ETH_PROXY_ON_ERROR         = "ETH_PROXY_ON_ERROR"
+	ETH_GETLOGS_BLOCK_LIMIT    = "ETH_GETLOGS_BLOCK_LIMIT"
+
+	VALIDATOR_ENABLED         = "VALIDATOR_ENABLED"
+	VALIDATOR_EVERY_NTH_BLOCK = "VALIDATOR_EVERY_NTH_BLOCK"
+
+	HTTP_TIMEOUT = "HTTP_TIMEOUT"
+
+	ETH_WS_PATH       = "ETH_WS_PATH"
+	ETH_HTTP_PATH     = "ETH_HTTP_PATH"
+	ETH_NODE_ID       = "ETH_NODE_ID"
+	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
+	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
+	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
+	ETH_CHAIN_ID      = "ETH_CHAIN_ID"
+
+	DATABASE_NAME                 = "DATABASE_NAME"
+	DATABASE_HOSTNAME             = "DATABASE_HOSTNAME"
+	DATABASE_PORT                 = "DATABASE_PORT"
+	DATABASE_USER                 = "DATABASE_USER"
+	DATABASE_PASSWORD             = "DATABASE_PASSWORD"
+	DATABASE_MAX_IDLE_CONNECTIONS = "DATABASE_MAX_IDLE_CONNECTIONS"
+	DATABASE_MAX_OPEN_CONNECTIONS = "DATABASE_MAX_OPEN_CONNECTIONS"
+	DATABASE_MAX_CONN_LIFETIME    = "DATABASE_MAX_CONN_LIFETIME"
 )
 
 // Config struct
 type Config struct {
-	DB            *postgres.DB
-	DBConfig      postgres.Config
-	WSEndpoint    string
-	HTTPEndpoint  string
-	IPCEndpoint   string
-	ChainConfig   *params.ChainConfig
-	DefaultSender *common.Address
-	RPCGasCap     *big.Int
-	Client        *rpc.Client
+	DB       *sqlx.DB
+	DBConfig postgres.Config
+
+	WSEnabled  bool
+	WSEndpoint string
+
+	HTTPEnabled  bool
+	HTTPEndpoint string
+
+	IPCEnabled  bool
+	IPCEndpoint string
+
+	EthGraphqlEnabled  bool
+	EthGraphqlEndpoint string
+
+	ChainConfig         *params.ChainConfig
+	DefaultSender       *common.Address
+	RPCGasCap           *big.Int
+	EthHttpEndpoint     string
+	Client              *rpc.Client
+	SupportStateDiff    bool
+	StateDiffTimeout    time.Duration
+	ForwardEthCalls     bool
+	ForwardGetStorageAt bool
+	ProxyOnError        bool
+	GetLogsBlockLimit   int64
+	NodeNetworkID       string
+
+	// Cache configuration.
+	GroupCache *ethServerShared.GroupCacheConfig
+
+	StateValidationEnabled       bool
+	StateValidationEveryNthBlock uint64
 }
 
 // NewConfig is used to initialize a watcher config from a .toml file
@@ -68,58 +123,131 @@ type Config struct {
 func NewConfig() (*Config, error) {
 	c := new(Config)
 
+	viper.BindEnv("server.httpPath", SERVER_HTTP_PATH)
 	viper.BindEnv("server.wsPath", SERVER_WS_PATH)
 	viper.BindEnv("server.ipcPath", SERVER_IPC_PATH)
-	viper.BindEnv("server.httpPath", SERVER_HTTP_PATH)
-	viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
-	viper.BindEnv("ethereum.defaultSender", ETH_DEFAULT_SENDER_ADDR)
+	viper.BindEnv("server.graphqlPath", SERVER_GRAPHQL_PATH)
+
+	viper.BindEnv("ethereum.httpPath", ETH_HTTP_PATH)
 	viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP)
+	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
+	viper.BindEnv("ethereum.supportsStateDiff", ETH_SUPPORTS_STATEDIFF)
+	viper.BindEnv("ethereum.stateDiffTimeout", ETH_STATEDIFF_TIMEOUT)
+	viper.BindEnv("ethereum.forwardEthCalls", ETH_FORWARD_ETH_CALLS)
+	viper.BindEnv("ethereum.forwardGetStorageAt", ETH_FORWARD_GET_STORAGE_AT)
+	viper.BindEnv("ethereum.proxyOnError", ETH_PROXY_ON_ERROR)
+	viper.BindEnv("ethereum.getLogsBlockLimit", ETH_GETLOGS_BLOCK_LIMIT)
+	viper.BindEnv("log.file", "LOG_FILE")
+	viper.BindEnv("log.level", "LOG_LEVEL")
 
-	c.DBConfig.Init()
-
+	c.dbInit()
 	ethHTTP := viper.GetString("ethereum.httpPath")
-	nodeInfo, cli, err := shared.GetEthNodeAndClient(fmt.Sprintf("http://%s", ethHTTP))
+	ethHTTPEndpoint := fmt.Sprintf("http://%s", ethHTTP)
+	nodeInfo, cli, err := getEthNodeAndClient(ethHTTPEndpoint)
+	c.NodeNetworkID = nodeInfo.NetworkID
 	if err != nil {
 		return nil, err
 	}
 	c.Client = cli
+	c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff")
+	c.ForwardEthCalls = viper.GetBool("ethereum.forwardEthCalls")
+	c.ForwardGetStorageAt = viper.GetBool("ethereum.forwardGetStorageAt")
+	c.ProxyOnError = viper.GetBool("ethereum.proxyOnError")
+	c.EthHttpEndpoint = ethHTTPEndpoint
 
-	wsPath := viper.GetString("server.wsPath")
-	if wsPath == "" {
-		wsPath = "127.0.0.1:8080"
+	if viper.IsSet("ethereum.getLogsBlockLimit") {
+		c.GetLogsBlockLimit = viper.GetInt64("ethereum.getLogsBlockLimit")
+	} else {
+		c.GetLogsBlockLimit = 500
 	}
-	c.WSEndpoint = wsPath
-	ipcPath := viper.GetString("server.ipcPath")
-	if ipcPath == "" {
-		home, err := os.UserHomeDir()
-		if err != nil {
-			return nil, err
+
+	// websocket server
+	wsEnabled := viper.GetBool("server.ws")
+	if wsEnabled {
+		wsPath := viper.GetString("server.wsPath")
+		if wsPath == "" {
+			wsPath = "127.0.0.1:8080"
 		}
-		ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
+		c.WSEndpoint = wsPath
 	}
-	c.IPCEndpoint = ipcPath
-	httpPath := viper.GetString("server.httpPath")
-	if httpPath == "" {
-		httpPath = "127.0.0.1:8081"
-	}
-	c.HTTPEndpoint = httpPath
-	overrideDBConnConfig(&c.DBConfig)
-	serveDB := utils.LoadPostgres(c.DBConfig, nodeInfo)
-	prom.RegisterDBCollector(c.DBConfig.Name, serveDB.DB)
-	c.DB = &serveDB
+	c.WSEnabled = wsEnabled
 
-	defaultSenderStr := viper.GetString("ethereum.defaultSender")
-	if defaultSenderStr != "" {
-		sender := common.HexToAddress(defaultSenderStr)
-		c.DefaultSender = &sender
+	// ipc server
+	ipcEnabled := viper.GetBool("server.ipc")
+	if ipcEnabled {
+		ipcPath := viper.GetString("server.ipcPath")
+		if ipcPath == "" {
+			home, err := os.UserHomeDir()
+			if err != nil {
+				return nil, err
+			}
+			ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
+		}
+		c.IPCEndpoint = ipcPath
 	}
+	c.IPCEnabled = ipcEnabled
+
+	// http server
+	httpEnabled := viper.GetBool("server.http")
+	if httpEnabled {
+		httpPath := viper.GetString("server.httpPath")
+		if httpPath == "" {
+			httpPath = "127.0.0.1:8081"
+		}
+		c.HTTPEndpoint = httpPath
+	}
+	c.HTTPEnabled = httpEnabled
+
+	// eth graphql endpoint
+	ethGraphqlEnabled := viper.GetBool("server.graphql")
+	if ethGraphqlEnabled {
+		ethGraphqlPath := viper.GetString("server.graphqlPath")
+		if ethGraphqlPath == "" {
+			ethGraphqlPath = "127.0.0.1:8082"
+		}
+		c.EthGraphqlEndpoint = ethGraphqlPath
+	}
+	c.EthGraphqlEnabled = ethGraphqlEnabled
+
+	overrideDBConnConfig(&c.DBConfig)
+	serveDB, err := ethServerShared.NewDB(c.DBConfig.DbConnectionString(), c.DBConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	prom.RegisterDBCollector(c.DBConfig.DatabaseName, serveDB)
+	c.DB = serveDB
+
 	rpcGasCapStr := viper.GetString("ethereum.rpcGasCap")
 	if rpcGasCapStr != "" {
 		if rpcGasCap, ok := new(big.Int).SetString(rpcGasCapStr, 10); ok {
 			c.RPCGasCap = rpcGasCap
 		}
+	} else {
+		c.RPCGasCap = big.NewInt(0)
 	}
-	c.ChainConfig, err = eth.ChainConfig(nodeInfo.ChainID)
+	if sdTimeout := viper.GetString("ethereum.stateDiffTimeout"); sdTimeout != "" {
+		var err error
+		if c.StateDiffTimeout, err = time.ParseDuration(sdTimeout); err != nil {
+			return nil, err
+		}
+	} else {
+		c.StateDiffTimeout = ethServerShared.DefaultStateDiffTimeout
+	}
+	if c.StateDiffTimeout < 0 {
+		return nil, errors.New("ethereum.stateDiffTimeout < 0")
+	}
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+	if chainConfigPath != "" {
+		c.ChainConfig, err = statediff.LoadConfig(chainConfigPath)
+	} else {
+		c.ChainConfig, err = statediff.ChainConfig(nodeInfo.ChainID)
+	}
+
+	c.loadGroupCacheConfig()
+
+	c.loadValidatorConfig()
+
 	return c, err
 }
 
@@ -128,6 +256,78 @@ func overrideDBConnConfig(con *postgres.Config) {
 	viper.BindEnv("database.server.maxOpen", SERVER_MAX_OPEN_CONNECTIONS)
 	viper.BindEnv("database.server.maxLifetime", SERVER_MAX_CONN_LIFETIME)
 	con.MaxIdle = viper.GetInt("database.server.maxIdle")
-	con.MaxOpen = viper.GetInt("database.server.maxOpen")
-	con.MaxLifetime = viper.GetInt("database.server.maxLifetime")
+	con.MaxConns = viper.GetInt("database.server.maxOpen")
+	con.MaxConnLifetime = time.Duration(viper.GetInt("database.server.maxLifetime"))
+}
+
+func (c *Config) dbInit() {
+	viper.BindEnv("database.name", DATABASE_NAME)
+	viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
+	viper.BindEnv("database.port", DATABASE_PORT)
+	viper.BindEnv("database.user", DATABASE_USER)
+	viper.BindEnv("database.password", DATABASE_PASSWORD)
+	viper.BindEnv("database.maxIdle", DATABASE_MAX_IDLE_CONNECTIONS)
+	viper.BindEnv("database.maxOpen", DATABASE_MAX_OPEN_CONNECTIONS)
+	viper.BindEnv("database.maxLifetime", DATABASE_MAX_CONN_LIFETIME)
+
+	c.DBConfig.DatabaseName = viper.GetString("database.name")
+	c.DBConfig.Hostname = viper.GetString("database.hostname")
+	c.DBConfig.Port = viper.GetInt("database.port")
+	c.DBConfig.Username = viper.GetString("database.user")
+	c.DBConfig.Password = viper.GetString("database.password")
+	c.DBConfig.MaxIdle = viper.GetInt("database.maxIdle")
+	c.DBConfig.MaxConns = viper.GetInt("database.maxOpen")
+	c.DBConfig.MaxConnLifetime = time.Duration(viper.GetInt("database.maxLifetime"))
+}
+
+func (c *Config) loadGroupCacheConfig() {
+	viper.BindEnv("groupcache.pool.enabled", ethServerShared.GcachePoolEnabled)
+	viper.BindEnv("groupcache.pool.httpEndpoint", ethServerShared.GcachePoolHttpPath)
+	viper.BindEnv("groupcache.pool.peerHttpEndpoints", ethServerShared.GcachePoolHttpPeers)
+	viper.BindEnv("groupcache.statedb.cacheSizeInMB", ethServerShared.GcacheStatedbCacheSize)
+	viper.BindEnv("groupcache.statedb.cacheExpiryInMins", ethServerShared.GcacheStatedbCacheExpiry)
+	viper.BindEnv("groupcache.statedb.logStatsIntervalInSecs", ethServerShared.GcacheStatedbLogStatsInterval)
+
+	gcc := ethServerShared.GroupCacheConfig{}
+	gcc.Pool.Enabled = viper.GetBool("groupcache.pool.enabled")
+	if gcc.Pool.Enabled {
+		gcc.Pool.HttpEndpoint = viper.GetString("groupcache.pool.httpEndpoint")
+		gcc.Pool.PeerHttpEndpoints = viper.GetStringSlice("groupcache.pool.peerHttpEndpoints")
+	}
+
+	// Irrespective of whether the pool is enabled, we always use the hot/local cache.
+	gcc.StateDB.CacheSizeInMB = viper.GetInt("groupcache.statedb.cacheSizeInMB")
+	gcc.StateDB.CacheExpiryInMins = viper.GetInt("groupcache.statedb.cacheExpiryInMins")
+	gcc.StateDB.LogStatsIntervalInSecs = viper.GetInt("groupcache.statedb.logStatsIntervalInSecs")
+
+	c.GroupCache = &gcc
+}
+
+func (c *Config) loadValidatorConfig() {
+	viper.BindEnv("validator.enabled", VALIDATOR_ENABLED)
+	viper.BindEnv("validator.everyNthBlock", VALIDATOR_EVERY_NTH_BLOCK)
+
+	c.StateValidationEnabled = viper.GetBool("validator.enabled")
+	c.StateValidationEveryNthBlock = viper.GetUint64("validator.everyNthBlock")
+}
+
+// GetEthNodeAndClient returns eth node info and client from path url
+func getEthNodeAndClient(path string) (node.Info, *rpc.Client, error) {
+	viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
+	viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
+	viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
+	viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
+	viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
+
+	rpcClient, err := rpc.Dial(path)
+	if err != nil {
+		return node.Info{}, nil, err
+	}
+	return node.Info{
+		ID:           viper.GetString("ethereum.nodeID"),
+		ClientName:   viper.GetString("ethereum.clientName"),
+		GenesisBlock: viper.GetString("ethereum.genesisBlock"),
+		NetworkID:    viper.GetString("ethereum.networkID"),
+		ChainID:      viper.GetUint64("ethereum.chainID"),
+	}, rpcClient, nil
 }