support custom chain config (#36)

* support custom chain config

* move loading config logic to a separate helper function

.github/workflows/on-pr.yaml

@@ -10,3 +10,26 @@ jobs:
       - uses: actions/checkout@v2
       - name: Run docker build
         run: make docker-build
+  test:
+    name: Run integration tests
+    env:
+      GOPATH: /tmp/go
+    strategy:
+      matrix:
+        go-version: [1.14.x, 1.15.x]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Create GOPATH
+        run: mkdir -p /tmp/go
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: actions/checkout@v2
+      - name: Run database
+        run: docker-compose up -d db
+      - name: Test
+        run: |
+          sleep 10
+          PGPASSWORD=password DATABASE_USER=vdbm DATABASE_PORT=8077 DATABASE_PASSWORD=password DATABASE_HOSTNAME=127.0.0.1 make test

Makefile

@@ -11,8 +11,7 @@ $(BIN)/ginkgo:
 ## Migration tool
 GOOSE = $(BIN)/goose
 $(BIN)/goose:
-	go get -u -d github.com/pressly/goose/cmd/goose
-	go build -tags='no_mysql no_sqlite' -o $(BIN)/goose github.com/pressly/goose/cmd/goose
+	go get -u github.com/pressly/goose/cmd/goose
 
 ## Source linter
 LINT = $(BIN)/golint
@@ -46,31 +45,53 @@ HOST_NAME = localhost
 PORT = 5432
 NAME =
 USER = postgres
-CONNECT_STRING=postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
+PASSWORD = password
+CONNECT_STRING=postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(NAME)?sslmode=disable
 
 #Test
 TEST_DB = vulcanize_testing
-TEST_CONNECT_STRING = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
+TEST_CONNECT_STRING = postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOSTNAME):$(DATABASE_PORT)/$(TEST_DB)?sslmode=disable
+TEST_CONNECT_STRING_LOCAL = postgresql://$(USER)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
 
 .PHONY: test
-test: | $(GINKGO) $(LINT)
+test: | $(GINKGO) $(GOOSE)
 	go vet ./...
 	go fmt ./...
-	dropdb --if-exists $(TEST_DB)
-	createdb $(TEST_DB)
+	export PGPASSWORD=$(DATABASE_PASSWORD)
+	dropdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) --if-exists $(TEST_DB)
+	createdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) $(TEST_DB)
 	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
-	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" reset
-	make migrate NAME=$(TEST_DB)
 	$(GINKGO) -r --skipPackage=integration_tests,integration
 
 .PHONY: integrationtest
-integrationtest: | $(GINKGO) $(LINT)
+integrationtest: | $(GINKGO) $(GOOSE)
 	go vet ./...
 	go fmt ./...
-	dropdb --if-exists $(TEST_DB)
-	createdb $(TEST_DB)
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" up
-	$(GOOSE) -dir db/migrations "$(TEST_CONNECT_STRING)" reset
+	export PGPASSWORD=$(DATABASE_PASSWORD)
+	dropdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) --if-exists $(TEST_DB)
+	createdb -h $(DATABASE_HOSTNAME) -p $(DATABASE_PORT) -U $(DATABASE_USER) $(TEST_DB)
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING)" up
+	$(GINKGO) -r integration_test/
+
+.PHONY: test_local
+test_local: | $(GINKGO) $(GOOSE)
+	go vet ./...
+	go fmt ./...
+	dropdb -h $(HOST_NAME) -p $(PORT) -U $(USER) --if-exists $(TEST_DB)
+	createdb -h $(HOST_NAME) -p $(PORT) -U $(USER) $(TEST_DB)
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" up
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" reset
+	make migrate NAME=$(TEST_DB)
+	$(GINKGO) -r --skipPackage=integration_tests,integration
+
+.PHONY: integrationtest_local
+integrationtest_local: | $(GINKGO) $(GOOSE)
+	go vet ./...
+	go fmt ./...
+	dropdb -h $(HOST_NAME) -p $(PORT) -U $(USER) --if-exists $(TEST_DB)
+	createdb -h $(HOST_NAME) -p $(PORT) -U $(USER) $(TEST_DB)
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" up
+	$(GOOSE) -dir db/migrations postgres "$(TEST_CONNECT_STRING_LOCAL)" reset
 	make migrate NAME=$(TEST_DB)
 	$(GINKGO) -r integration_test/
 

README.md

@@ -14,7 +14,7 @@
 ## Background
 NOTE: WIP
 
-ipld-eth-server is used to service queries against the indexed Ethereum IPLD objects indexed by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer).
+ipld-eth-server is used to service queries against the Ethereum IPLD objects indexed by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer).
 
 It exposes standard Ethereum JSON RPC endpoints on top of the database, in some cases these endpoints can leverage the unique indexes to improve query performance.
 Additional, unique endpoints are exposed which utilize the new indexes and state diff data objects.
@@ -66,10 +66,23 @@ The corresponding CLI flags can be found with the `./ipld-eth-server serve --hel
     ipcPath = "~/.vulcanize/vulcanize.ipc" # $SERVER_IPC_PATH
     wsPath = "127.0.0.1:8081" # $SERVER_WS_PATH
     httpPath = "127.0.0.1:8082" # $SERVER_HTTP_PATH
+    graphql = true # $SERVER_GRAPHQL
+    graphqlEndpoint = "" # $SERVER_GRAPHQL_ENDPOINT
+
+[ethereum]
+    chainID = "1" # $ETH_CHAIN_ID
+    defaultSender = "" # $ETH_DEFAULT_SENDER_ADDR
+    rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
+    httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    nodeID = "arch1" # $ETH_NODE_ID
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
+    networkID = "1" # $ETH_NETWORK_ID
 ```
 
-The `database` fields are for connecting to a Postgres database that has been/is being populated by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer).
-The `server` fields set the paths for exposing the ipld-eth-server endpoints
+The `database` fields are for connecting to a Postgres database that has been/is being populated by [ipld-eth-indexer](https://github.com/vulcanize/ipld-eth-indexer)  
+The `server` fields set the paths for exposing the ipld-eth-server endpoints  
+The `ethereum` fields set the chainID and default sender address to use for EVM simulation, and can optionally be used to configure a remote eth node to forward cache misses to  
 
 
 ### Endpoints
@@ -80,19 +93,48 @@ TODO: Port the IPLD RPC subscription endpoints after the decoupling
 ipld-eth-server currently recapitulates portions of the Ethereum JSON-RPC api standard.
 
 The currently supported standard endpoints are:  
+`eth_call`  
+`eth_getBalance`  
+`eth_getStorageAt`  
+`eth_getCode`  
+`eth_getProof`  
 `eth_blockNumber`  
-`eth_getLogs`  
 `eth_getHeaderByNumber`  
+`eth_getHeaderByHash`  
 `eth_getBlockByNumber`  
 `eth_getBlockByHash`  
+`eth_getTransactionCount`  
+`eth_getBlockTransactionCountByHash`  
+`eth_getBlockTransactionCountByNumber`  
 `eth_getTransactionByHash`  
+`eth_getRawTransactionByHash`  
+`eth_getTransactionByBlockHashAndIndex`  
+`eth_getTransactionByBlockNumberAndIndex`  
+`eth_getRawTransactionByBlockHashAndIndex`  
+`eth_getRawTransactionByBlockNumberAndIndex`  
+`eth_getTransactionReceipt`  
+`eth_getLogs`  
+`eth_getUncleCountByBlockHash`  
+`eth_getUncleCountByBlockNumber`  
+`eth_getUncleByBlockHashAndIndex`  
+`eth_getUncleByBlockNumberAndIndex`  
 
-TODO: Add the rest of the standard endpoints add unique endpoints (e.g. getSlice)
+TODO: Add the rest of the standard endpoints and unique endpoints (e.g. getSlice)
 
 ### Testing
 `make test` will run the unit tests  
 `make test` setups a clean `vulcanize_testing` db
 
+## Monitoring
+
+* Enable http server and metrics using parameters `--http --metrics`
+* ipld-eth-server exposes prometheus metrics at `/metric` endpoint
+* start prometheus using `monitoring/prometheus.yml` config (`prometheus --config.file=monitoring/prometheus.yml`)
+* start grafana, connect to prometheus datasource and import dashboard from `monitoring/grafana/dashboard_main.json`
+
+![](monitoring/grafana.png)
+
+
 ## Contributing
 Contributions are welcome!
 

cmd/root.go

@@ -24,6 +24,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"github.com/vulcanize/ipld-eth-server/pkg/prom"
 )
 
 var (
@@ -45,7 +46,8 @@ func Execute() {
 }
 
 func initFuncs(cmd *cobra.Command, args []string) {
-	logfile := viper.GetString("logfile")
+	viper.BindEnv("log.file", "LOGRUS_FILE")
+	logfile := viper.GetString("log.file")
 	if logfile != "" {
 		file, err := os.OpenFile(logfile,
 			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
@@ -62,6 +64,19 @@ func initFuncs(cmd *cobra.Command, args []string) {
 	if err := logLevel(); err != nil {
 		log.Fatal("Could not set log level: ", err)
 	}
+
+	if viper.GetBool("metrics") {
+		prom.Init()
+	}
+
+	if viper.GetBool("prom.http") {
+		addr := fmt.Sprintf(
+			"%s:%s",
+			viper.GetString("prom.http.addr"),
+			viper.GetString("prom.http.port"),
+		)
+		prom.Serve(addr)
+	}
 }
 
 func logLevel() error {
@@ -84,22 +99,34 @@ func init() {
 	viper.AutomaticEnv()
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
-	rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
 	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
 	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
 	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
 	rootCmd.PersistentFlags().String("database-user", "", "database user")
 	rootCmd.PersistentFlags().String("database-password", "", "database password")
 	rootCmd.PersistentFlags().String("client-ipcPath", "", "location of geth.ipc file")
-	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
+	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(), "log level (trace, debug, info, warn, error, fatal, panic)")
+	rootCmd.PersistentFlags().String("log-file", "", "file path for logging")
+
+	rootCmd.PersistentFlags().Bool("metrics", false, "enable metrics")
+
+	rootCmd.PersistentFlags().Bool("prom-http", false, "enable http service for prometheus")
+	rootCmd.PersistentFlags().String("prom-http-addr", "127.0.0.1", "http host for prometheus")
+	rootCmd.PersistentFlags().String("prom-http-port", "8090", "http port for prometheus")
 
-	viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
 	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
 	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
 	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
 	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
+	viper.BindPFlag("log.file", rootCmd.PersistentFlags().Lookup("log-file"))
+
+	viper.BindPFlag("metrics", rootCmd.PersistentFlags().Lookup("metrics"))
+
+	viper.BindPFlag("prom.http", rootCmd.PersistentFlags().Lookup("prom-http"))
+	viper.BindPFlag("prom.http.addr", rootCmd.PersistentFlags().Lookup("prom-http-addr"))
+	viper.BindPFlag("prom.http.port", rootCmd.PersistentFlags().Lookup("prom-http-port"))
 }
 
 func initConfig() {

cmd/serve.go

@@ -20,6 +20,8 @@ import (
 	"os/signal"
 	"sync"
 
+	"github.com/vulcanize/ipld-eth-server/pkg/graphql"
+
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -27,6 +29,7 @@ import (
 
 	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
 
+	srpc "github.com/vulcanize/ipld-eth-server/pkg/rpc"
 	s "github.com/vulcanize/ipld-eth-server/pkg/serve"
 	v "github.com/vulcanize/ipld-eth-server/version"
 )
@@ -68,54 +71,88 @@ func serve() {
 	if err := startServers(server, serverConfig); err != nil {
 		logWithCommand.Fatal(err)
 	}
+	graphQL, err := startGraphQL(server)
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
 
 	shutdown := make(chan os.Signal)
 	signal.Notify(shutdown, os.Interrupt)
 	<-shutdown
+	if graphQL != nil {
+		graphQL.Stop()
+	}
 	server.Stop()
 	wg.Wait()
 }
 
 func startServers(server s.Server, settings *s.Config) error {
 	logWithCommand.Info("starting up IPC server")
-	_, _, err := rpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
+	_, _, err := srpc.StartIPCEndpoint(settings.IPCEndpoint, server.APIs())
 	if err != nil {
 		return err
 	}
 	logWithCommand.Info("starting up WS server")
-	_, _, err = rpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb"}, nil, true)
+	_, _, err = srpc.StartWSEndpoint(settings.WSEndpoint, server.APIs(), []string{"vdb"}, nil, true)
 	if err != nil {
 		return err
 	}
 	logWithCommand.Info("starting up HTTP server")
-	_, _, err = rpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"eth"}, nil, nil, rpc.HTTPTimeouts{})
+	_, err = srpc.StartHTTPEndpoint(settings.HTTPEndpoint, server.APIs(), []string{"eth"}, nil, []string{"*"}, rpc.HTTPTimeouts{})
 	return err
 }
 
+func startGraphQL(server s.Server) (graphQLServer *graphql.Service, err error) {
+	viper.BindEnv("server.graphql", "SERVER_GRAPHQL")
+	if viper.GetBool("server.graphql") {
+		logWithCommand.Info("starting up GraphQL server")
+		viper.BindEnv("server.graphqlEndpoint", "SERVER_GRAPHQL_ENDPOINT")
+		endPoint := viper.GetString("server.graphqlEndpoint")
+		if endPoint != "" {
+			graphQLServer, err = graphql.New(server.Backend(), endPoint, nil, []string{"*"}, rpc.HTTPTimeouts{})
+			if err != nil {
+				return
+			}
+			err = graphQLServer.Start(nil)
+		}
+	}
+	return
+}
+
 func init() {
 	rootCmd.AddCommand(serveCmd)
 
 	// flags for all config variables
+	serveCmd.PersistentFlags().Bool("server-graphql", false, "turn on the graphql server")
+	serveCmd.PersistentFlags().String("server-graphql-endpoint", "", "endpoint url for graphql server")
 	serveCmd.PersistentFlags().String("server-ws-path", "", "vdb server ws path")
 	serveCmd.PersistentFlags().String("server-http-path", "", "vdb server http path")
 	serveCmd.PersistentFlags().String("server-ipc-path", "", "vdb server ipc path")
 
-	serveCmd.PersistentFlags().String("eth-ws-path", "", "ws url for ethereum node")
 	serveCmd.PersistentFlags().String("eth-http-path", "", "http url for ethereum node")
 	serveCmd.PersistentFlags().String("eth-node-id", "", "eth node id")
-	serveCmd.PersistentFlags().String("eth-client-name", "", "eth client name")
-	serveCmd.PersistentFlags().String("eth-genesis-block", "", "eth genesis block hash")
-	serveCmd.PersistentFlags().String("eth-network-id", "", "eth network id")
+	serveCmd.PersistentFlags().String("eth-client-name", "Geth", "eth client name")
+	serveCmd.PersistentFlags().String("eth-genesis-block", "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3", "eth genesis block hash")
+	serveCmd.PersistentFlags().String("eth-network-id", "1", "eth network id")
+	serveCmd.PersistentFlags().String("eth-chain-id", "1", "eth chain id")
+	serveCmd.PersistentFlags().String("eth-default-sender", "", "default sender address")
+	serveCmd.PersistentFlags().String("eth-rpc-gas-cap", "", "rpc gas cap (for eth_Call execution)")
+	serveCmd.PersistentFlags().String("eth-chain-config", "", "json chain config file location")
 
 	// and their bindings
+	viper.BindPFlag("server.graphql", serveCmd.PersistentFlags().Lookup("server-graphql"))
+	viper.BindPFlag("server.graphqlEndpoint", serveCmd.PersistentFlags().Lookup("server-graphql-endpoint"))
 	viper.BindPFlag("server.wsPath", serveCmd.PersistentFlags().Lookup("server-ws-path"))
 	viper.BindPFlag("server.httpPath", serveCmd.PersistentFlags().Lookup("server-http-path"))
 	viper.BindPFlag("server.ipcPath", serveCmd.PersistentFlags().Lookup("server-ipc-path"))
 
-	viper.BindPFlag("ethereum.wsPath", serveCmd.PersistentFlags().Lookup("eth-ws-path"))
 	viper.BindPFlag("ethereum.httpPath", serveCmd.PersistentFlags().Lookup("eth-http-path"))
 	viper.BindPFlag("ethereum.nodeID", serveCmd.PersistentFlags().Lookup("eth-node-id"))
 	viper.BindPFlag("ethereum.clientName", serveCmd.PersistentFlags().Lookup("eth-client-name"))
 	viper.BindPFlag("ethereum.genesisBlock", serveCmd.PersistentFlags().Lookup("eth-genesis-block"))
 	viper.BindPFlag("ethereum.networkID", serveCmd.PersistentFlags().Lookup("eth-network-id"))
+	viper.BindPFlag("ethereum.chainID", serveCmd.PersistentFlags().Lookup("eth-chain-id"))
+	viper.BindPFlag("ethereum.defaultSender", serveCmd.PersistentFlags().Lookup("eth-default-sender"))
+	viper.BindPFlag("ethereum.rpcGasCap", serveCmd.PersistentFlags().Lookup("eth-rpc-gas-cap"))
+	viper.BindPFlag("ethereum.chainConfig", serveCmd.PersistentFlags().Lookup("eth-chain-config"))
 }

cmd/subscribe.go

@@ -28,8 +28,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-
 	"github.com/vulcanize/ipld-eth-server/pkg/client"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	w "github.com/vulcanize/ipld-eth-server/pkg/serve"
@@ -83,7 +81,7 @@ func subscribe() {
 				logWithCommand.Error(payload.Err)
 				continue
 			}
-			var ethData eth2.IPLDs
+			var ethData eth.IPLDs
 			if err := rlp.DecodeBytes(payload.Data, &ethData); err != nil {
 				logWithCommand.Error(err)
 				continue

db/migrations/00012_add_chain_id_to_nodes.sql

@@ -0,0 +1,21 @@
+-- +goose Up
+ALTER TABLE public.nodes
+ADD COLUMN chain_id INTEGER DEFAULT 1;
+
+ALTER TABLE public.nodes
+DROP CONSTRAINT node_uc;
+
+ALTER TABLE public.nodes
+ADD CONSTRAINT node_uc
+UNIQUE (genesis_block, network_id, node_id, chain_id);
+
+-- +goose Down
+ALTER TABLE public.nodes
+DROP CONSTRAINT node_uc;
+
+ALTER TABLE public.nodes
+ADD CONSTRAINT node_uc
+UNIQUE (genesis_block, network_id, node_id);
+
+ALTER TABLE public.nodes
+DROP COLUMN chain_id;

db/migrations/00013_potgraphile_triggers.sql

@@ -0,0 +1,69 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE FUNCTION eth.graphql_subscription() returns TRIGGER as $$
+declare
+    table_name text = TG_ARGV[0];
+    attribute text = TG_ARGV[1];
+    id text;
+begin
+    execute 'select $1.' || quote_ident(attribute)
+        using new
+        into id;
+    perform pg_notify('postgraphile:' || table_name,
+                      json_build_object(
+                              '__node__', json_build_array(
+                              table_name,
+                              id
+                          )
+                          )::text
+        );
+    return new;
+end;
+$$ language plpgsql;
+-- +goose StatementEnd
+
+CREATE TRIGGER header_cids_ai
+    after INSERT ON eth.header_cids
+    for each row
+    execute procedure eth.graphql_subscription('header_cids', 'id');
+
+CREATE TRIGGER receipt_cids_ai
+    after INSERT ON eth.receipt_cids
+    for each row
+    execute procedure eth.graphql_subscription('receipt_cids', 'id');
+
+CREATE TRIGGER state_accounts_ai
+    after INSERT ON eth.state_accounts
+    for each row
+    execute procedure eth.graphql_subscription('state_accounts', 'id');
+
+CREATE TRIGGER state_cids_ai
+    after INSERT ON eth.state_cids
+    for each row
+    execute procedure eth.graphql_subscription('state_cids', 'id');
+
+CREATE TRIGGER storage_cids_ai
+    after INSERT ON eth.storage_cids
+    for each row
+    execute procedure eth.graphql_subscription('storage_cids', 'id');
+
+CREATE TRIGGER transaction_cids_ai
+    after INSERT ON eth.transaction_cids
+    for each row
+    execute procedure eth.graphql_subscription('transaction_cids', 'id');
+
+CREATE TRIGGER uncle_cids_ai
+    after INSERT ON eth.uncle_cids
+    for each row
+    execute procedure eth.graphql_subscription('uncle_cids', 'id');
+
+-- +goose Down
+DROP TRIGGER uncle_cids_ai ON eth.uncle_cids;
+DROP TRIGGER transaction_cids_ai ON eth.transaction_cids;
+DROP TRIGGER storage_cids_ai ON eth.storage_cids;
+DROP TRIGGER state_cids_ai ON eth.state_cids;
+DROP TRIGGER state_accounts_ai ON eth.state_accounts;
+DROP TRIGGER receipt_cids_ai ON eth.receipt_cids;
+DROP TRIGGER header_cids_ai ON eth.header_cids;
+
+DROP FUNCTION eth.graphql_subscription();

db/migrations/00014_create_cid_indexes.sql

@@ -0,0 +1,121 @@
+-- +goose Up
+-- header indexes
+CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
+
+CREATE INDEX block_hash_index ON eth.header_cids USING btree (block_hash);
+
+CREATE INDEX header_cid_index ON eth.header_cids USING btree (cid);
+
+CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
+
+CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
+
+CREATE INDEX timestamp_index ON eth.header_cids USING brin (timestamp);
+
+-- transaction indexes
+CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
+
+CREATE INDEX tx_hash_index ON eth.transaction_cids USING btree (tx_hash);
+
+CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
+
+CREATE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
+
+CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
+
+CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
+
+-- receipt indexes
+CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id);
+
+CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
+
+CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
+
+CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
+
+CREATE INDEX rct_contract_hash_index ON eth.receipt_cids USING btree (contract_hash);
+
+CREATE INDEX rct_topic0_index ON eth.receipt_cids USING gin (topic0s);
+
+CREATE INDEX rct_topic1_index ON eth.receipt_cids USING gin (topic1s);
+
+CREATE INDEX rct_topic2_index ON eth.receipt_cids USING gin (topic2s);
+
+CREATE INDEX rct_topic3_index ON eth.receipt_cids USING gin (topic3s);
+
+CREATE INDEX rct_log_contract_index ON eth.receipt_cids USING gin (log_contracts);
+
+-- state node indexes
+CREATE INDEX state_header_id_index ON eth.state_cids USING btree (header_id);
+
+CREATE INDEX state_leaf_key_index ON eth.state_cids USING btree (state_leaf_key);
+
+CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid);
+
+CREATE INDEX state_mh_index ON eth.state_cids USING btree (mh_key);
+
+CREATE INDEX state_path_index ON eth.state_cids USING btree (state_path);
+
+-- storage node indexes
+CREATE INDEX storage_state_id_index ON eth.storage_cids USING btree (state_id);
+
+CREATE INDEX storage_leaf_key_index ON eth.storage_cids USING btree (storage_leaf_key);
+
+CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid);
+
+CREATE INDEX storage_mh_index ON eth.storage_cids USING btree (mh_key);
+
+CREATE INDEX storage_path_index ON eth.storage_cids USING btree (storage_path);
+
+-- state accounts indexes
+CREATE INDEX account_state_id_index ON eth.state_accounts USING btree (state_id);
+
+CREATE INDEX storage_root_index ON eth.state_accounts USING btree (storage_root);
+
+-- +goose Down
+-- state account indexes
+DROP INDEX eth.storage_root_index;
+DROP INDEX eth.account_state_id_index;
+
+-- storage node indexes
+DROP INDEX eth.storage_path_index;
+DROP INDEX eth.storage_mh_index;
+DROP INDEX eth.storage_cid_index;
+DROP INDEX eth.storage_leaf_key_index;
+DROP INDEX eth.storage_state_id_index;
+
+-- state node indexes
+DROP INDEX eth.state_path_index;
+DROP INDEX eth.state_mh_index;
+DROP INDEX eth.state_cid_index;
+DROP INDEX eth.state_leaf_key_index;
+DROP INDEX eth.state_header_id_index;
+
+-- receipt indexes
+DROP INDEX eth.rct_log_contract_index;
+DROP INDEX eth.rct_topic3_index;
+DROP INDEX eth.rct_topic2_index;
+DROP INDEX eth.rct_topic1_index;
+DROP INDEX eth.rct_topic0_index;
+DROP INDEX eth.rct_contract_hash_index;
+DROP INDEX eth.rct_contract_index;
+DROP INDEX eth.rct_mh_index;
+DROP INDEX eth.rct_cid_index;
+DROP INDEX eth.rct_tx_id_index;
+
+-- transaction indexes
+DROP INDEX eth.tx_src_index;
+DROP INDEX eth.tx_dst_index;
+DROP INDEX eth.tx_mh_index;
+DROP INDEX eth.tx_cid_index;
+DROP INDEX eth.tx_hash_index;
+DROP INDEX eth.tx_header_id_index;
+
+-- header indexes
+DROP INDEX eth.timestamp_index;
+DROP INDEX eth.state_root_index;
+DROP INDEX eth.header_mh_index;
+DROP INDEX eth.header_cid_index;
+DROP INDEX eth.block_hash_index;
+DROP INDEX eth.block_number_index;

db/migrations/00015_create_canonical_hash_finder_functions.sql

@@ -0,0 +1,48 @@
+-- +goose Up
+-- +goose StatementBegin
+-- returns the number of child headers that reference backwards to the header with the provided hash
+CREATE OR REPLACE FUNCTION header_weight(hash VARCHAR(66)) RETURNS BIGINT
+AS $$
+  WITH RECURSIVE validator AS (
+          SELECT block_hash, parent_hash, block_number
+          FROM eth.header_cids
+          WHERE block_hash = hash
+      UNION
+          SELECT eth.header_cids.block_hash, eth.header_cids.parent_hash, eth.header_cids.block_number
+          FROM eth.header_cids
+          INNER JOIN validator
+            ON eth.header_cids.parent_hash = validator.block_hash
+            AND eth.header_cids.block_number = validator.block_number + 1
+  )
+  SELECT COUNT(*) FROM validator;
+$$ LANGUAGE SQL;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+-- returns the id for the header at the provided height which is heaviest
+CREATE OR REPLACE FUNCTION canonical_header(height BIGINT) RETURNS INT AS
+$BODY$
+DECLARE
+  current_weight INT;
+  heaviest_weight INT DEFAULT 0;
+  heaviest_id INT;
+  r eth.header_cids%ROWTYPE;
+BEGIN
+  FOR r IN SELECT * FROM eth.header_cids
+  WHERE block_number = height
+  LOOP
+    SELECT INTO current_weight * FROM header_weight(r.block_hash);
+    IF current_weight > heaviest_weight THEN
+        heaviest_weight := current_weight;
+        heaviest_id := r.id;
+    END IF;
+  END LOOP;
+  RETURN heaviest_id;
+END
+$BODY$
+LANGUAGE 'plpgsql';
+-- +goose StatementEnd
+
+-- +goose Down
+DROP FUNCTION header_weight;
+DROP FUNCTION canonical_header;

db/migrations/00016_remove_deployment_from_tx_cids.sql

@@ -0,0 +1,7 @@
+-- +goose Up
+ALTER TABLE eth.transaction_cids
+DROP COLUMN deployment;
+
+-- +goose Down
+ALTER TABLE eth.transaction_cids
+ADD COLUMN deployment BOOL NOT NULL DEFAULT FALSE;

db/migrations/00017_state_and_storage_ids_use_big_serial.sql

@@ -0,0 +1,25 @@
+-- +goose Up
+ALTER TABLE eth.storage_cids
+ALTER COLUMN state_id TYPE BIGINT;
+
+ALTER TABLE eth.state_accounts
+ALTER COLUMN state_id TYPE BIGINT;
+
+ALTER TABLE eth.state_cids
+ALTER COLUMN id TYPE BIGINT;
+
+ALTER TABLE eth.storage_cids
+ALTER COLUMN id TYPE BIGINT;
+
+-- +goose Down
+ALTER TABLE eth.storage_cids
+ALTER COLUMN id TYPE INTEGER;
+
+ALTER TABLE eth.state_cids
+ALTER COLUMN id TYPE INTEGER;
+
+ALTER TABLE eth.state_accounts
+ALTER COLUMN state_id TYPE INTEGER;
+
+ALTER TABLE eth.storage_cids
+ALTER COLUMN state_id TYPE INTEGER;

db/migrations/00018_create_check_if_removed_functions.sql

@@ -0,0 +1,39 @@
+-- +goose Up
+-- +goose StatementBegin
+-- returns if a storage node at the provided path was removed in the range > the provided height and <= the provided block hash
+CREATE OR REPLACE FUNCTION was_storage_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
+AS $$
+  SELECT exists(SELECT 1
+					FROM eth.storage_cids
+            INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
+            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+					WHERE storage_path = path
+					AND block_number > height
+					AND block_number <= (SELECT block_number
+										FROM eth.header_cids
+										WHERE block_hash = hash)
+					AND storage_cids.node_type = 3
+					LIMIT 1);
+$$ LANGUAGE SQL;
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+-- returns if a state node at the provided path was removed in the range > the provided height and <= the provided block hash
+CREATE OR REPLACE FUNCTION was_state_removed(path BYTEA, height BIGINT, hash VARCHAR(66)) RETURNS BOOLEAN
+AS $$
+  SELECT exists(SELECT 1
+					FROM eth.state_cids
+            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+					WHERE state_path = path
+					AND block_number > height
+					AND block_number <= (SELECT block_number
+										FROM eth.header_cids
+										WHERE block_hash = hash)
+					AND state_cids.node_type = 3
+					LIMIT 1);
+$$ LANGUAGE SQL;
+-- +goose StatementEnd
+
+-- +goose Down
+DROP FUNCTION was_storage_removed;
+DROP FUNCTION was_state_removed;

db/migrations/00019_updated_canonical_hash_finder_functions.sql

@@ -0,0 +1,121 @@
+-- +goose Up
+-- +goose StatementBegin
+CREATE TYPE child_result AS (
+  has_child BOOLEAN,
+  children eth.header_cids[]
+);
+
+CREATE OR REPLACE FUNCTION has_child(hash VARCHAR(66), height BIGINT) RETURNS child_result AS
+$BODY$
+DECLARE
+  child_height INT;
+  temp_child eth.header_cids;
+  new_child_result child_result;
+BEGIN
+  child_height = height + 1;
+  -- short circuit if there are no children
+  SELECT exists(SELECT 1
+                FROM eth.header_cids
+                WHERE parent_hash = hash
+                AND block_number = child_height
+                LIMIT 1)
+  INTO new_child_result.has_child;
+  -- collect all the children for this header
+  IF new_child_result.has_child THEN
+    FOR temp_child IN
+      SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
+    LOOP
+      new_child_result.children = array_append(new_child_result.children, temp_child);
+    END LOOP;
+  END IF;
+  RETURN new_child_result;
+END
+$BODY$
+LANGUAGE 'plpgsql';
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE OR REPLACE FUNCTION canonical_header_from_array(headers eth.header_cids[]) RETURNS eth.header_cids AS
+$BODY$
+DECLARE
+  canonical_header eth.header_cids;
+  canonical_child eth.header_cids;
+  header eth.header_cids;
+  current_child_result child_result;
+  child_headers eth.header_cids[];
+  current_header_with_child eth.header_cids;
+  has_children_count INT DEFAULT 0;
+BEGIN
+  -- for each header in the provided set
+  FOREACH header IN ARRAY headers
+  LOOP
+    -- check if it has any children
+    current_child_result = has_child(header.block_hash, header.block_number);
+    IF current_child_result.has_child THEN
+      -- if it does, take note
+      has_children_count = has_children_count + 1;
+      current_header_with_child = header;
+      -- and add the children to the growing set of child headers
+      child_headers = array_cat(child_headers, current_child_result.children);
+    END IF;
+  END LOOP;
+  -- if none of the headers had children, none is more canonical than the other
+  IF has_children_count = 0 THEN
+    -- return the first one selected
+    SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
+  -- if only one header had children, it can be considered the heaviest/canonical header of the set
+  ELSIF has_children_count = 1 THEN
+    -- return the only header with a child
+    canonical_header = current_header_with_child;
+  -- if there are multiple headers with children
+  ELSE
+    -- find the canonical header from the child set
+    canonical_child = canonical_header_from_array(child_headers);
+    -- the header that is parent to this header, is the canonical header at this level
+    SELECT * INTO canonical_header FROM unnest(headers)
+    WHERE block_hash = canonical_child.parent_hash;
+  END IF;
+  RETURN canonical_header;
+END
+$BODY$
+LANGUAGE 'plpgsql';
+-- +goose StatementEnd
+
+-- +goose StatementBegin
+CREATE OR REPLACE FUNCTION canonical_header_id(height BIGINT) RETURNS INTEGER AS
+$BODY$
+DECLARE
+  canonical_header eth.header_cids;
+  headers eth.header_cids[];
+  header_count INT;
+  temp_header eth.header_cids;
+BEGIN
+  -- collect all headers at this height
+  FOR temp_header IN
+    SELECT * FROM eth.header_cids WHERE block_number = height
+  LOOP
+    headers = array_append(headers, temp_header);
+  END LOOP;
+  -- count the number of headers collected
+  header_count = array_length(headers, 1);
+  -- if we have less than 1 header, return NULL
+  IF header_count IS NULL OR header_count < 1 THEN
+    RETURN NULL;
+  -- if we have one header, return its id
+  ELSIF header_count = 1 THEN
+    RETURN headers[1].id;
+  -- if we have multiple headers we need to determine which one is canonical
+  ELSE
+    canonical_header = canonical_header_from_array(headers);
+    RETURN canonical_header.id;
+  END IF;
+END;
+$BODY$
+LANGUAGE 'plpgsql';
+-- +goose StatementEnd
+
+-- +goose Down
+DROP FUNCTION canonical_header_id;
+DROP FUNCTION canonical_header_from_array;
+DROP FUNCTION has_child;
+DROP TYPE child_result;

db/migrations/00020_add_poststate_and_status_to_receipt_cids.sql

@@ -0,0 +1,13 @@
+-- +goose Up
+ALTER TABLE eth.receipt_cids
+ADD COLUMN post_state VARCHAR(66);
+
+ALTER TABLE eth.receipt_cids
+ADD COLUMN post_status INTEGER;
+
+-- +goose Down
+ALTER TABLE eth.receipt_cids
+DROP COLUMN post_status;
+
+ALTER TABLE eth.receipt_cids
+DROP COLUMN post_state;

db/schema.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 12.1
--- Dumped by pg_dump version 12.1
+-- Dumped from database version 12.4
+-- Dumped by pg_dump version 12.4
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -65,6 +65,253 @@ COMMENT ON TABLE eth.header_cids IS '@name EthHeaderCids';
 COMMENT ON COLUMN eth.header_cids.node_id IS '@name EthNodeID';
 
 
+--
+-- Name: child_result; Type: TYPE; Schema: public; Owner: -
+--
+
+CREATE TYPE public.child_result AS (
+	has_child boolean,
+	children eth.header_cids[]
+);
+
+
+--
+-- Name: graphql_subscription(); Type: FUNCTION; Schema: eth; Owner: -
+--
+
+CREATE FUNCTION eth.graphql_subscription() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $_$
+declare
+    table_name text = TG_ARGV[0];
+    attribute text = TG_ARGV[1];
+    id text;
+begin
+    execute 'select $1.' || quote_ident(attribute)
+        using new
+        into id;
+    perform pg_notify('postgraphile:' || table_name,
+                      json_build_object(
+                              '__node__', json_build_array(
+                              table_name,
+                              id
+                          )
+                          )::text
+        );
+    return new;
+end;
+$_$;
+
+
+--
+-- Name: canonical_header(bigint); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.canonical_header(height bigint) RETURNS integer
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  current_weight INT;
+  heaviest_weight INT DEFAULT 0;
+  heaviest_id INT;
+  r eth.header_cids%ROWTYPE;
+BEGIN
+  FOR r IN SELECT * FROM eth.header_cids
+  WHERE block_number = height
+  LOOP
+    SELECT INTO current_weight * FROM header_weight(r.block_hash);
+    IF current_weight > heaviest_weight THEN
+        heaviest_weight := current_weight;
+        heaviest_id := r.id;
+    END IF;
+  END LOOP;
+  RETURN heaviest_id;
+END
+$$;
+
+
+--
+-- Name: canonical_header_from_array(eth.header_cids[]); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.canonical_header_from_array(headers eth.header_cids[]) RETURNS eth.header_cids
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  canonical_header eth.header_cids;
+  canonical_child eth.header_cids;
+  header eth.header_cids;
+  current_child_result child_result;
+  child_headers eth.header_cids[];
+  current_header_with_child eth.header_cids;
+  has_children_count INT DEFAULT 0;
+BEGIN
+  -- for each header in the provided set
+  FOREACH header IN ARRAY headers
+  LOOP
+    -- check if it has any children
+    current_child_result = has_child(header.block_hash, header.block_number);
+    IF current_child_result.has_child THEN
+      -- if it does, take note
+      has_children_count = has_children_count + 1;
+      current_header_with_child = header;
+      -- and add the children to the growing set of child headers
+      child_headers = array_cat(child_headers, current_child_result.children);
+    END IF;
+  END LOOP;
+  -- if none of the headers had children, none is more canonical than the other
+  IF has_children_count = 0 THEN
+    -- return the first one selected
+    SELECT * INTO canonical_header FROM unnest(headers) LIMIT 1;
+  -- if only one header had children, it can be considered the heaviest/canonical header of the set
+  ELSIF has_children_count = 1 THEN
+    -- return the only header with a child
+    canonical_header = current_header_with_child;
+  -- if there are multiple headers with children
+  ELSE
+    -- find the canonical header from the child set
+    canonical_child = canonical_header_from_array(child_headers);
+    -- the header that is parent to this header, is the canonical header at this level
+    SELECT * INTO canonical_header FROM unnest(headers)
+    WHERE block_hash = canonical_child.parent_hash;
+  END IF;
+  RETURN canonical_header;
+END
+$$;
+
+
+--
+-- Name: canonical_header_id(bigint); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.canonical_header_id(height bigint) RETURNS integer
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  canonical_header eth.header_cids;
+  headers eth.header_cids[];
+  header_count INT;
+  temp_header eth.header_cids;
+BEGIN
+  -- collect all headers at this height
+  FOR temp_header IN
+    SELECT * FROM eth.header_cids WHERE block_number = height
+  LOOP
+    headers = array_append(headers, temp_header);
+  END LOOP;
+  -- count the number of headers collected
+  header_count = array_length(headers, 1);
+  -- if we have less than 1 header, return NULL
+  IF header_count IS NULL OR header_count < 1 THEN
+    RETURN NULL;
+  -- if we have one header, return its id
+  ELSIF header_count = 1 THEN
+    RETURN headers[1].id;
+  -- if we have multiple headers we need to determine which one is canonical
+  ELSE
+    canonical_header = canonical_header_from_array(headers);
+    RETURN canonical_header.id;
+  END IF;
+END;
+$$;
+
+
+--
+-- Name: has_child(character varying, bigint); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.has_child(hash character varying, height bigint) RETURNS public.child_result
+    LANGUAGE plpgsql
+    AS $$
+DECLARE
+  child_height INT;
+  temp_child eth.header_cids;
+  new_child_result child_result;
+BEGIN
+  child_height = height + 1;
+  -- short circuit if there are no children
+  SELECT exists(SELECT 1
+                FROM eth.header_cids
+                WHERE parent_hash = hash
+                AND block_number = child_height
+                LIMIT 1)
+  INTO new_child_result.has_child;
+  -- collect all the children for this header
+  IF new_child_result.has_child THEN
+    FOR temp_child IN
+      SELECT * FROM eth.header_cids WHERE parent_hash = hash AND block_number = child_height
+    LOOP
+      new_child_result.children = array_append(new_child_result.children, temp_child);
+    END LOOP;
+  END IF;
+  RETURN new_child_result;
+END
+$$;
+
+
+--
+-- Name: header_weight(character varying); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.header_weight(hash character varying) RETURNS bigint
+    LANGUAGE sql
+    AS $$
+  WITH RECURSIVE validator AS (
+          SELECT block_hash, parent_hash, block_number
+          FROM eth.header_cids
+          WHERE block_hash = hash
+      UNION
+          SELECT eth.header_cids.block_hash, eth.header_cids.parent_hash, eth.header_cids.block_number
+          FROM eth.header_cids
+          INNER JOIN validator
+            ON eth.header_cids.parent_hash = validator.block_hash
+            AND eth.header_cids.block_number = validator.block_number + 1
+  )
+  SELECT COUNT(*) FROM validator;
+$$;
+
+
+--
+-- Name: was_state_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.was_state_removed(path bytea, height bigint, hash character varying) RETURNS boolean
+    LANGUAGE sql
+    AS $$
+  SELECT exists(SELECT 1
+					FROM eth.state_cids
+            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+					WHERE state_path = path
+					AND block_number > height
+					AND block_number <= (SELECT block_number
+										FROM eth.header_cids
+										WHERE block_hash = hash)
+					AND state_cids.node_type = 3
+					LIMIT 1);
+$$;
+
+
+--
+-- Name: was_storage_removed(bytea, bigint, character varying); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.was_storage_removed(path bytea, height bigint, hash character varying) RETURNS boolean
+    LANGUAGE sql
+    AS $$
+  SELECT exists(SELECT 1
+					FROM eth.storage_cids
+            INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
+            INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+					WHERE storage_path = path
+					AND block_number > height
+					AND block_number <= (SELECT block_number
+										FROM eth.header_cids
+										WHERE block_hash = hash)
+					AND storage_cids.node_type = 3
+					LIMIT 1);
+$$;
+
+
 --
 -- Name: header_cids_id_seq; Type: SEQUENCE; Schema: eth; Owner: -
 --
@@ -100,7 +347,9 @@ CREATE TABLE eth.receipt_cids (
     topic1s character varying(66)[],
     topic2s character varying(66)[],
     topic3s character varying(66)[],
-    log_contracts character varying(66)[]
+    log_contracts character varying(66)[],
+    post_state character varying(66),
+    post_status integer
 );
 
 
@@ -130,7 +379,7 @@ ALTER SEQUENCE eth.receipt_cids_id_seq OWNED BY eth.receipt_cids.id;
 
 CREATE TABLE eth.state_accounts (
     id integer NOT NULL,
-    state_id integer NOT NULL,
+    state_id bigint NOT NULL,
     balance numeric NOT NULL,
     nonce integer NOT NULL,
     code_hash bytea NOT NULL,
@@ -163,7 +412,7 @@ ALTER SEQUENCE eth.state_accounts_id_seq OWNED BY eth.state_accounts.id;
 --
 
 CREATE TABLE eth.state_cids (
-    id integer NOT NULL,
+    id bigint NOT NULL,
     header_id integer NOT NULL,
     state_leaf_key character varying(66),
     cid text NOT NULL,
@@ -199,8 +448,8 @@ ALTER SEQUENCE eth.state_cids_id_seq OWNED BY eth.state_cids.id;
 --
 
 CREATE TABLE eth.storage_cids (
-    id integer NOT NULL,
-    state_id integer NOT NULL,
+    id bigint NOT NULL,
+    state_id bigint NOT NULL,
     storage_leaf_key character varying(66),
     cid text NOT NULL,
     mh_key text NOT NULL,
@@ -243,7 +492,6 @@ CREATE TABLE eth.transaction_cids (
     mh_key text NOT NULL,
     dst character varying(66) NOT NULL,
     src character varying(66) NOT NULL,
-    deployment boolean NOT NULL,
     tx_data bytea
 );
 
@@ -361,7 +609,8 @@ CREATE TABLE public.nodes (
     client_name character varying,
     genesis_block character varying(66),
     network_id character varying,
-    node_id character varying(128)
+    node_id character varying(128),
+    chain_id integer DEFAULT 1
 );
 
 
@@ -595,7 +844,7 @@ ALTER TABLE ONLY public.goose_db_version
 --
 
 ALTER TABLE ONLY public.nodes
-    ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id);
+    ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id, chain_id);
 
 
 --
@@ -606,6 +855,293 @@ ALTER TABLE ONLY public.nodes
     ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
 
 
+--
+-- Name: account_state_id_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX account_state_id_index ON eth.state_accounts USING btree (state_id);
+
+
+--
+-- Name: block_hash_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX block_hash_index ON eth.header_cids USING btree (block_hash);
+
+
+--
+-- Name: block_number_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX block_number_index ON eth.header_cids USING brin (block_number);
+
+
+--
+-- Name: header_cid_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX header_cid_index ON eth.header_cids USING btree (cid);
+
+
+--
+-- Name: header_mh_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX header_mh_index ON eth.header_cids USING btree (mh_key);
+
+
+--
+-- Name: rct_cid_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
+
+
+--
+-- Name: rct_contract_hash_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX rct_contract_hash_index ON eth.receipt_cids USING btree (contract_hash);
+
+
+--
+-- Name: rct_contract_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
+
+
+--
+-- Name: rct_log_contract_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX rct_log_contract_index ON eth.receipt_cids USING gin (log_contracts);
+
+
+--
+-- Name: rct_mh_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
+
+
+--
+-- Name: rct_topic0_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX rct_topic0_index ON eth.receipt_cids USING gin (topic0s);
+
+
+--
+-- Name: rct_topic1_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX rct_topic1_index ON eth.receipt_cids USING gin (topic1s);
+
+
+--
+-- Name: rct_topic2_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX rct_topic2_index ON eth.receipt_cids USING gin (topic2s);
+
+
+--
+-- Name: rct_topic3_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX rct_topic3_index ON eth.receipt_cids USING gin (topic3s);
+
+
+--
+-- Name: rct_tx_id_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id);
+
+
+--
+-- Name: state_cid_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX state_cid_index ON eth.state_cids USING btree (cid);
+
+
+--
+-- Name: state_header_id_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX state_header_id_index ON eth.state_cids USING btree (header_id);
+
+
+--
+-- Name: state_leaf_key_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX state_leaf_key_index ON eth.state_cids USING btree (state_leaf_key);
+
+
+--
+-- Name: state_mh_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX state_mh_index ON eth.state_cids USING btree (mh_key);
+
+
+--
+-- Name: state_path_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX state_path_index ON eth.state_cids USING btree (state_path);
+
+
+--
+-- Name: state_root_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX state_root_index ON eth.header_cids USING btree (state_root);
+
+
+--
+-- Name: storage_cid_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX storage_cid_index ON eth.storage_cids USING btree (cid);
+
+
+--
+-- Name: storage_leaf_key_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX storage_leaf_key_index ON eth.storage_cids USING btree (storage_leaf_key);
+
+
+--
+-- Name: storage_mh_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX storage_mh_index ON eth.storage_cids USING btree (mh_key);
+
+
+--
+-- Name: storage_path_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX storage_path_index ON eth.storage_cids USING btree (storage_path);
+
+
+--
+-- Name: storage_root_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX storage_root_index ON eth.state_accounts USING btree (storage_root);
+
+
+--
+-- Name: storage_state_id_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX storage_state_id_index ON eth.storage_cids USING btree (state_id);
+
+
+--
+-- Name: timestamp_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX timestamp_index ON eth.header_cids USING brin ("timestamp");
+
+
+--
+-- Name: tx_cid_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX tx_cid_index ON eth.transaction_cids USING btree (cid);
+
+
+--
+-- Name: tx_dst_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX tx_dst_index ON eth.transaction_cids USING btree (dst);
+
+
+--
+-- Name: tx_hash_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX tx_hash_index ON eth.transaction_cids USING btree (tx_hash);
+
+
+--
+-- Name: tx_header_id_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX tx_header_id_index ON eth.transaction_cids USING btree (header_id);
+
+
+--
+-- Name: tx_mh_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX tx_mh_index ON eth.transaction_cids USING btree (mh_key);
+
+
+--
+-- Name: tx_src_index; Type: INDEX; Schema: eth; Owner: -
+--
+
+CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
+
+
+--
+-- Name: header_cids header_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
+--
+
+CREATE TRIGGER header_cids_ai AFTER INSERT ON eth.header_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('header_cids', 'id');
+
+
+--
+-- Name: receipt_cids receipt_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
+--
+
+CREATE TRIGGER receipt_cids_ai AFTER INSERT ON eth.receipt_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('receipt_cids', 'id');
+
+
+--
+-- Name: state_accounts state_accounts_ai; Type: TRIGGER; Schema: eth; Owner: -
+--
+
+CREATE TRIGGER state_accounts_ai AFTER INSERT ON eth.state_accounts FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('state_accounts', 'id');
+
+
+--
+-- Name: state_cids state_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
+--
+
+CREATE TRIGGER state_cids_ai AFTER INSERT ON eth.state_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('state_cids', 'id');
+
+
+--
+-- Name: storage_cids storage_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
+--
+
+CREATE TRIGGER storage_cids_ai AFTER INSERT ON eth.storage_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('storage_cids', 'id');
+
+
+--
+-- Name: transaction_cids transaction_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
+--
+
+CREATE TRIGGER transaction_cids_ai AFTER INSERT ON eth.transaction_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('transaction_cids', 'id');
+
+
+--
+-- Name: uncle_cids uncle_cids_ai; Type: TRIGGER; Schema: eth; Owner: -
+--
+
+CREATE TRIGGER uncle_cids_ai AFTER INSERT ON eth.uncle_cids FOR EACH ROW EXECUTE FUNCTION eth.graphql_subscription('uncle_cids', 'id');
+
+
 --
 -- Name: header_cids header_cids_mh_key_fkey; Type: FK CONSTRAINT; Schema: eth; Owner: -
 --

environments/example.toml

@@ -11,4 +11,17 @@
 [server]
     ipcPath = "~/.vulcanize/vulcanize.ipc" # $SERVER_IPC_PATH
     wsPath = "127.0.0.1:8081" # $SERVER_WS_PATH
-    httpPath = "127.0.0.1:8082" # $SERVER_HTTP_PATH
+    httpPath = "127.0.0.1:8082" # $SERVER_HTTP_PATH
+    graphql = true # $SERVER_GRAPHQL
+    graphqlEndpoint = "127.0.0.1:8083" # $SERVER_GRAPHQL_ENDPOINT
+
+[ethereum]
+    chainID = "1" # $ETH_CHAIN_ID
+    defaultSender = "" # $ETH_DEFAULT_SENDER_ADDR
+    rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
+    httpPath = "127.0.0.1:8545" # $ETH_HTTP_PATH
+    supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF
+    nodeID = "arch1" # $ETH_NODE_ID
+    clientName = "Geth" # $ETH_CLIENT_NAME
+    genesisBlock = "0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3" # $ETH_GENESIS_BLOCK
+    networkID = "1" # $ETH_NETWORK_ID

go.mod

@@ -3,21 +3,27 @@ module github.com/vulcanize/ipld-eth-server
 go 1.13
 
 require (
-	github.com/ethereum/go-ethereum v1.9.11
-	github.com/ipfs/go-cid v0.0.5
-	github.com/ipfs/go-ipfs-blockstore v1.0.0
+	github.com/ethereum/go-ethereum v1.9.25
+	github.com/graph-gophers/graphql-go v0.0.0-20201003130358-c5bdf3b1108e
+	github.com/ipfs/go-block-format v0.0.2
+	github.com/ipfs/go-cid v0.0.7
+	github.com/ipfs/go-ipfs-blockstore v1.0.1
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
 	github.com/ipfs/go-ipld-format v0.2.0
 	github.com/jmoiron/sqlx v1.2.0
-	github.com/lib/pq v1.5.2
-	github.com/multiformats/go-multihash v0.0.13
-	github.com/onsi/ginkgo v1.12.1
+	github.com/lib/pq v1.8.0
+	github.com/multiformats/go-multihash v0.0.14
+	github.com/onsi/ginkgo v1.15.0
 	github.com/onsi/gomega v1.10.1
+	github.com/prometheus/client_golang v1.5.1
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/viper v1.7.0
-	github.com/vulcanize/ipld-eth-indexer v0.2.0-alpha
-	github.com/vulcanize/pg-ipfs-ethdb v0.0.1-alpha
+	github.com/vulcanize/ipld-eth-indexer v0.7.1-alpha
+	github.com/vulcanize/ipfs-ethdb v0.0.2-alpha
+	golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b // indirect
 )
 
-replace github.com/ethereum/go-ethereum v1.9.11 => github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5
+replace github.com/ethereum/go-ethereum v1.9.25 => github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.15
+
+replace github.com/vulcanize/ipfs-ethdb v0.0.2-alpha => github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha

go.sum

@@ -35,18 +35,18 @@ github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxB
 github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
 github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
-github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3oM7gXIttpYDAJXpVNnSCiUMYBLIZ6cb1t+Ip982MRo=
 github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s=
 github.com/VictoriaMetrics/fastcache v1.5.3 h1:2odJnXLbFZcoV9KYtQ+7TH1UOq3dn3AssMgieaezkR4=
 github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE=
+github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw=
+github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -54,7 +54,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
-github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A=
@@ -66,6 +65,7 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
@@ -78,7 +78,6 @@ github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcug
 github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
 github.com/btcsuite/btcd v0.0.0-20190629003639-c26ffa870fd8/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
 github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI=
-github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw=
 github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
 github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
@@ -91,7 +90,6 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
-github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -136,12 +134,17 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
 github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA=
+github.com/dop251/goja v0.0.0-20200219165308-d1232e640a87/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA=
+github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c h1:JHHhtb9XWJrGNMcrVP6vyzO4dusgi/HnceHTgxSejUM=
 github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa h1:XKAhUk/dtp+CV0VO6mhG2V7jA9vbcGcnYF/Ay9NjZrY=
 github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa/go.mod h1:cdorVVzy1fhmEqmtgqkoE3bYtCfSCkVyjTyCIo22xvs=
 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302/go.mod h1:qBlWZqWeVx9BjvqBsnC/8RUlAYpIFmPvgROcw0n1scE=
+github.com/ethereum/go-ethereum v1.9.11/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
+github.com/ethereum/go-ethereum v1.9.15/go.mod h1:slT8bPPRhXsyNTwHQxrOnjuTZ1sDXRajW11EkJ84QJ0=
 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg=
 github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -163,20 +166,17 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
-github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
@@ -204,16 +204,18 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 h1:ur2rms48b3Ep1dxh7aUV2FZEQ8jEVO2F6ILKx8ofkAg=
+github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -226,7 +228,6 @@ github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE0
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4=
 github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
@@ -234,6 +235,8 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
+github.com/graph-gophers/graphql-go v0.0.0-20201003130358-c5bdf3b1108e h1:IpssFbpfPSx/3c7x601Npx+UOQ4tqd0Rk4sObCQ+zlQ=
+github.com/graph-gophers/graphql-go v0.0.0-20201003130358-c5bdf3b1108e/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -268,14 +271,16 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/holiman/uint256 v1.1.1 h1:4JywC80b+/hSfljFlEBLHrrh+CIONLDz9NuFl0af4Mw=
+github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huin/goupnp v0.0.0-20161224104101-679507af18f3/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag=
 github.com/huin/goupnp v0.0.0-20180415215157-1395d1447324/go.mod h1:MZ2ZmwcBpvOoJ22IJsc7va19ZwoheaBk43rKg12SKag=
 github.com/huin/goupnp v1.0.0 h1:wg75sLpL6DZqwHQN6E1Cfk6mtfzS45z8OV+ic+DtHRo=
 github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc=
 github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 h1:FSeK4fZCo8u40n2JMnyAsd6x7+SbvoOMHvQOU/n10P4=
 github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
 github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
@@ -295,6 +300,7 @@ github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbR
 github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
 github.com/ipfs/go-blockservice v0.1.1/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
 github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
+github.com/ipfs/go-blockservice v0.1.3 h1:9XgsPMwwWJSC9uVr2pMDsW2qFTBSkxpGMhmna8mIjPM=
 github.com/ipfs/go-blockservice v0.1.3/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
@@ -302,6 +308,8 @@ github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUP
 github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
 github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU=
 github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
+github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY=
+github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I=
 github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s=
 github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
 github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
@@ -311,6 +319,8 @@ github.com/ipfs/go-datastore v0.3.0/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRV
 github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
 github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
+github.com/ipfs/go-datastore v0.4.2 h1:h8/n7WPzhp239kkLws+epN3Ic7YtcBPgcaXfEfdVDWM=
+github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8=
 github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA=
 github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps=
@@ -336,6 +346,8 @@ github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2Is
 github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ=
 github.com/ipfs/go-ipfs-blockstore v1.0.0 h1:pmFp5sFYsYVvMOp9X01AK3s85usVcLvkBTRsN6SnfUA=
 github.com/ipfs/go-ipfs-blockstore v1.0.0/go.mod h1:knLVdhVU9L7CC4T+T4nvGdeUIPAXlnd9zmXfp+9MIjU=
+github.com/ipfs/go-ipfs-blockstore v1.0.1 h1:fnuVj4XdZp4yExhd0CnUwAiMNJHiPnfInhiuwz4lW1w=
+github.com/ipfs/go-ipfs-blockstore v1.0.1/go.mod h1:MGNZlHNEnR4KGgPHM3/k8lBySIOK2Ve+0KjZubKlaOE=
 github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk=
 github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw=
 github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
@@ -347,6 +359,7 @@ github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWF
 github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs=
 github.com/ipfs/go-ipfs-ds-help v1.0.0 h1:bEQ8hMGs80h0sR8O4tfDgV6B01aaF9qeTrujrTLYV3g=
 github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE=
+github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6rY30GUu7G6LUfpMvM=
 github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM=
 github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0=
 github.com/ipfs/go-ipfs-files v0.0.2/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
@@ -373,6 +386,7 @@ github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL3
 github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs=
 github.com/ipfs/go-ipld-git v0.0.3/go.mod h1:RuvMXa9qtJpDbqngyICCU/d+cmLFXxLsbIclmD0Lcr0=
 github.com/ipfs/go-ipns v0.0.2/go.mod h1:WChil4e0/m9cIINWLxZe1Jtf77oz5L05rO2ei/uKJ5U=
+github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
 github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I=
 github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
@@ -406,6 +420,7 @@ github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vU
 github.com/ipfs/go-unixfs v0.1.0/go.mod h1:lysk5ELhOso8+Fed9U1QTGey2ocsfaZ18h0NCO2Fj9s=
 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
 github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw=
+github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
 github.com/ipfs/interface-go-ipfs-core v0.2.7/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s=
 github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g=
@@ -414,6 +429,7 @@ github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1
 github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
+github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA=
 github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
 github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
@@ -423,10 +439,13 @@ github.com/jbenet/go-is-domain v1.0.3/go.mod h1:xbRLRb0S7FgzDBTJlguhDVwLYM/5yNtv
 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c/go.mod h1:sdx1xVM9UuLw1tXnhJWN3piypTUO3vCIHYmG15KE/dU=
 github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs=
 github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk=
+github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 h1:bspPhN+oKYFk5fcGNuQzp6IGzYQSenLEgH3s6jkXrWw=
 github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY=
+github.com/jbenet/goprocess v0.1.3 h1:YKyIEECS/XvcfHtBzxtjBBbWK+MbvA6dG8ASiqwvr10=
 github.com/jbenet/goprocess v0.1.3/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
 github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o=
 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
+github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
@@ -439,7 +458,6 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -451,23 +469,21 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
 github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.5.2 h1:yTSXVswvWUOQ3k1sd7vJfDrbSl8lKuscqFJRqjC0ifw=
 github.com/lib/pq v1.5.2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
+github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ=
 github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ=
 github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
@@ -684,7 +700,9 @@ github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGD
 github.com/marten-seemann/qtls v0.9.1/go.mod h1:T1MmAdDPyISzxlK6kjRr0pcZFBVd1OZbBb/j3cvzHhk=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -693,16 +711,19 @@ github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HN
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
 github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
@@ -715,6 +736,7 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8Rv
 github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
 github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo=
 github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
@@ -738,6 +760,8 @@ github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc=
 github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
 github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI=
 github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA=
+github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4=
+github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM=
 github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
 github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
 github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44=
@@ -761,6 +785,8 @@ github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysj
 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
 github.com/multiformats/go-multibase v0.0.2 h1:2pAgScmS1g9XjH7EtAfNhTuyrWYEWcxy0G5Wo85hWDA=
 github.com/multiformats/go-multibase v0.0.2/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs=
+github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk=
+github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc=
 github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U=
 github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po=
 github.com/multiformats/go-multihash v0.0.6/go.mod h1:XuKXPp8VHcTygube3OWZC+aZrA+H1IhmjoCDtJc7PXM=
@@ -770,6 +796,8 @@ github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa
 github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
 github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc=
 github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
+github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
+github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
 github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
 github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
@@ -785,6 +813,8 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c h1:1RHs3tNxjXGHeul8z2t6H2N2TlAqpKe5yryJztRx4Jk=
@@ -796,6 +826,9 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4=
+github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
@@ -812,12 +845,13 @@ github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222 h1:goeTyGkArOZIVOMA0d
 github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM=
 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
 github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
@@ -827,21 +861,25 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA=
 github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=
@@ -850,15 +888,20 @@ github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeC
 github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 h1:8DPul/X0IT/1TNMIxoKLwdemEOBBHDC/K4EB16Cw5WE=
 github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM5v0CgENFktkkbg1sfpgM3h20=
 github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil v2.20.5-0.20200531151128-663af789c085+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/gopsutil v2.20.5+incompatible h1:tYH07UPoQt0OCQdgWWMgYHy3/a9bcxNpBIysykNIP7I=
+github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
 github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
 github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
@@ -887,12 +930,10 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8=
 github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -931,13 +972,14 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs=
 github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
+github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk=
+github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -951,10 +993,20 @@ github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49u
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5 h1:U+BqhjRLR22e9OEm8cgWC3Eq3bh8G6azjNpXeenfCG4=
 github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.5/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
-github.com/vulcanize/ipld-eth-indexer v0.2.0-alpha h1:+XVaC7TsA0K278YWpfqdrNxwgC6hY6fBaN8w2/e1Lts=
-github.com/vulcanize/ipld-eth-indexer v0.2.0-alpha/go.mod h1:SuMBscFfcBHYlQuzDDd4by+R0S3gaAFjrOU+uQfAefE=
-github.com/vulcanize/pg-ipfs-ethdb v0.0.1-alpha h1:Y7j0Hw1jgVVOg+eUGUr7OgH+gOBID0DwbsfZV1KoL7I=
-github.com/vulcanize/pg-ipfs-ethdb v0.0.1-alpha/go.mod h1:OuqE4r2LGWAtDVx3s1yaAzDcwy+LEAqrWaE1L8UfrGY=
+github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.8 h1:7TK52k55uvSl+1SCKYYFelzH1NrpvEcDrpeU9nUDIpI=
+github.com/vulcanize/go-ethereum v1.9.11-statediff-0.0.8/go.mod h1:7oC0Ni6dosMv5pxMigm6s0hN8g4haJMBnqmmo0D9YfQ=
+github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.14 h1:DFN3WmdStbNWfTx5qOGDlZkoUwVxlcs4fGqPwzczE+8=
+github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.14/go.mod h1:q5X5BpZkLp+OgWByNdZ8NudGkq0Vq4LjdrtmyyICUw4=
+github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.15 h1:f/frl9tMTKCcu2yh1tOgwXeQn3GJ2+21Zp0lSZzl1v0=
+github.com/vulcanize/go-ethereum v1.9.25-statediff-0.0.15/go.mod h1:q5X5BpZkLp+OgWByNdZ8NudGkq0Vq4LjdrtmyyICUw4=
+github.com/vulcanize/ipfs-ethdb v0.0.2 h1:Ogp9k+OMKIn2xo9Y9VoNTjd5v1kXcjL+k3cDQwqpUPU=
+github.com/vulcanize/ipfs-ethdb v0.0.2/go.mod h1:9yDJ8PCVKK4auwM8KgVLVkGM05Atp2OITJvt6cSvsHs=
+github.com/vulcanize/ipld-eth-indexer v0.7.0-alpha h1:VuvKC2qLajohczJX74lwBrpr/X8ZNSyC23jIdN5Ppxg=
+github.com/vulcanize/ipld-eth-indexer v0.7.0-alpha/go.mod h1:LYN2c28N9ceDf66tppc+TpaLqs4UcUlLTvUvg3Awccc=
+github.com/vulcanize/ipld-eth-indexer v0.7.1-alpha h1:By8Yyt8IjneVhL+xne/V6VLumKcUx06iE5Kq2rFphzA=
+github.com/vulcanize/ipld-eth-indexer v0.7.1-alpha/go.mod h1:LYN2c28N9ceDf66tppc+TpaLqs4UcUlLTvUvg3Awccc=
+github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha h1:wvhpjrRuxjudAael5zhNwGymce/ggvOhHo1q9VJ0NjU=
+github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha/go.mod h1:WIceyu15uxQ4JanLxzKXn4iAyJJSmHAJNWux7qYXGEQ=
 github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=
 github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
@@ -962,6 +1014,7 @@ github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45
 github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc=
+github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo=
 github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM=
 github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE=
 github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8=
@@ -982,6 +1035,7 @@ github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+m
 github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -1001,7 +1055,6 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
 go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
-go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo=
@@ -1027,14 +1080,19 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 h1:Q7tZBpemrlsc2I7IyODzhtallWRSm4Q0d09pL6XbQtU=
 golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
 golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
@@ -1047,13 +1105,16 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1083,9 +1144,14 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 h1:eDrdRpKgkcCqKZQwyZRyeFZgfqt37SL7Kv3tok06cKE=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1099,6 +1165,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1141,13 +1208,26 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8=
 golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210218145245-beda7e5e158e h1:f5mksnk+hgXHnImpZoWj64ja99j9zV7YUgrVG95uFE4=
+golang.org/x/sys v0.0.0-20210218145245-beda7e5e158e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b h1:lAZ0/chPUDWwjqosYR0X4M490zQhMsiJ4K3DbA7o+3g=
+golang.org/x/sys v0.0.0-20210218155724-8ebf48af031b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1175,13 +1255,20 @@ golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191114200427-caa0b0f7d508/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
@@ -1195,7 +1282,6 @@ google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -1227,7 +1313,6 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
@@ -1235,9 +1320,11 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU=
 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c=
 gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
+gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200603215123-a4a8cb9d2cbc/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
+gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0=
+gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
 gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8=
@@ -1259,7 +1346,6 @@ honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

monitoring/grafana/dashboard_main.json

@@ -0,0 +1,353 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 4,
+  "links": [],
+  "panels": [
+    {
+      "datasource": null,
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "7.2.1",
+      "targets": [
+        {
+          "expr": "ipld_eth_server_ws_count",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Websocket Connection Count",
+      "type": "gauge"
+    },
+    {
+      "datasource": null,
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 5,
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "mean"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true
+      },
+      "pluginVersion": "7.2.1",
+      "targets": [
+        {
+          "expr": "ipld_eth_server_ipc_count",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "IPC Connection Count",
+      "type": "gauge"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "rate(ipld_eth_server_http_count[1m])",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "HTTP requests per second",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": null,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "hiddenSeries": false,
+      "id": 7,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.9, rate(ipld_eth_server_http_duration_bucket[1m]))",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "0.9",
+          "refId": "B"
+        },
+        {
+          "expr": "histogram_quantile(0.75, rate(ipld_eth_server_http_duration_bucket[1m]))",
+          "interval": "",
+          "legendFormat": "0.75",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.5, rate(ipld_eth_server_http_duration_bucket[1m]))",
+          "interval": "",
+          "legendFormat": "0.5",
+          "refId": "C"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "HTTP Requests duration (percentile)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "s",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 26,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "ipld-eth-server",
+  "uid": "lFVEvNtGk",
+  "version": 7
+}

monitoring/prometheus.yml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 10s
+
+scrape_configs:
+  - job_name: 'ipld-eth-server'
+    static_configs:
+    - targets: ['localhost:8090']

pkg/eth/api.go

@@ -18,16 +18,28 @@ package eth
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
 	"math/big"
-
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
+	"time"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
 // APIName is the namespace for the watcher's eth api
@@ -37,14 +49,80 @@ const APIName = "eth"
 const APIVersion = "0.0.1"
 
 type PublicEthAPI struct {
+	// Local db backend
 	B *Backend
+
+	// Proxy node for forwarding cache misses
+	supportsStateDiff bool // Whether or not the remote node supports the statediff_writeStateDiffAt endpoint, if it does we can fill the local cache when we hit a miss
+	rpc               *rpc.Client
+	ethClient         *ethclient.Client
 }
 
 // NewPublicEthAPI creates a new PublicEthAPI with the provided underlying Backend
-func NewPublicEthAPI(b *Backend) *PublicEthAPI {
-	return &PublicEthAPI{
-		B: b,
+func NewPublicEthAPI(b *Backend, client *rpc.Client, supportsStateDiff bool) *PublicEthAPI {
+	var ethClient *ethclient.Client
+	if client != nil {
+		ethClient = ethclient.NewClient(client)
 	}
+	return &PublicEthAPI{
+		B:                 b,
+		supportsStateDiff: supportsStateDiff,
+		rpc:               client,
+		ethClient:         ethClient,
+	}
+}
+
+/*
+
+Headers and blocks
+
+*/
+
+// GetHeaderByNumber returns the requested canonical block header.
+// * When blockNr is -1 the chain head is returned.
+// * We cannot support pending block calls since we do not have an active miner
+func (pea *PublicEthAPI) GetHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (map[string]interface{}, error) {
+	header, err := pea.B.HeaderByNumber(ctx, number)
+	if header != nil && err == nil {
+		return pea.rpcMarshalHeader(header)
+	}
+	if pea.ethClient != nil {
+		if header, err := pea.ethClient.HeaderByNumber(ctx, big.NewInt(number.Int64())); header != nil && err == nil {
+			go pea.writeStateDiffAt(number.Int64())
+			return pea.rpcMarshalHeader(header)
+		}
+	}
+	return nil, err
+}
+
+// GetHeaderByHash returns the requested header by hash.
+func (pea *PublicEthAPI) GetHeaderByHash(ctx context.Context, hash common.Hash) map[string]interface{} {
+	header, err := pea.B.HeaderByHash(ctx, hash)
+	if header != nil && err == nil {
+		if res, err := pea.rpcMarshalHeader(header); err == nil {
+			return res
+		}
+	}
+	if pea.ethClient != nil {
+		if header, err := pea.ethClient.HeaderByHash(ctx, hash); header != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
+			if res, err := pea.rpcMarshalHeader(header); err != nil {
+				return res
+			}
+		}
+	}
+	return nil
+}
+
+// rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field
+func (pea *PublicEthAPI) rpcMarshalHeader(header *types.Header) (map[string]interface{}, error) {
+	fields := RPCMarshalHeader(header)
+	td, err := pea.B.GetTd(header.Hash())
+	if err != nil {
+		return nil, err
+	}
+	fields["totalDifficulty"] = (*hexutil.Big)(td)
+	return fields, nil
 }
 
 // BlockNumber returns the block number of the chain head.
@@ -53,10 +131,396 @@ func (pea *PublicEthAPI) BlockNumber() hexutil.Uint64 {
 	return hexutil.Uint64(number)
 }
 
+// GetBlockByNumber returns the requested canonical block.
+// * When blockNr is -1 the chain head is returned.
+// * We cannot support pending block calls since we do not have an active miner
+// * When fullTx is true all transactions in the block are returned, otherwise
+//   only the transaction hash is returned.
+func (pea *PublicEthAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) {
+	block, err := pea.B.BlockByNumber(ctx, number)
+	if block != nil && err == nil {
+		return pea.rpcMarshalBlock(block, true, fullTx)
+	}
+	if pea.ethClient != nil {
+		if block, err := pea.ethClient.BlockByNumber(ctx, big.NewInt(number.Int64())); block != nil && err == nil {
+			go pea.writeStateDiffAt(number.Int64())
+			return pea.rpcMarshalBlock(block, true, fullTx)
+		}
+	}
+	return nil, err
+}
+
+// GetBlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full
+// detail, otherwise only the transaction hash is returned.
+func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, fullTx bool) (map[string]interface{}, error) {
+	block, err := pea.B.BlockByHash(ctx, hash)
+	if block != nil && err == nil {
+		return pea.rpcMarshalBlock(block, true, fullTx)
+	}
+	if pea.ethClient != nil {
+		if block, err := pea.ethClient.BlockByHash(ctx, hash); block != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
+			return pea.rpcMarshalBlock(block, true, fullTx)
+		}
+	}
+	return nil, err
+}
+
+/*
+
+Uncles
+
+*/
+
+// GetUncleByBlockNumberAndIndex returns the uncle block for the given block hash and index. When fullTx is true
+// all transactions in the block are returned in full detail, otherwise only the transaction hash is returned.
+func (pea *PublicEthAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (map[string]interface{}, error) {
+	block, err := pea.B.BlockByNumber(ctx, blockNr)
+	if block != nil && err == nil {
+		uncles := block.Uncles()
+		if index >= hexutil.Uint(len(uncles)) {
+			logrus.Debugf("uncle with index %s request at block number %d was not found", index.String(), blockNr.Int64())
+			return nil, nil
+		}
+		block = types.NewBlockWithHeader(uncles[index])
+		return pea.rpcMarshalBlock(block, false, false)
+	}
+	if pea.rpc != nil {
+		if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockNumberAndIndex", blockNr, index); uncle != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
+			return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false)
+		}
+	}
+	return nil, err
+}
+
+// GetUncleByBlockHashAndIndex returns the uncle block for the given block hash and index. When fullTx is true
+// all transactions in the block are returned in full detail, otherwise only the transaction hash is returned.
+func (pea *PublicEthAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) (map[string]interface{}, error) {
+	block, err := pea.B.BlockByHash(ctx, blockHash)
+	if block != nil {
+		uncles := block.Uncles()
+		if index >= hexutil.Uint(len(uncles)) {
+			logrus.Debugf("uncle with index %s request at block hash %s was not found", index.String(), blockHash.Hex())
+			return nil, nil
+		}
+		block = types.NewBlockWithHeader(uncles[index])
+		return pea.rpcMarshalBlock(block, false, false)
+	}
+	if pea.rpc != nil {
+		if uncle, uncleHashes, err := getBlockAndUncleHashes(pea.rpc, ctx, "eth_getUncleByBlockHashAndIndex", blockHash, index); uncle != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
+			return pea.rpcMarshalBlockWithUncleHashes(uncle, uncleHashes, false, false)
+		}
+	}
+	return nil, err
+}
+
+// GetUncleCountByBlockNumber returns number of uncles in the block for the given block number
+func (pea *PublicEthAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint {
+	if block, err := pea.B.BlockByNumber(ctx, blockNr); block != nil && err == nil {
+		n := hexutil.Uint(len(block.Uncles()))
+		return &n
+	}
+	if pea.rpc != nil {
+		var num *hexutil.Uint
+		if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockNumber", blockNr); num != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
+			return num
+		}
+	}
+	return nil
+}
+
+// GetUncleCountByBlockHash returns number of uncles in the block for the given block hash
+func (pea *PublicEthAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint {
+	if block, err := pea.B.BlockByHash(ctx, blockHash); block != nil && err == nil {
+		n := hexutil.Uint(len(block.Uncles()))
+		return &n
+	}
+	if pea.rpc != nil {
+		var num *hexutil.Uint
+		if err := pea.rpc.CallContext(ctx, &num, "eth_getUncleCountByBlockHash", blockHash); num != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
+			return num
+		}
+	}
+	return nil
+}
+
+/*
+
+Transactions
+
+*/
+
+// GetTransactionCount returns the number of transactions the given address has sent for the given block number
+func (pea *PublicEthAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) {
+	count, err := pea.localGetTransactionCount(ctx, address, blockNrOrHash)
+	if count != nil && err == nil {
+		return count, nil
+	}
+	if pea.rpc != nil {
+		var num *hexutil.Uint64
+		if err := pea.rpc.CallContext(ctx, &num, "eth_getTransactionCount", address, blockNrOrHash); num != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
+			return num, nil
+		}
+	}
+	return nil, err
+}
+
+func (pea *PublicEthAPI) localGetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) {
+	account, err := pea.B.GetAccountByNumberOrHash(ctx, address, blockNrOrHash)
+	if err != nil {
+		return nil, err
+	}
+	nonce := hexutil.Uint64(account.Nonce)
+	return &nonce, nil
+}
+
+// GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number.
+func (pea *PublicEthAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint {
+	if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil {
+		n := hexutil.Uint(len(block.Transactions()))
+		return &n
+	}
+	if pea.rpc != nil {
+		var num *hexutil.Uint
+		if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByNumber", blockNr); num != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
+			return num
+		}
+	}
+	return nil
+}
+
+// GetBlockTransactionCountByHash returns the number of transactions in the block with the given hash.
+func (pea *PublicEthAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint {
+	if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil {
+		n := hexutil.Uint(len(block.Transactions()))
+		return &n
+	}
+	if pea.rpc != nil {
+		var num *hexutil.Uint
+		if err := pea.rpc.CallContext(ctx, &num, "eth_getBlockTransactionCountByHash", blockHash); num != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
+			return num
+		}
+	}
+	return nil
+}
+
+// GetTransactionByBlockNumberAndIndex returns the transaction for the given block number and index.
+func (pea *PublicEthAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) *RPCTransaction {
+	if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil {
+		return newRPCTransactionFromBlockIndex(block, uint64(index))
+	}
+	if pea.rpc != nil {
+		var tx *RPCTransaction
+		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
+			return tx
+		}
+	}
+	return nil
+}
+
+// GetTransactionByBlockHashAndIndex returns the transaction for the given block hash and index.
+func (pea *PublicEthAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) *RPCTransaction {
+	if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil {
+		return newRPCTransactionFromBlockIndex(block, uint64(index))
+	}
+	if pea.rpc != nil {
+		var tx *RPCTransaction
+		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
+			return tx
+		}
+	}
+	return nil
+}
+
+// GetRawTransactionByBlockNumberAndIndex returns the bytes of the transaction for the given block number and index.
+func (pea *PublicEthAPI) GetRawTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) hexutil.Bytes {
+	if block, _ := pea.B.BlockByNumber(ctx, blockNr); block != nil {
+		return newRPCRawTransactionFromBlockIndex(block, uint64(index))
+	}
+	if pea.rpc != nil {
+		var tx hexutil.Bytes
+		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockNumberAndIndex", blockNr, index); tx != nil && err == nil {
+			go pea.writeStateDiffAt(blockNr.Int64())
+			return tx
+		}
+	}
+	return nil
+}
+
+// GetRawTransactionByBlockHashAndIndex returns the bytes of the transaction for the given block hash and index.
+func (pea *PublicEthAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) hexutil.Bytes {
+	if block, _ := pea.B.BlockByHash(ctx, blockHash); block != nil {
+		return newRPCRawTransactionFromBlockIndex(block, uint64(index))
+	}
+	if pea.rpc != nil {
+		var tx hexutil.Bytes
+		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByBlockHashAndIndex", blockHash, index); tx != nil && err == nil {
+			go pea.writeStateDiffFor(blockHash)
+			return tx
+		}
+	}
+	return nil
+}
+
+// GetTransactionByHash returns the transaction for the given hash
+// eth ipld-eth-server cannot currently handle pending/tx_pool txs
+func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) {
+	tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash)
+	if tx != nil && err == nil {
+		return NewRPCTransaction(tx, blockHash, blockNumber, index), nil
+	}
+	if pea.rpc != nil {
+		var tx *RPCTransaction
+		if err := pea.rpc.CallContext(ctx, &tx, "eth_getTransactionByHash", hash); tx != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
+			return tx, nil
+		}
+	}
+	return nil, err
+}
+
+// GetRawTransactionByHash returns the bytes of the transaction for the given hash.
+func (pea *PublicEthAPI) GetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) {
+	// Retrieve a finalized transaction, or a pooled otherwise
+	tx, _, _, _, err := pea.B.GetTransaction(ctx, hash)
+	if tx != nil && err == nil {
+		return rlp.EncodeToBytes(tx)
+	}
+	if pea.rpc != nil {
+		var tx hexutil.Bytes
+		if err := pea.rpc.CallContext(ctx, &tx, "eth_getRawTransactionByHash", hash); tx != nil && err == nil {
+			go pea.writeStateDiffFor(hash)
+			return tx, nil
+		}
+	}
+	return nil, err
+}
+
+/*
+
+Receipts and Logs
+
+*/
+
+// GetTransactionReceipt returns the transaction receipt for the given transaction hash.
+func (pea *PublicEthAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) {
+	receipt, err := pea.localGetTransactionReceipt(ctx, hash)
+	if receipt != nil && err == nil {
+		return receipt, nil
+	}
+	if pea.rpc != nil {
+		if receipt := pea.remoteGetTransactionReceipt(ctx, hash); receipt != nil {
+			go pea.writeStateDiffFor(hash)
+			return receipt, nil
+		}
+	}
+	return nil, err
+}
+
+func (pea *PublicEthAPI) localGetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) {
+	// TODO: this can be optimized for Postgres
+	tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash)
+	if err != nil {
+		return nil, err
+	}
+	if tx == nil {
+		return nil, nil
+	}
+	receipts, err := pea.B.GetReceipts(ctx, blockHash)
+	if err != nil {
+		return nil, err
+	}
+	if len(receipts) <= int(index) {
+		return nil, nil
+	}
+	receipt := receipts[index]
+
+	var signer types.Signer = types.FrontierSigner{}
+	if tx.Protected() {
+		signer = types.NewEIP155Signer(tx.ChainId())
+	}
+	from, _ := types.Sender(signer, tx)
+
+	fields := map[string]interface{}{
+		"blockHash":         blockHash,
+		"blockNumber":       hexutil.Uint64(blockNumber),
+		"transactionHash":   hash,
+		"transactionIndex":  hexutil.Uint64(index),
+		"from":              from,
+		"to":                tx.To(),
+		"gasUsed":           hexutil.Uint64(receipt.GasUsed),
+		"cumulativeGasUsed": hexutil.Uint64(receipt.CumulativeGasUsed),
+		"contractAddress":   nil,
+		"logs":              receipt.Logs,
+		"logsBloom":         receipt.Bloom,
+	}
+
+	// Assign receipt status or post state.
+	if len(receipt.PostState) > 0 {
+		fields["root"] = hexutil.Bytes(receipt.PostState)
+	} else {
+		fields["status"] = hexutil.Uint(receipt.Status)
+	}
+	if receipt.Logs == nil {
+		fields["logs"] = []*types.Log{}
+	}
+	// If the ContractAddress is 20 0x0 bytes, assume it is not a contract creation
+	if receipt.ContractAddress != (common.Address{}) {
+		fields["contractAddress"] = receipt.ContractAddress
+	}
+	return fields, nil
+}
+
+func (pea *PublicEthAPI) remoteGetTransactionReceipt(ctx context.Context, hash common.Hash) map[string]interface{} {
+	var rct *RPCReceipt
+	if err := pea.rpc.CallContext(ctx, &rct, "eth_getTransactionReceipt", hash); rct != nil && err == nil {
+		return map[string]interface{}{
+			"blockHash":         rct.BlockHash,
+			"blockNumber":       rct.BlockNumber,
+			"transactionHash":   rct.TransactionHash,
+			"transactionIndex":  rct.TransactionIndex,
+			"from":              rct.From,
+			"to":                rct.To,
+			"gasUsed":           rct.GasUsed,
+			"cumulativeGasUsed": rct.CumulativeGsUsed,
+			"contractAddress":   rct.ContractAddress,
+			"logs":              rct.Logs,
+			"logsBloom":         rct.Bloom,
+			"root":              rct.Root,
+			"status":            rct.Status,
+		}
+	}
+	return nil
+}
+
 // GetLogs returns logs matching the given argument that are stored within the state.
 //
 // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_getlogs
 func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery) ([]*types.Log, error) {
+	logs, err := pea.localGetLogs(ctx, crit)
+	if err != nil && pea.rpc != nil {
+		if arg, err := toFilterArg(crit); err == nil {
+			var res []*types.Log
+			if err := pea.rpc.CallContext(ctx, &res, "eth_getLogs", arg); err == nil {
+				go pea.writeStateDiffWithCriteria(crit)
+				return res, nil
+			}
+		}
+	}
+	return logs, err
+}
+
+func (pea *PublicEthAPI) localGetLogs(ctx context.Context, crit ethereum.FilterQuery) ([]*types.Log, error) {
+	// TODO: this can be optimized away from using the old cid retriever and ipld fetcher interfaces
 	// Convert FilterQuery into ReceiptFilter
 	addrStrs := make([]string, len(crit.Addresses))
 	for i, addr := range crit.Addresses {
@@ -113,11 +577,7 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery)
 	startingBlock := crit.FromBlock
 	endingBlock := crit.ToBlock
 	if startingBlock == nil {
-		startingBlockInt, err := pea.B.Retriever.RetrieveFirstBlockNumber()
-		if err != nil {
-			return nil, err
-		}
-		startingBlock = big.NewInt(startingBlockInt)
+		startingBlock = common.Big0
 	}
 	if endingBlock == nil {
 		endingBlockInt, err := pea.B.Retriever.RetrieveLastBlockNumber()
@@ -147,51 +607,390 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery)
 	return logs, err // need to return err variable so that we return the err = tx.Commit() assignment in the defer
 }
 
-// GetHeaderByNumber returns the requested canonical block header.
-// * When blockNr is -1 the chain head is returned.
-// * We cannot support pending block calls since we do not have an active miner
-func (pea *PublicEthAPI) GetHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (map[string]interface{}, error) {
-	header, err := pea.B.HeaderByNumber(ctx, number)
-	if header != nil && err == nil {
-		return pea.rpcMarshalHeader(header)
+/*
+
+State and Storage
+
+*/
+
+// GetBalance returns the amount of wei for the given address in the state of the
+// given block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta
+// block numbers are also allowed.
+func (pea *PublicEthAPI) GetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) {
+	bal, err := pea.localGetBalance(ctx, address, blockNrOrHash)
+	if bal != nil && err == nil {
+		return bal, nil
+	}
+	if pea.rpc != nil {
+		var res *hexutil.Big
+		if err := pea.rpc.CallContext(ctx, &res, "eth_getBalance", address, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
+			return res, nil
+		}
 	}
 	return nil, err
 }
 
-// GetBlockByNumber returns the requested canonical block.
-// * When blockNr is -1 the chain head is returned.
-// * We cannot support pending block calls since we do not have an active miner
-// * When fullTx is true all transactions in the block are returned, otherwise
-//   only the transaction hash is returned.
-func (pea *PublicEthAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) {
-	block, err := pea.B.BlockByNumber(ctx, number)
-	if block != nil && err == nil {
-		return pea.rpcMarshalBlock(block, true, fullTx)
-	}
-	return nil, err
-}
-
-// GetBlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full
-// detail, otherwise only the transaction hash is returned.
-func (pea *PublicEthAPI) GetBlockByHash(ctx context.Context, hash common.Hash, fullTx bool) (map[string]interface{}, error) {
-	block, err := pea.B.BlockByHash(ctx, hash)
-	if block != nil {
-		return pea.rpcMarshalBlock(block, true, fullTx)
-	}
-	return nil, err
-}
-
-// GetTransactionByHash returns the transaction for the given hash
-// eth ipld-eth-server cannot currently handle pending/tx_pool txs
-func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) {
-	// Try to return an already finalized transaction
-	tx, blockHash, blockNumber, index, err := pea.B.GetTransaction(ctx, hash)
+func (pea *PublicEthAPI) localGetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) {
+	account, err := pea.B.GetAccountByNumberOrHash(ctx, address, blockNrOrHash)
 	if err != nil {
 		return nil, err
 	}
-	if tx != nil {
-		return NewRPCTransaction(tx, blockHash, blockNumber, index), nil
+	return (*hexutil.Big)(account.Balance), nil
+}
+
+// GetStorageAt returns the storage from the state at the given address, key and
+// block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block
+// numbers are also allowed.
+func (pea *PublicEthAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) {
+	storageVal, err := pea.B.GetStorageByNumberOrHash(ctx, address, common.HexToHash(key), blockNrOrHash)
+	if storageVal != nil && err == nil {
+		return storageVal, nil
 	}
-	// Transaction unknown, return as such
-	return nil, nil
+	if pea.rpc != nil {
+		var res hexutil.Bytes
+		if err := pea.rpc.CallContext(ctx, &res, "eth_getStorageAt", address, key, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
+			return res, nil
+		}
+	}
+	return nil, err
+}
+
+// GetCode returns the code stored at the given address in the state for the given block number.
+func (pea *PublicEthAPI) GetCode(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) {
+	code, err := pea.B.GetCodeByNumberOrHash(ctx, address, blockNrOrHash)
+	if code != nil && err == nil {
+		return code, nil
+	}
+	if pea.rpc != nil {
+		var res hexutil.Bytes
+		if err := pea.rpc.CallContext(ctx, &res, "eth_getCode", address, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
+			return res, nil
+		}
+	}
+	return nil, err
+}
+
+// GetProof returns the Merkle-proof for a given account and optionally some storage keys.
+func (pea *PublicEthAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) {
+	proof, err := pea.localGetProof(ctx, address, storageKeys, blockNrOrHash)
+	if proof != nil && err == nil {
+		return proof, nil
+	}
+	if pea.rpc != nil {
+		var res *AccountResult
+		if err := pea.rpc.CallContext(ctx, &res, "eth_getProof", address, storageKeys, blockNrOrHash); res != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
+			return res, nil
+		}
+	}
+	return nil, err
+}
+
+func (pea *PublicEthAPI) localGetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) {
+	state, _, err := pea.B.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash)
+	if state == nil || err != nil {
+		return nil, err
+	}
+
+	storageTrie := state.StorageTrie(address)
+	storageHash := types.EmptyRootHash
+	codeHash := state.GetCodeHash(address)
+	storageProof := make([]StorageResult, len(storageKeys))
+
+	// if we have a storageTrie, (which means the account exists), we can update the storagehash
+	if storageTrie != nil {
+		storageHash = storageTrie.Hash()
+	} else {
+		// no storageTrie means the account does not exist, so the codeHash is the hash of an empty bytearray.
+		codeHash = crypto.Keccak256Hash(nil)
+	}
+
+	// create the proof for the storageKeys
+	for i, key := range storageKeys {
+		if storageTrie != nil {
+			proof, storageError := state.GetStorageProof(address, common.HexToHash(key))
+			if storageError != nil {
+				return nil, storageError
+			}
+			storageProof[i] = StorageResult{key, (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), toHexSlice(proof)}
+		} else {
+			storageProof[i] = StorageResult{key, &hexutil.Big{}, []string{}}
+		}
+	}
+
+	// create the accountProof
+	accountProof, proofErr := state.GetProof(address)
+	if proofErr != nil {
+		return nil, proofErr
+	}
+
+	return &AccountResult{
+		Address:      address,
+		AccountProof: toHexSlice(accountProof),
+		Balance:      (*hexutil.Big)(state.GetBalance(address)),
+		CodeHash:     codeHash,
+		Nonce:        hexutil.Uint64(state.GetNonce(address)),
+		StorageHash:  storageHash,
+		StorageProof: storageProof,
+	}, state.Error()
+}
+
+// Call executes the given transaction on the state for the given block number.
+//
+// Additionally, the caller can specify a batch of contract for fields overriding.
+//
+// Note, this function doesn't make and changes in the state/blockchain and is
+// useful to execute and retrieve values.
+func (pea *PublicEthAPI) Call(ctx context.Context, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *map[common.Address]account) (hexutil.Bytes, error) {
+	var accounts map[common.Address]account
+	if overrides != nil {
+		accounts = *overrides
+	}
+	res, _, failed, err := DoCall(ctx, pea.B, args, blockNrOrHash, accounts, 5*time.Second, pea.B.Config.RPCGasCap)
+	if (failed || err != nil) && pea.rpc != nil {
+		var hex hexutil.Bytes
+		if err := pea.rpc.CallContext(ctx, &hex, "eth_call", args, blockNrOrHash, overrides); hex != nil && err == nil {
+			go pea.writeStateDiffAtOrFor(blockNrOrHash)
+			return hex, nil
+		}
+	}
+	if failed && err == nil {
+		return nil, errors.New("eth_call failed without error")
+	}
+	return (hexutil.Bytes)(res), err
+}
+
+func DoCall(ctx context.Context, b *Backend, args CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides map[common.Address]account, timeout time.Duration, globalGasCap *big.Int) ([]byte, uint64, bool, error) {
+	defer func(start time.Time) {
+		logrus.Debugf("Executing EVM call finished %s runtime %s", time.Now().String(), time.Since(start).String())
+	}(time.Now())
+	state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash)
+	if state == nil || err != nil {
+		return nil, 0, false, err
+	}
+	// Set sender address or use a default if none specified
+	var addr common.Address
+	if args.From == nil {
+		if b.Config.DefaultSender != nil {
+			addr = *b.Config.DefaultSender
+		}
+	} else {
+		addr = *args.From
+	}
+	// Override the fields of specified contracts before execution.
+	for addr, account := range overrides {
+		// Override account nonce.
+		if account.Nonce != nil {
+			state.SetNonce(addr, uint64(*account.Nonce))
+		}
+		// Override account(contract) code.
+		if account.Code != nil {
+			state.SetCode(addr, *account.Code)
+		}
+		// Override account balance.
+		if account.Balance != nil {
+			state.SetBalance(addr, (*big.Int)(*account.Balance))
+		}
+		if account.State != nil && account.StateDiff != nil {
+			return nil, 0, false, fmt.Errorf("account %s has both 'state' and 'stateDiff'", addr.Hex())
+		}
+		// Replace entire state if caller requires.
+		if account.State != nil {
+			state.SetStorage(addr, *account.State)
+		}
+		// Apply state diff into specified accounts.
+		if account.StateDiff != nil {
+			for key, value := range *account.StateDiff {
+				state.SetState(addr, key, value)
+			}
+		}
+	}
+	// Set default gas & gas price if none were set
+	gas := uint64(math.MaxUint64 / 2)
+	if args.Gas != nil {
+		gas = uint64(*args.Gas)
+	}
+	if globalGasCap != nil && globalGasCap.Uint64() < gas {
+		logrus.Warnf("Caller gas above allowance, capping; requested: %d, cap: %d", gas, globalGasCap)
+		gas = globalGasCap.Uint64()
+	}
+	gasPrice := new(big.Int).SetUint64(params.GWei)
+	if args.GasPrice != nil {
+		gasPrice = args.GasPrice.ToInt()
+	}
+
+	value := new(big.Int)
+	if args.Value != nil {
+		value = args.Value.ToInt()
+	}
+
+	var data []byte
+	if args.Data != nil {
+		data = []byte(*args.Data)
+	}
+
+	// Create new call message
+	msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, data, false)
+
+	// Setup context so it may be cancelled the call has completed
+	// or, in case of unmetered gas, setup a context with a timeout.
+	var cancel context.CancelFunc
+	if timeout > 0 {
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+	} else {
+		ctx, cancel = context.WithCancel(ctx)
+	}
+	// Make sure the context is cancelled when the call has completed
+	// this makes sure resources are cleaned up.
+	defer cancel()
+
+	// Get a new instance of the EVM.
+	evm, err := b.GetEVM(ctx, msg, state, header)
+	if err != nil {
+		return nil, 0, false, err
+	}
+	// Wait for the context to be done and cancel the evm. Even if the
+	// EVM has finished, cancelling may be done (repeatedly)
+	go func() {
+		<-ctx.Done()
+		evm.Cancel()
+	}()
+
+	// Setup the gas pool (also for unmetered requests)
+	// and apply the message.
+	gp := new(core.GasPool).AddGas(math.MaxUint64)
+	result, err := core.ApplyMessage(evm, msg, gp)
+	if err != nil {
+		return nil, 0, false, fmt.Errorf("execution failed: %v", err)
+	}
+	// If the timer caused an abort, return an appropriate error message
+	if evm.Cancelled() {
+		return nil, 0, false, fmt.Errorf("execution aborted (timeout = %v)", timeout)
+	}
+	return result.Return(), result.UsedGas, result.Failed(), err
+}
+
+// writeStateDiffAtOrFor calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffAtOrFor(blockNrOrHash rpc.BlockNumberOrHash) {
+	// short circuit right away if the proxy doesn't support diffing
+	if !pea.supportsStateDiff {
+		return
+	}
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		pea.writeStateDiffAt(blockNr.Int64())
+		return
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		pea.writeStateDiffFor(hash)
+	}
+}
+
+// writeStateDiffWithCriteria calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffWithCriteria(crit ethereum.FilterQuery) {
+	// short circuit right away if the proxy doesn't support diffing
+	if !pea.supportsStateDiff {
+		return
+	}
+	if crit.BlockHash != nil {
+		pea.writeStateDiffFor(*crit.BlockHash)
+		return
+	}
+	var start, end int64
+	if crit.FromBlock != nil {
+		start = crit.FromBlock.Int64()
+	}
+	if crit.ToBlock != nil {
+		end = crit.ToBlock.Int64()
+	} else {
+		end = start
+	}
+	for i := start; i <= end; i++ {
+		pea.writeStateDiffAt(i)
+	}
+}
+
+// writeStateDiffAt calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffAt(height int64) {
+	if !pea.supportsStateDiff {
+		return
+	}
+	// we use a separate context than the one provided by the client
+	ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second)
+	defer cancel()
+	var data json.RawMessage
+	params := statediff.Params{
+		IntermediateStateNodes:   true,
+		IntermediateStorageNodes: true,
+		IncludeBlock:             true,
+		IncludeReceipts:          true,
+		IncludeTD:                true,
+		IncludeCode:              true,
+	}
+	if err := pea.rpc.CallContext(ctx, &data, "statediff_writeStateDiffAt", uint64(height), params); err != nil {
+		logrus.Errorf("writeStateDiffAt %d faild with err %s", height, err.Error())
+	}
+}
+
+// writeStateDiffFor calls out to the proxy statediffing geth client to fill in a gap in the index
+func (pea *PublicEthAPI) writeStateDiffFor(blockHash common.Hash) {
+	if !pea.supportsStateDiff {
+		return
+	}
+	// we use a separate context than the one provided by the client
+	ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second)
+	defer cancel()
+	var data json.RawMessage
+	params := statediff.Params{
+		IntermediateStateNodes:   true,
+		IntermediateStorageNodes: true,
+		IncludeBlock:             true,
+		IncludeReceipts:          true,
+		IncludeTD:                true,
+		IncludeCode:              true,
+	}
+	if err := pea.rpc.CallContext(ctx, &data, "statediff_writeStateDiffFor", blockHash, params); err != nil {
+		logrus.Errorf("writeStateDiffFor %s faild with err %s", blockHash.Hex(), err.Error())
+	}
+}
+
+// rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field
+func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
+	fields, err := RPCMarshalBlock(b, inclTx, fullTx)
+	if err != nil {
+		return nil, err
+	}
+	if inclTx {
+		td, err := pea.B.GetTd(b.Hash())
+		if err != nil {
+			return nil, err
+		}
+		fields["totalDifficulty"] = (*hexutil.Big)(td)
+	}
+	return fields, err
+}
+
+// rpcMarshalBlockWithUncleHashes uses the generalized output filler, then adds the total difficulty field
+func (pea *PublicEthAPI) rpcMarshalBlockWithUncleHashes(b *types.Block, uncleHashes []common.Hash, inclTx bool, fullTx bool) (map[string]interface{}, error) {
+	fields, err := RPCMarshalBlockWithUncleHashes(b, uncleHashes, inclTx, fullTx)
+	if err != nil {
+		return nil, err
+	}
+	td, err := pea.B.GetTd(b.Hash())
+	if err != nil {
+		return nil, err
+	}
+	fields["totalDifficulty"] = (*hexutil.Big)(td)
+	return fields, err
+}
+
+// toHexSlice creates a slice of hex-strings based on []byte.
+func toHexSlice(b [][]byte) []string {
+	r := make([]string, len(b))
+	for i := range b {
+		r[i] = hexutil.Encode(b[i])
+	}
+	return r
 }

pkg/eth/backend.go

@@ -24,38 +24,103 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/consensus"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/bloombits"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	ipfsethdb "github.com/vulcanize/pg-ipfs-ethdb"
+	"github.com/ethereum/go-ethereum/trie"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	"github.com/vulcanize/ipfs-ethdb"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
+	shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared"
+
 	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
 var (
-	errPendingBlockNumber = errors.New("pending block number not supported")
+	errPendingBlockNumber  = errors.New("pending block number not supported")
+	errNegativeBlockNumber = errors.New("negative block number not supported")
+)
+
+const (
+	RetrieveCanonicalBlockHashByNumber = `SELECT block_hash FROM eth.header_cids
+									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
+									WHERE id = (SELECT canonical_header_id($1))`
+	RetrieveCanonicalHeaderByNumber = `SELECT cid, data FROM eth.header_cids
+									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
+									WHERE id = (SELECT canonical_header_id($1))`
+	RetrieveTD = `SELECT td FROM eth.header_cids
+			WHERE header_cids.block_hash = $1`
+	RetrieveRPCTransaction = `SELECT blocks.data, block_hash, block_number, index FROM public.blocks, eth.transaction_cids, eth.header_cids
+			WHERE blocks.key = transaction_cids.mh_key
+			AND transaction_cids.header_id = header_cids.id
+			AND transaction_cids.tx_hash = $1`
+	RetrieveCodeHashByLeafKeyAndBlockHash = `SELECT code_hash FROM eth.state_accounts, eth.state_cids, eth.header_cids
+											WHERE state_accounts.state_id = state_cids.id
+											AND state_cids.header_id = header_cids.id
+											AND state_leaf_key = $1
+											AND block_number <= (SELECT block_number
+																FROM eth.header_cids
+																WHERE block_hash = $2)
+											AND header_cids.id = (SELECT canonical_header_id(block_number))
+											ORDER BY block_number DESC
+											LIMIT 1`
+	RetrieveCodeByMhKey = `SELECT data FROM public.blocks WHERE key = $1`
 )
 
 type Backend struct {
-	Retriever *CIDRetriever
-	Fetcher   *IPLDFetcher
-	DB        *postgres.DB
-	EthDB     ethdb.Database
+	// underlying postgres db
+	DB *postgres.DB
+
+	// postgres db interfaces
+	Retriever     *CIDRetriever
+	Fetcher       *IPLDFetcher
+	IPLDRetriever *IPLDRetriever
+
+	// ethereum interfaces
+	EthDB         ethdb.Database
+	StateDatabase state.Database
+
+	Config *Config
 }
 
-func NewEthBackend(db *postgres.DB) (*Backend, error) {
+type Config struct {
+	ChainConfig   *params.ChainConfig
+	VmConfig      vm.Config
+	DefaultSender *common.Address
+	RPCGasCap     *big.Int
+}
+
+func NewEthBackend(db *postgres.DB, c *Config) (*Backend, error) {
 	r := NewCIDRetriever(db)
+	ethDB := ipfsethdb.NewDatabase(db.DB)
 	return &Backend{
-		Retriever: r,
-		Fetcher:   NewIPLDFetcher(db),
-		DB:        db,
-		EthDB:     ipfsethdb.NewDatabase(db.DB),
+		DB:            db,
+		Retriever:     r,
+		Fetcher:       NewIPLDFetcher(db),
+		IPLDRetriever: NewIPLDRetriever(db),
+		EthDB:         ethDB,
+		StateDatabase: state.NewDatabase(ethDB),
+		Config:        c,
 	}, nil
 }
 
+// ChainDb returns the backend's underlying chain database
+func (b *Backend) ChainDb() ethdb.Database {
+	return b.EthDB
+}
+
+// HeaderByNumber gets the canonical header for the provided block number
 func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumber) (*types.Header, error) {
 	var err error
 	number := blockNumber.Int64()
@@ -65,28 +130,62 @@ func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumbe
 			return nil, err
 		}
 	}
+	if blockNumber == rpc.EarliestBlockNumber {
+		number, err = b.Retriever.RetrieveFirstBlockNumber()
+		if err != nil {
+			return nil, err
+		}
+	}
 	if blockNumber == rpc.PendingBlockNumber {
 		return nil, errPendingBlockNumber
 	}
-
-	var headerBytes []byte
-	pgStr := `SELECT DISTINCT ON (block_number) data
-			FROM public.blocks INNER JOIN eth.header_cids ON (header_cids.mh_key = blocks.key)
-			WHERE block_number = $1
-			ORDER BY block_number, times_validated DESC`
-	if err := b.DB.Get(&headerBytes, pgStr, number); err != nil {
-		return nil, fmt.Errorf("header at block %d is not available; err %s", number, err.Error())
+	if number < 0 {
+		return nil, errNegativeBlockNumber
 	}
+	_, canonicalHeaderRLP, err := b.GetCanonicalHeader(uint64(number))
+	if err != nil {
+		return nil, err
+	}
+
 	header := new(types.Header)
-	return header, rlp.DecodeBytes(headerBytes, header)
+	return header, rlp.DecodeBytes(canonicalHeaderRLP, header)
 }
 
-// GetTd retrieves and returns the total difficulty at the given block hash
+// HeaderByHash gets the header for the provided block hash
+func (b *Backend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) {
+	_, headerRLP, err := b.IPLDRetriever.RetrieveHeaderByHash(hash)
+	if err != nil {
+		return nil, err
+	}
+	header := new(types.Header)
+	return header, rlp.DecodeBytes(headerRLP, header)
+}
+
+// HeaderByNumberOrHash gets the header for the provided block hash or number
+func (b *Backend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.HeaderByNumber(ctx, blockNr)
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		header, err := b.HeaderByHash(ctx, hash)
+		if err != nil {
+			return nil, err
+		}
+		if header == nil {
+			return nil, errors.New("header for hash not found")
+		}
+		if blockNrOrHash.RequireCanonical && b.GetCanonicalHash(header.Number.Uint64()) != hash {
+			return nil, errors.New("hash is not currently canonical")
+		}
+		return header, nil
+	}
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
+}
+
+// GetTd gets the total difficulty at the given block hash
 func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) {
-	pgStr := `SELECT td FROM eth.header_cids
-			WHERE header_cids.block_hash = $1`
 	var tdStr string
-	err := b.DB.Get(&tdStr, pgStr, blockHash.String())
+	err := b.DB.Get(&tdStr, RetrieveTD, blockHash.String())
 	if err != nil {
 		return nil, err
 	}
@@ -97,31 +196,41 @@ func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) {
 	return td, nil
 }
 
-// GetLogs returns all the logs for the given block hash
-func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
-	pgStr := `SELECT blocks.data FROM public.blocks, eth.receipt_cids, eth.transaction_cids, eth.header_cids
-			WHERE header_cids.block_hash = $1
-			AND receipt_cids.tx_id = transaction_cids.id
-			AND transaction_cids.header_id = header_cids.id
-			ORDER BY transaction_cids.index ASC`
-	var receiptBytes [][]byte
-	if err := b.DB.Select(&receiptBytes, pgStr, hash.Hex()); err != nil {
-		return nil, err
+// CurrentBlock returns the current block
+func (b *Backend) CurrentBlock() *types.Block {
+	block, _ := b.BlockByNumber(context.Background(), rpc.LatestBlockNumber)
+	return block
+}
+
+// BlockByNumberOrHash returns block by number or hash
+func (b *Backend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.BlockByNumber(ctx, blockNr)
 	}
-	logs := make([][]*types.Log, len(receiptBytes))
-	for i, rctBytes := range receiptBytes {
-		var rct types.Receipt
-		if err := rlp.DecodeBytes(rctBytes, &rct); err != nil {
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		header, err := b.HeaderByHash(ctx, hash)
+		if err != nil {
 			return nil, err
 		}
-		logs[i] = rct.Logs
+		if header == nil {
+			return nil, errors.New("header for hash not found")
+		}
+		if blockNrOrHash.RequireCanonical && b.GetCanonicalHash(header.Number.Uint64()) != hash {
+			return nil, errors.New("hash is not currently canonical")
+		}
+		block, err := b.BlockByHash(ctx, hash)
+		if err != nil {
+			return nil, err
+		}
+		if block == nil {
+			return nil, errors.New("header found, but block body is missing")
+		}
+		return block, nil
 	}
-	return logs, nil
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
 }
 
 // BlockByNumber returns the requested canonical block.
-// Since the ipld-eth-server database can contain forked blocks, it is recommended to fetch BlockByHash as
-// fetching by number can return non-deterministic results (returns the first block found at that height)
 func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber) (*types.Block, error) {
 	var err error
 	number := blockNumber.Int64()
@@ -131,11 +240,26 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 			return nil, err
 		}
 	}
+	if blockNumber == rpc.EarliestBlockNumber {
+		number, err = b.Retriever.RetrieveFirstBlockNumber()
+		if err != nil {
+			return nil, err
+		}
+	}
 	if blockNumber == rpc.PendingBlockNumber {
 		return nil, errPendingBlockNumber
 	}
+	if number < 0 {
+		return nil, errNegativeBlockNumber
+	}
+	// Get the canonical hash
+	canonicalHash := b.GetCanonicalHash(uint64(number))
+	if err != nil {
+		return nil, err
+	}
 	// Retrieve all the CIDs for the block
-	headerCID, uncleCIDs, txCIDs, rctCIDs, err := b.Retriever.RetrieveBlockByNumber(number)
+	// TODO: optimize this by retrieving iplds directly rather than the cids first (this is remanent from when we fetched iplds through ipfs blockservice interface)
+	headerCID, uncleCIDs, txCIDs, rctCIDs, err := b.Retriever.RetrieveBlockByHash(canonicalHash)
 	if err != nil {
 		return nil, err
 	}
@@ -205,7 +329,7 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
 		receipts = append(receipts, &receipt)
 	}
 	// Compose everything together into a complete block
-	return types.NewBlock(&header, transactions, uncles, receipts), err
+	return types.NewBlock(&header, transactions, uncles, receipts, new(trie.Trie)), err
 }
 
 // BlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full
@@ -282,7 +406,7 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
 		receipts = append(receipts, &receipt)
 	}
 	// Compose everything together into a complete block
-	return types.NewBlock(&header, transactions, uncles, receipts), err
+	return types.NewBlock(&header, transactions, uncles, receipts, new(trie.Trie)), err
 }
 
 // GetTransaction retrieves a tx by hash
@@ -294,11 +418,7 @@ func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*type
 		BlockNumber uint64 `db:"block_number"`
 		Index       uint64 `db:"index"`
 	}
-	pgStr := `SELECT blocks.data, block_hash, block_number, index FROM public.blocks, eth.transaction_cids, eth.header_cids
-			WHERE blocks.key = transaction_cids.mh_key
-			AND transaction_cids.header_id = header_cids.id
-			AND transaction_cids.tx_hash = $1`
-	if err := b.DB.Get(&tempTxStruct, pgStr, txHash.String()); err != nil {
+	if err := b.DB.Get(&tempTxStruct, RetrieveRPCTransaction, txHash.String()); err != nil {
 		return nil, common.Hash{}, 0, 0, err
 	}
 	var transaction types.Transaction
@@ -308,205 +428,259 @@ func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*type
 	return &transaction, common.HexToHash(tempTxStruct.BlockHash), tempTxStruct.BlockNumber, tempTxStruct.Index, nil
 }
 
-// extractLogsOfInterest returns logs from the receipt IPLD
-func extractLogsOfInterest(rctIPLDs []ipfs.BlockModel, wantedTopics [][]string) ([]*types.Log, error) {
-	var logs []*types.Log
-	for _, rctIPLD := range rctIPLDs {
-		rctRLP := rctIPLD
-		var rct types.Receipt
-		if err := rlp.DecodeBytes(rctRLP.Data, &rct); err != nil {
+// GetReceipts retrieves receipts for provided block hash
+func (b *Backend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {
+	_, receiptBytes, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(hash)
+	if err != nil {
+		return nil, err
+	}
+	rcts := make(types.Receipts, len(receiptBytes))
+	for i, rctBytes := range receiptBytes {
+		rct := new(types.Receipt)
+		if err := rlp.DecodeBytes(rctBytes, rct); err != nil {
 			return nil, err
 		}
-		for _, log := range rct.Logs {
-			if wanted := wantedLog(wantedTopics, log.Topics); wanted == true {
-				logs = append(logs, log)
-			}
+		rcts[i] = rct
+	}
+	return rcts, nil
+}
+
+// GetLogs returns all the logs for the given block hash
+func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
+	_, receiptBytes, err := b.IPLDRetriever.RetrieveReceiptsByBlockHash(hash)
+	if err != nil {
+		return nil, err
+	}
+	logs := make([][]*types.Log, len(receiptBytes))
+	for i, rctBytes := range receiptBytes {
+		var rct types.Receipt
+		if err := rlp.DecodeBytes(rctBytes, &rct); err != nil {
+			return nil, err
 		}
+		logs[i] = rct.Logs
 	}
 	return logs, nil
 }
 
-// returns true if the log matches on the filter
-func wantedLog(wantedTopics [][]string, actualTopics []common.Hash) bool {
-	// actualTopics will always have length <= 4
-	// wantedTopics will always have length 4
-	matches := 0
-	for i, actualTopic := range actualTopics {
-		// If we have topics in this filter slot, count as a match if the actualTopic matches one of the ones in this filter slot
-		if len(wantedTopics[i]) > 0 {
-			matches += sliceContainsHash(wantedTopics[i], actualTopic)
+// StateAndHeaderByNumberOrHash returns the statedb and header for the provided block number or hash
+func (b *Backend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.StateAndHeaderByNumber(ctx, blockNr)
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		header, err := b.HeaderByHash(ctx, hash)
+		if err != nil {
+			return nil, nil, err
+		}
+		if header == nil {
+			return nil, nil, errors.New("header for hash not found")
+		}
+		if blockNrOrHash.RequireCanonical && b.GetCanonicalHash(header.Number.Uint64()) != hash {
+			return nil, nil, errors.New("hash is not currently canonical")
+		}
+		stateDb, err := state.New(header.Root, b.StateDatabase, nil)
+		return stateDb, header, err
+	}
+	return nil, nil, errors.New("invalid arguments; neither block nor hash specified")
+}
+
+// StateAndHeaderByNumber returns the statedb and header for a provided block number
+func (b *Backend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) {
+	// Pending state is only known by the miner
+	if number == rpc.PendingBlockNumber {
+		return nil, nil, errPendingBlockNumber
+	}
+	// Otherwise resolve the block number and return its state
+	header, err := b.HeaderByNumber(ctx, number)
+	if err != nil {
+		return nil, nil, err
+	}
+	if header == nil {
+		return nil, nil, errors.New("header not found")
+	}
+	stateDb, err := state.New(header.Root, b.StateDatabase, nil)
+	return stateDb, header, err
+}
+
+// GetCanonicalHash gets the canonical hash for the provided number, if there is one
+func (b *Backend) GetCanonicalHash(number uint64) common.Hash {
+	var hashResult string
+	if err := b.DB.Get(&hashResult, RetrieveCanonicalBlockHashByNumber, number); err != nil {
+		return common.Hash{}
+	}
+	return common.HexToHash(hashResult)
+}
+
+type rowResult struct {
+	CID  string
+	Data []byte
+}
+
+// GetCanonicalHeader gets the canonical header for the provided number, if there is one
+func (b *Backend) GetCanonicalHeader(number uint64) (string, []byte, error) {
+	headerResult := new(rowResult)
+	return headerResult.CID, headerResult.Data, b.DB.QueryRowx(RetrieveCanonicalHeaderByNumber, number).StructScan(headerResult)
+}
+
+// GetEVM constructs and returns a vm.EVM
+func (b *Backend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, error) {
+	state.SetBalance(msg.From(), math.MaxBig256)
+	vmctx := core.NewEVMBlockContext(header, b, nil)
+	txContext := core.NewEVMTxContext(msg)
+	return vm.NewEVM(vmctx, txContext, state, b.Config.ChainConfig, b.Config.VmConfig), nil
+}
+
+// GetAccountByNumberOrHash returns the account object for the provided address at the block corresponding to the provided number or hash
+func (b *Backend) GetAccountByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*state.Account, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.GetAccountByNumber(ctx, address, uint64(blockNr.Int64()))
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		return b.GetAccountByHash(ctx, address, hash)
+	}
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
+}
+
+// GetAccountByNumber returns the account object for the provided address at the canonical block at the provided height
+func (b *Backend) GetAccountByNumber(ctx context.Context, address common.Address, number uint64) (*state.Account, error) {
+	hash := b.GetCanonicalHash(number)
+	if hash == (common.Hash{}) {
+		return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number)
+	}
+	return b.GetAccountByHash(ctx, address, hash)
+}
+
+// GetAccountByHash returns the account object for the provided address at the block with the provided hash
+func (b *Backend) GetAccountByHash(ctx context.Context, address common.Address, hash common.Hash) (*state.Account, error) {
+	_, accountRlp, err := b.IPLDRetriever.RetrieveAccountByAddressAndBlockHash(address, hash)
+	if err != nil {
+		return nil, err
+	}
+	acct := new(state.Account)
+	return acct, rlp.DecodeBytes(accountRlp, acct)
+}
+
+// GetCodeByNumberOrHash returns the byte code for the contract deployed at the provided address at the block with the provided hash or block number
+func (b *Backend) GetCodeByNumberOrHash(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) ([]byte, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.GetCodeByNumber(ctx, address, uint64(blockNr.Int64()))
+	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		return b.GetCodeByHash(ctx, address, hash)
+	}
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
+}
+
+// GetCodeByNumber returns the byte code for the contract deployed at the provided address at the canonical block with the provided block number
+func (b *Backend) GetCodeByNumber(ctx context.Context, address common.Address, number uint64) ([]byte, error) {
+	hash := b.GetCanonicalHash(number)
+	if hash == (common.Hash{}) {
+		return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number)
+	}
+	return b.GetCodeByHash(ctx, address, hash)
+}
+
+// GetCodeByHash returns the byte code for the contract deployed at the provided address at the block with the provided hash
+func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, hash common.Hash) ([]byte, error) {
+	codeHash := make([]byte, 0)
+	leafKey := crypto.Keccak256Hash(address.Bytes())
+	// Begin tx
+	tx, err := b.DB.Beginx()
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if p := recover(); p != nil {
+			shared.Rollback(tx)
+			panic(p)
+		} else if err != nil {
+			shared.Rollback(tx)
 		} else {
-			// Filter slot is empty, not matching any topics at this slot => counts as a match
-			matches++
+			err = tx.Commit()
 		}
+	}()
+	if err := tx.Get(&codeHash, RetrieveCodeHashByLeafKeyAndBlockHash, leafKey.Hex(), hash.Hex()); err != nil {
+		return nil, err
 	}
-	if matches == len(actualTopics) {
-		return true
-	}
-	return false
-}
-
-// returns 1 if the slice contains the hash, 0 if it does not
-func sliceContainsHash(slice []string, hash common.Hash) int {
-	for _, str := range slice {
-		if str == hash.String() {
-			return 1
-		}
-	}
-	return 0
-}
-
-// rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires
-// a `PublicEthAPI`.
-func (pea *PublicEthAPI) rpcMarshalHeader(header *types.Header) (map[string]interface{}, error) {
-	fields := RPCMarshalHeader(header)
-	td, err := pea.B.GetTd(header.Hash())
+	mhKey, err := shared2.MultihashKeyFromKeccak256(common.BytesToHash(codeHash))
 	if err != nil {
 		return nil, err
 	}
-	fields["totalDifficulty"] = (*hexutil.Big)(td)
-	return fields, nil
+	code := make([]byte, 0)
+	err = tx.Get(&code, RetrieveCodeByMhKey, mhKey)
+	return code, err
 }
 
-// RPCMarshalHeader converts the given header to the RPC output.
-// This function is eth/internal so we have to make our own version here...
-func RPCMarshalHeader(head *types.Header) map[string]interface{} {
-	return map[string]interface{}{
-		"number":           (*hexutil.Big)(head.Number),
-		"hash":             head.Hash(),
-		"parentHash":       head.ParentHash,
-		"nonce":            head.Nonce,
-		"mixHash":          head.MixDigest,
-		"sha3Uncles":       head.UncleHash,
-		"logsBloom":        head.Bloom,
-		"stateRoot":        head.Root,
-		"miner":            head.Coinbase,
-		"difficulty":       (*hexutil.Big)(head.Difficulty),
-		"extraData":        hexutil.Bytes(head.Extra),
-		"size":             hexutil.Uint64(head.Size()),
-		"gasLimit":         hexutil.Uint64(head.GasLimit),
-		"gasUsed":          hexutil.Uint64(head.GasUsed),
-		"timestamp":        hexutil.Uint64(head.Time),
-		"transactionsRoot": head.TxHash,
-		"receiptsRoot":     head.ReceiptHash,
+// GetStorageByNumberOrHash returns the storage value for the provided contract address an storage key at the block corresponding to the provided number or hash
+func (b *Backend) GetStorageByNumberOrHash(ctx context.Context, address common.Address, storageLeafKey common.Hash, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) {
+	if blockNr, ok := blockNrOrHash.Number(); ok {
+		return b.GetStorageByNumber(ctx, address, storageLeafKey, uint64(blockNr.Int64()))
 	}
+	if hash, ok := blockNrOrHash.Hash(); ok {
+		return b.GetStorageByHash(ctx, address, storageLeafKey, hash)
+	}
+	return nil, errors.New("invalid arguments; neither block nor hash specified")
 }
 
-// rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires
-// a `PublicBlockchainAPI`.
-func (pea *PublicEthAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
-	fields, err := RPCMarshalBlock(b, inclTx, fullTx)
+// GetStorageByNumber returns the storage value for the provided contract address an storage key at the block corresponding to the provided number
+func (b *Backend) GetStorageByNumber(ctx context.Context, address common.Address, storageLeafKey common.Hash, number uint64) (hexutil.Bytes, error) {
+	hash := b.GetCanonicalHash(number)
+	if hash == (common.Hash{}) {
+		return nil, fmt.Errorf("no canoncial block hash found for provided height (%d)", number)
+	}
+	return b.GetStorageByHash(ctx, address, storageLeafKey, hash)
+}
+
+// GetStorageByHash returns the storage value for the provided contract address an storage key at the block corresponding to the provided hash
+func (b *Backend) GetStorageByHash(ctx context.Context, address common.Address, storageLeafKey, hash common.Hash) (hexutil.Bytes, error) {
+	_, storageRlp, err := b.IPLDRetriever.RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(address, storageLeafKey, hash)
+	return storageRlp, err
+}
+
+// Engine satisfied the ChainContext interface
+func (b *Backend) Engine() consensus.Engine {
+	// TODO: we need to support more than just ethash based engines
+	return ethash.NewFaker()
+}
+
+// GetHeader satisfied the ChainContext interface
+func (b *Backend) GetHeader(hash common.Hash, height uint64) *types.Header {
+	header, err := b.HeaderByHash(context.Background(), hash)
 	if err != nil {
-		return nil, err
-	}
-	td, err := pea.B.GetTd(b.Hash())
-	if err != nil {
-		return nil, err
-	}
-	fields["totalDifficulty"] = (*hexutil.Big)(td)
-	return fields, err
-}
-
-// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are
-// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain
-// transaction hashes.
-func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
-	fields := RPCMarshalHeader(block.Header())
-	fields["size"] = hexutil.Uint64(block.Size())
-
-	if inclTx {
-		formatTx := func(tx *types.Transaction) (interface{}, error) {
-			return tx.Hash(), nil
-		}
-		if fullTx {
-			formatTx = func(tx *types.Transaction) (interface{}, error) {
-				return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil
-			}
-		}
-		txs := block.Transactions()
-		transactions := make([]interface{}, len(txs))
-		var err error
-		for i, tx := range txs {
-			if transactions[i], err = formatTx(tx); err != nil {
-				return nil, err
-			}
-		}
-		fields["transactions"] = transactions
-	}
-	uncles := block.Uncles()
-	uncleHashes := make([]common.Hash, len(uncles))
-	for i, uncle := range uncles {
-		uncleHashes[i] = uncle.Hash()
-	}
-	fields["uncles"] = uncleHashes
-
-	return fields, nil
-}
-
-// NewRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
-func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction {
-	for idx, tx := range b.Transactions() {
-		if tx.Hash() == hash {
-			return newRPCTransactionFromBlockIndex(b, uint64(idx))
-		}
-	}
-	return nil
-}
-
-// newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation.
-func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransaction {
-	txs := b.Transactions()
-	if index >= uint64(len(txs)) {
 		return nil
 	}
-	return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index)
+	return header
 }
 
-// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
-type RPCTransaction struct {
-	BlockHash        *common.Hash    `json:"blockHash"`
-	BlockNumber      *hexutil.Big    `json:"blockNumber"`
-	From             common.Address  `json:"from"`
-	Gas              hexutil.Uint64  `json:"gas"`
-	GasPrice         *hexutil.Big    `json:"gasPrice"`
-	Hash             common.Hash     `json:"hash"`
-	Input            hexutil.Bytes   `json:"input"`
-	Nonce            hexutil.Uint64  `json:"nonce"`
-	To               *common.Address `json:"to"`
-	TransactionIndex *hexutil.Uint64 `json:"transactionIndex"`
-	Value            *hexutil.Big    `json:"value"`
-	V                *hexutil.Big    `json:"v"`
-	R                *hexutil.Big    `json:"r"`
-	S                *hexutil.Big    `json:"s"`
+// RPCGasCap returns the configured gas cap for the rpc server
+func (b *Backend) RPCGasCap() *big.Int {
+	return b.Config.RPCGasCap
 }
 
-// NewRPCTransaction returns a transaction that will serialize to the RPC
-// representation, with the given location metadata set (if available).
-func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
-	var signer types.Signer = types.FrontierSigner{}
-	if tx.Protected() {
-		signer = types.NewEIP155Signer(tx.ChainId())
-	}
-	from, _ := types.Sender(signer, tx)
-	v, r, s := tx.RawSignatureValues()
+func (b *Backend) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription {
+	panic("implement me")
+}
 
-	result := &RPCTransaction{
-		From:     from,
-		Gas:      hexutil.Uint64(tx.Gas()),
-		GasPrice: (*hexutil.Big)(tx.GasPrice()),
-		Hash:     tx.Hash(),
-		Input:    hexutil.Bytes(tx.Data()), // somehow this is ending up `nil`
-		Nonce:    hexutil.Uint64(tx.Nonce()),
-		To:       tx.To(),
-		Value:    (*hexutil.Big)(tx.Value()),
-		V:        (*hexutil.Big)(v),
-		R:        (*hexutil.Big)(r),
-		S:        (*hexutil.Big)(s),
-	}
-	if blockHash != (common.Hash{}) {
-		result.BlockHash = &blockHash
-		result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber))
-		result.TransactionIndex = (*hexutil.Uint64)(&index)
-	}
-	return result
+func (b *Backend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription {
+	panic("implement me")
+}
+
+func (b *Backend) BloomStatus() (uint64, uint64) {
+	panic("implement me")
+}
+
+func (b *Backend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {
+	panic("implement me")
 }

pkg/eth/backend_utils.go

@@ -0,0 +1,320 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/rpc"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+
+	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+)
+
+// RPCMarshalHeader converts the given header to the RPC output.
+// This function is eth/internal so we have to make our own version here...
+func RPCMarshalHeader(head *types.Header) map[string]interface{} {
+	return map[string]interface{}{
+		"number":           (*hexutil.Big)(head.Number),
+		"hash":             head.Hash(),
+		"parentHash":       head.ParentHash,
+		"nonce":            head.Nonce,
+		"mixHash":          head.MixDigest,
+		"sha3Uncles":       head.UncleHash,
+		"logsBloom":        head.Bloom,
+		"stateRoot":        head.Root,
+		"miner":            head.Coinbase,
+		"difficulty":       (*hexutil.Big)(head.Difficulty),
+		"extraData":        hexutil.Bytes(head.Extra),
+		"size":             hexutil.Uint64(head.Size()),
+		"gasLimit":         hexutil.Uint64(head.GasLimit),
+		"gasUsed":          hexutil.Uint64(head.GasUsed),
+		"timestamp":        hexutil.Uint64(head.Time),
+		"transactionsRoot": head.TxHash,
+		"receiptsRoot":     head.ReceiptHash,
+	}
+}
+
+// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are
+// returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain
+// transaction hashes.
+func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) {
+	fields := RPCMarshalHeader(block.Header())
+	fields["size"] = hexutil.Uint64(block.Size())
+
+	if inclTx {
+		formatTx := func(tx *types.Transaction) (interface{}, error) {
+			return tx.Hash(), nil
+		}
+		if fullTx {
+			formatTx = func(tx *types.Transaction) (interface{}, error) {
+				return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil
+			}
+		}
+		txs := block.Transactions()
+		transactions := make([]interface{}, len(txs))
+		var err error
+		for i, tx := range txs {
+			if transactions[i], err = formatTx(tx); err != nil {
+				return nil, err
+			}
+		}
+		fields["transactions"] = transactions
+	}
+	uncles := block.Uncles()
+	uncleHashes := make([]common.Hash, len(uncles))
+	for i, uncle := range uncles {
+		uncleHashes[i] = uncle.Hash()
+	}
+	fields["uncles"] = uncleHashes
+
+	return fields, nil
+}
+
+// RPCMarshalBlockWithUncleHashes marshals the block with the provided uncle hashes
+func RPCMarshalBlockWithUncleHashes(block *types.Block, uncleHashes []common.Hash, inclTx bool, fullTx bool) (map[string]interface{}, error) {
+	fields := RPCMarshalHeader(block.Header())
+	fields["size"] = hexutil.Uint64(block.Size())
+
+	if inclTx {
+		formatTx := func(tx *types.Transaction) (interface{}, error) {
+			return tx.Hash(), nil
+		}
+		if fullTx {
+			formatTx = func(tx *types.Transaction) (interface{}, error) {
+				return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil
+			}
+		}
+		txs := block.Transactions()
+		transactions := make([]interface{}, len(txs))
+		var err error
+		for i, tx := range txs {
+			if transactions[i], err = formatTx(tx); err != nil {
+				return nil, err
+			}
+		}
+		fields["transactions"] = transactions
+	}
+	fields["uncles"] = uncleHashes
+
+	return fields, nil
+}
+
+// NewRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
+func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction {
+	for idx, tx := range b.Transactions() {
+		if tx.Hash() == hash {
+			return newRPCTransactionFromBlockIndex(b, uint64(idx))
+		}
+	}
+	return nil
+}
+
+// NewRPCTransaction returns a transaction that will serialize to the RPC
+// representation, with the given location metadata set (if available).
+func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
+	var signer types.Signer = types.FrontierSigner{}
+	if tx.Protected() {
+		signer = types.NewEIP155Signer(tx.ChainId())
+	}
+	from, _ := types.Sender(signer, tx)
+	v, r, s := tx.RawSignatureValues()
+
+	result := &RPCTransaction{
+		From:     from,
+		Gas:      hexutil.Uint64(tx.Gas()),
+		GasPrice: (*hexutil.Big)(tx.GasPrice()),
+		Hash:     tx.Hash(),
+		Input:    hexutil.Bytes(tx.Data()), // somehow this is ending up `nil`
+		Nonce:    hexutil.Uint64(tx.Nonce()),
+		To:       tx.To(),
+		Value:    (*hexutil.Big)(tx.Value()),
+		V:        (*hexutil.Big)(v),
+		R:        (*hexutil.Big)(r),
+		S:        (*hexutil.Big)(s),
+	}
+	if blockHash != (common.Hash{}) {
+		result.BlockHash = &blockHash
+		result.BlockNumber = (*hexutil.Big)(new(big.Int).SetUint64(blockNumber))
+		result.TransactionIndex = (*hexutil.Uint64)(&index)
+	}
+	return result
+}
+
+type rpcBlock struct {
+	Hash         common.Hash      `json:"hash"`
+	Transactions []rpcTransaction `json:"transactions"`
+	UncleHashes  []common.Hash    `json:"uncles"`
+}
+
+type rpcTransaction struct {
+	tx *types.Transaction
+	txExtraInfo
+}
+
+type txExtraInfo struct {
+	BlockNumber *string         `json:"blockNumber,omitempty"`
+	BlockHash   *common.Hash    `json:"blockHash,omitempty"`
+	From        *common.Address `json:"from,omitempty"`
+}
+
+func (tx *rpcTransaction) UnmarshalJSON(msg []byte) error {
+	if err := json.Unmarshal(msg, &tx.tx); err != nil {
+		return err
+	}
+	return json.Unmarshal(msg, &tx.txExtraInfo)
+}
+
+func getBlockAndUncleHashes(cli *rpc.Client, ctx context.Context, method string, args ...interface{}) (*types.Block, []common.Hash, error) {
+	var raw json.RawMessage
+	err := cli.CallContext(ctx, &raw, method, args...)
+	if err != nil {
+		return nil, nil, err
+	} else if len(raw) == 0 {
+		return nil, nil, ethereum.NotFound
+	}
+	// Decode header and transactions.
+	var head *types.Header
+	var body rpcBlock
+	if err := json.Unmarshal(raw, &head); err != nil {
+		return nil, nil, err
+	}
+	if err := json.Unmarshal(raw, &body); err != nil {
+		return nil, nil, err
+	}
+	// Quick-verify transaction and uncle lists. This mostly helps with debugging the server.
+	if head.UncleHash == types.EmptyUncleHash && len(body.UncleHashes) > 0 {
+		return nil, nil, fmt.Errorf("server returned non-empty uncle list but block header indicates no uncles")
+	}
+	if head.UncleHash != types.EmptyUncleHash && len(body.UncleHashes) == 0 {
+		return nil, nil, fmt.Errorf("server returned empty uncle list but block header indicates uncles")
+	}
+	if head.TxHash == types.EmptyRootHash && len(body.Transactions) > 0 {
+		return nil, nil, fmt.Errorf("server returned non-empty transaction list but block header indicates no transactions")
+	}
+	if head.TxHash != types.EmptyRootHash && len(body.Transactions) == 0 {
+		return nil, nil, fmt.Errorf("server returned empty transaction list but block header indicates transactions")
+	}
+	txs := make([]*types.Transaction, len(body.Transactions))
+	for i, tx := range body.Transactions {
+		txs[i] = tx.tx
+	}
+	return types.NewBlockWithHeader(head).WithBody(txs, nil), body.UncleHashes, nil
+}
+
+// newRPCRawTransactionFromBlockIndex returns the bytes of a transaction given a block and a transaction index.
+func newRPCRawTransactionFromBlockIndex(b *types.Block, index uint64) hexutil.Bytes {
+	txs := b.Transactions()
+	if index >= uint64(len(txs)) {
+		return nil
+	}
+	blob, _ := rlp.EncodeToBytes(txs[index])
+	return blob
+}
+
+// newRPCTransactionFromBlockIndex returns a transaction that will serialize to the RPC representation.
+func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransaction {
+	txs := b.Transactions()
+	if index >= uint64(len(txs)) {
+		return nil
+	}
+	return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index)
+}
+
+// extractLogsOfInterest returns logs from the receipt IPLD
+func extractLogsOfInterest(rctIPLDs []ipfs.BlockModel, wantedTopics [][]string) ([]*types.Log, error) {
+	var logs []*types.Log
+	for _, rctIPLD := range rctIPLDs {
+		rctRLP := rctIPLD
+		var rct types.Receipt
+		if err := rlp.DecodeBytes(rctRLP.Data, &rct); err != nil {
+			return nil, err
+		}
+		for _, log := range rct.Logs {
+			if wanted := wantedLog(wantedTopics, log.Topics); wanted == true {
+				logs = append(logs, log)
+			}
+		}
+	}
+	return logs, nil
+}
+
+// returns true if the log matches on the filter
+func wantedLog(wantedTopics [][]string, actualTopics []common.Hash) bool {
+	// actualTopics will always have length <= 4
+	// wantedTopics will always have length 4
+	matches := 0
+	for i, actualTopic := range actualTopics {
+		// If we have topics in this filter slot, count as a match if the actualTopic matches one of the ones in this filter slot
+		if len(wantedTopics[i]) > 0 {
+			matches += sliceContainsHash(wantedTopics[i], actualTopic)
+		} else {
+			// Filter slot is empty, not matching any topics at this slot => counts as a match
+			matches++
+		}
+	}
+	if matches == len(actualTopics) {
+		return true
+	}
+	return false
+}
+
+// returns 1 if the slice contains the hash, 0 if it does not
+func sliceContainsHash(slice []string, hash common.Hash) int {
+	for _, str := range slice {
+		if str == hash.String() {
+			return 1
+		}
+	}
+	return 0
+}
+
+func toFilterArg(q ethereum.FilterQuery) (interface{}, error) {
+	arg := map[string]interface{}{
+		"address": q.Addresses,
+		"topics":  q.Topics,
+	}
+	if q.BlockHash != nil {
+		arg["blockHash"] = *q.BlockHash
+		if q.FromBlock != nil || q.ToBlock != nil {
+			return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock")
+		}
+	} else {
+		if q.FromBlock == nil {
+			arg["fromBlock"] = "0x0"
+		} else {
+			arg["fromBlock"] = toBlockNumArg(q.FromBlock)
+		}
+		arg["toBlock"] = toBlockNumArg(q.ToBlock)
+	}
+	return arg, nil
+}
+
+func toBlockNumArg(number *big.Int) string {
+	if number == nil {
+		return "latest"
+	}
+	return hexutil.EncodeBig(number)
+}

pkg/eth/cid_retriever.go

@@ -26,7 +26,7 @@ import (
 	"github.com/lib/pq"
 	log "github.com/sirupsen/logrus"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/shared"
@@ -36,7 +36,7 @@ import (
 type Retriever interface {
 	RetrieveFirstBlockNumber() (int64, error)
 	RetrieveLastBlockNumber() (int64, error)
-	Retrieve(filter SubscriptionSettings, blockNumber int64) ([]eth2.CIDWrapper, bool, error)
+	Retrieve(filter SubscriptionSettings, blockNumber int64) ([]CIDWrapper, bool, error)
 }
 
 // CIDRetriever satisfies the CIDRetriever interface for ethereum
@@ -66,7 +66,7 @@ func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
 }
 
 // Retrieve is used to retrieve all of the CIDs which conform to the passed StreamFilters
-func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64) ([]eth2.CIDWrapper, bool, error) {
+func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64) ([]CIDWrapper, bool, error) {
 	log.Debug("retrieving cids")
 
 	// Begin new db tx
@@ -91,10 +91,10 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 		log.Error("header cid retrieval error")
 		return nil, true, err
 	}
-	cws := make([]eth2.CIDWrapper, len(headers))
+	cws := make([]CIDWrapper, len(headers))
 	empty := true
 	for i, header := range headers {
-		cw := new(eth2.CIDWrapper)
+		cw := new(CIDWrapper)
 		cw.BlockNumber = big.NewInt(blockNumber)
 		if !filter.HeaderFilter.Off {
 			cw.Header = header
@@ -164,18 +164,18 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 }
 
 // RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
-func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]eth2.HeaderModel, error) {
+func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]eth.HeaderModel, error) {
 	log.Debug("retrieving header cids for block ", blockNumber)
-	headers := make([]eth2.HeaderModel, 0)
+	headers := make([]eth.HeaderModel, 0)
 	pgStr := `SELECT * FROM eth.header_cids
 				WHERE block_number = $1`
 	return headers, tx.Select(&headers, pgStr, blockNumber)
 }
 
 // RetrieveUncleCIDsByHeaderID retrieves and returns all of the uncle cids for the provided header
-func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth2.UncleModel, error) {
+func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth.UncleModel, error) {
 	log.Debug("retrieving uncle cids for block id ", headerID)
-	headers := make([]eth2.UncleModel, 0)
+	headers := make([]eth.UncleModel, 0)
 	pgStr := `SELECT * FROM eth.uncle_cids
 				WHERE header_id = $1`
 	return headers, tx.Select(&headers, pgStr, headerID)
@@ -183,15 +183,14 @@ func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64
 
 // RetrieveTxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters
 // also returns the ids for the returned transaction cids
-func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]eth2.TxModel, error) {
+func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]eth.TxModel, error) {
 	log.Debug("retrieving transaction cids for header id ", headerID)
 	args := make([]interface{}, 0, 3)
-	results := make([]eth2.TxModel, 0)
+	results := make([]eth.TxModel, 0)
 	id := 1
 	pgStr := fmt.Sprintf(`SELECT transaction_cids.id, transaction_cids.header_id,
  			transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.mh_key,
- 			transaction_cids.dst, transaction_cids.src, transaction_cids.index,
-			transaction_cids.tx_data, transaction_cids.deployment
+ 			transaction_cids.dst, transaction_cids.src, transaction_cids.index, transaction_cids.tx_data
  			FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
 			WHERE header_cids.id = $%d`, id)
 	args = append(args, headerID)
@@ -211,7 +210,7 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID
 
 // RetrieveRctCIDsByHeaderID retrieves and returns all of the rct cids at the provided header ID that conform to the provided
 // filter parameters and correspond to the provided tx ids
-func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]eth2.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]eth.ReceiptModel, error) {
 	log.Debug("retrieving receipt cids for header id ", headerID)
 	args := make([]interface{}, 0, 4)
 	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
@@ -285,13 +284,13 @@ func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter Receip
 		}
 	}
 	pgStr += ` ORDER BY transaction_cids.index`
-	receiptCids := make([]eth2.ReceiptModel, 0)
+	receiptCids := make([]eth.ReceiptModel, 0)
 	return receiptCids, tx.Select(&receiptCids, pgStr, args...)
 }
 
 // RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight or block hash that conform to the provided
 // filter parameters and correspond to the provided tx ids
-func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]eth2.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]eth.ReceiptModel, error) {
 	log.Debug("retrieving receipt cids for block ", blockNumber)
 	args := make([]interface{}, 0, 5)
 	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
@@ -373,7 +372,7 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b
 		}
 	}
 	pgStr += ` ORDER BY transaction_cids.index`
-	receiptCids := make([]eth2.ReceiptModel, 0)
+	receiptCids := make([]eth.ReceiptModel, 0)
 	return receiptCids, tx.Select(&receiptCids, pgStr, args...)
 }
 
@@ -387,7 +386,7 @@ func hasTopics(topics [][]string) bool {
 }
 
 // RetrieveStateCIDs retrieves and returns all of the state node cids at the provided header ID that conform to the provided filter parameters
-func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID int64) ([]eth2.StateNodeModel, error) {
+func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID int64) ([]eth.StateNodeModel, error) {
 	log.Debug("retrieving state cids for header id ", headerID)
 	args := make([]interface{}, 0, 2)
 	pgStr := `SELECT state_cids.id, state_cids.header_id,
@@ -407,12 +406,12 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
 	if !stateFilter.IntermediateNodes {
 		pgStr += ` AND state_cids.node_type = 2`
 	}
-	stateNodeCIDs := make([]eth2.StateNodeModel, 0)
+	stateNodeCIDs := make([]eth.StateNodeModel, 0)
 	return stateNodeCIDs, tx.Select(&stateNodeCIDs, pgStr, args...)
 }
 
 // RetrieveStorageCIDs retrieves and returns all of the storage node cids at the provided header id that conform to the provided filter parameters
-func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID int64) ([]eth2.StorageNodeWithStateKeyModel, error) {
+func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID int64) ([]eth.StorageNodeWithStateKeyModel, error) {
 	log.Debug("retrieving storage cids for header id ", headerID)
 	args := make([]interface{}, 0, 3)
 	pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_leaf_key, storage_cids.node_type,
@@ -440,18 +439,18 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF
 	if !storageFilter.IntermediateNodes {
 		pgStr += ` AND storage_cids.node_type = 2`
 	}
-	storageNodeCIDs := make([]eth2.StorageNodeWithStateKeyModel, 0)
+	storageNodeCIDs := make([]eth.StorageNodeWithStateKeyModel, 0)
 	return storageNodeCIDs, tx.Select(&storageNodeCIDs, pgStr, args...)
 }
 
 // RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash
-func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth2.HeaderModel, []eth2.UncleModel, []eth2.TxModel, []eth2.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth.HeaderModel, []eth.UncleModel, []eth.TxModel, []eth.ReceiptModel, error) {
 	log.Debug("retrieving block cids for block hash ", blockHash.String())
 
 	// Begin new db tx
 	tx, err := ecr.db.Beginx()
 	if err != nil {
-		return eth2.HeaderModel{}, nil, nil, nil, err
+		return eth.HeaderModel{}, nil, nil, nil, err
 	}
 	defer func() {
 		if p := recover(); p != nil {
@@ -467,17 +466,17 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth2.Header
 	headerCID, err := ecr.RetrieveHeaderCIDByHash(tx, blockHash)
 	if err != nil {
 		log.Error("header cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
+		return eth.HeaderModel{}, nil, nil, nil, err
 	}
 	uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.ID)
 	if err != nil {
 		log.Error("uncle cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
+		return eth.HeaderModel{}, nil, nil, nil, err
 	}
 	txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.ID)
 	if err != nil {
 		log.Error("tx cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
+		return eth.HeaderModel{}, nil, nil, nil, err
 	}
 	txIDs := make([]int64, len(txCIDs))
 	for i, txCID := range txCIDs {
@@ -491,13 +490,13 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth2.Header
 }
 
 // RetrieveBlockByNumber returns all of the CIDs needed to compose an entire block, for a given block number
-func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth2.HeaderModel, []eth2.UncleModel, []eth2.TxModel, []eth2.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth.HeaderModel, []eth.UncleModel, []eth.TxModel, []eth.ReceiptModel, error) {
 	log.Debug("retrieving block cids for block number ", blockNumber)
 
 	// Begin new db tx
 	tx, err := ecr.db.Beginx()
 	if err != nil {
-		return eth2.HeaderModel{}, nil, nil, nil, err
+		return eth.HeaderModel{}, nil, nil, nil, err
 	}
 	defer func() {
 		if p := recover(); p != nil {
@@ -513,20 +512,20 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth2.HeaderMo
 	headerCID, err := ecr.RetrieveHeaderCIDs(tx, blockNumber)
 	if err != nil {
 		log.Error("header cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
+		return eth.HeaderModel{}, nil, nil, nil, err
 	}
 	if len(headerCID) < 1 {
-		return eth2.HeaderModel{}, nil, nil, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber)
+		return eth.HeaderModel{}, nil, nil, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber)
 	}
 	uncleCIDs, err := ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID[0].ID)
 	if err != nil {
 		log.Error("uncle cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
+		return eth.HeaderModel{}, nil, nil, nil, err
 	}
 	txCIDs, err := ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].ID)
 	if err != nil {
 		log.Error("tx cid retrieval error")
-		return eth2.HeaderModel{}, nil, nil, nil, err
+		return eth.HeaderModel{}, nil, nil, nil, err
 	}
 	txIDs := make([]int64, len(txCIDs))
 	for i, txCID := range txCIDs {
@@ -540,26 +539,26 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth2.HeaderMo
 }
 
 // RetrieveHeaderCIDByHash returns the header for the given block hash
-func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (eth2.HeaderModel, error) {
+func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (eth.HeaderModel, error) {
 	log.Debug("retrieving header cids for block hash ", blockHash.String())
 	pgStr := `SELECT * FROM eth.header_cids
 			WHERE block_hash = $1`
-	var headerCID eth2.HeaderModel
+	var headerCID eth.HeaderModel
 	return headerCID, tx.Get(&headerCID, pgStr, blockHash.String())
 }
 
 // RetrieveTxCIDsByHeaderID retrieves all tx CIDs for the given header id
-func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth2.TxModel, error) {
+func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth.TxModel, error) {
 	log.Debug("retrieving tx cids for block id ", headerID)
 	pgStr := `SELECT * FROM eth.transaction_cids
 			WHERE header_id = $1
 			ORDER BY index`
-	var txCIDs []eth2.TxModel
+	var txCIDs []eth.TxModel
 	return txCIDs, tx.Select(&txCIDs, pgStr, headerID)
 }
 
 // RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs
-func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]eth2.ReceiptModel, error) {
+func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]eth.ReceiptModel, error) {
 	log.Debugf("retrieving receipt cids for tx ids %v", txIDs)
 	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
  			receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
@@ -568,6 +567,6 @@ func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64)
 			WHERE tx_id = ANY($1::INTEGER[])
 			AND receipt_cids.tx_id = transaction_cids.id
 			ORDER BY transaction_cids.index`
-	var rctCIDs []eth2.ReceiptModel
+	var rctCIDs []eth.ReceiptModel
 	return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txIDs))
 }

pkg/eth/cid_retriever_test.go

@@ -19,17 +19,19 @@ package eth_test
 import (
 	"math/big"
 
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
 	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
+	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
 )
 
 var (
@@ -52,7 +54,7 @@ var (
 			Off: true,
 		},
 		ReceiptFilter: eth.ReceiptFilter{
-			LogAddresses: []string{mocks.Address.String()},
+			LogAddresses: []string{test_helpers.Address.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -94,7 +96,7 @@ var (
 				{"0x0000000000000000000000000000000000000000000000000000000000000004"},
 				{"0x0000000000000000000000000000000000000000000000000000000000000006"},
 			},
-			LogAddresses: []string{mocks.Address.String()},
+			LogAddresses: []string{test_helpers.Address.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -117,7 +119,7 @@ var (
 				{"0x0000000000000000000000000000000000000000000000000000000000000004"},
 				{"0x0000000000000000000000000000000000000000000000000000000000000007"}, // This topic won't match on the mocks.Address.String() contract receipt
 			},
-			LogAddresses: []string{mocks.Address.String()},
+			LogAddresses: []string{test_helpers.Address.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -137,7 +139,7 @@ var (
 		},
 		ReceiptFilter: eth.ReceiptFilter{
 			Topics:       [][]string{{"0x0000000000000000000000000000000000000000000000000000000000000005"}},
-			LogAddresses: []string{mocks.Address.String(), mocks.AnotherAddress.String()},
+			LogAddresses: []string{test_helpers.Address.String(), test_helpers.AnotherAddress.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -172,7 +174,7 @@ var (
 			Off: true,
 		},
 		TxFilter: eth.TxFilter{
-			Dst: []string{mocks.AnotherAddress.String()}, // We only filter for one of the trxs so we will only get the one corresponding receipt
+			Dst: []string{test_helpers.AnotherAddress.String()}, // We only filter for one of the trxs so we will only get the one corresponding receipt
 		},
 		ReceiptFilter: eth.ReceiptFilter{
 			MatchTxs:     true,
@@ -199,7 +201,7 @@ var (
 			Off: true,
 		},
 		StateFilter: eth.StateFilter{
-			Addresses: []string{mocks.AccountAddresss.Hex()},
+			Addresses: []string{test_helpers.AccountAddresss.Hex()},
 		},
 		StorageFilter: eth.StorageFilter{
 			Off: true,
@@ -226,7 +228,7 @@ var _ = Describe("Retriever", func() {
 
 	Describe("Retrieve", func() {
 		BeforeEach(func() {
-			err := repo.Publish(mocks.MockConvertedPayload)
+			err := repo.Publish(test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
@@ -234,34 +236,34 @@ var _ = Describe("Retriever", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids)).To(Equal(1))
-			Expect(cids[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
-			expectedHeaderCID := mocks.MockCIDWrapper.Header
+			Expect(cids[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
+			expectedHeaderCID := test_helpers.MockCIDWrapper.Header
 			expectedHeaderCID.ID = cids[0].Header.ID
 			expectedHeaderCID.NodeID = cids[0].Header.NodeID
 			Expect(cids[0].Header).To(Equal(expectedHeaderCID))
 			Expect(len(cids[0].Transactions)).To(Equal(3))
-			Expect(eth.TxModelsContainsCID(cids[0].Transactions, mocks.MockCIDWrapper.Transactions[0].CID)).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids[0].Transactions, mocks.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids[0].Transactions, mocks.MockCIDWrapper.Transactions[2].CID)).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[0].CID)).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cids[0].Transactions, test_helpers.MockCIDWrapper.Transactions[2].CID)).To(BeTrue())
 			Expect(len(cids[0].Receipts)).To(Equal(3))
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, mocks.MockCIDWrapper.Receipts[0].CID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, mocks.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, mocks.MockCIDWrapper.Receipts[2].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[0].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, test_helpers.MockCIDWrapper.Receipts[2].CID)).To(BeTrue())
 			Expect(len(cids[0].StateNodes)).To(Equal(2))
 			for _, stateNode := range cids[0].StateNodes {
-				if stateNode.CID == mocks.State1CID.String() {
-					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
+				if stateNode.CID == test_helpers.State1CID.String() {
+					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(test_helpers.ContractLeafKey).Hex()))
 					Expect(stateNode.NodeType).To(Equal(2))
 					Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
 				}
-				if stateNode.CID == mocks.State2CID.String() {
-					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
+				if stateNode.CID == test_helpers.State2CID.String() {
+					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(test_helpers.AccountLeafKey).Hex()))
 					Expect(stateNode.NodeType).To(Equal(2))
 					Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
 				}
 			}
 			Expect(len(cids[0].StorageNodes)).To(Equal(1))
-			expectedStorageNodeCIDs := mocks.MockCIDWrapper.StorageNodes
+			expectedStorageNodeCIDs := test_helpers.MockCIDWrapper.StorageNodes
 			expectedStorageNodeCIDs[0].ID = cids[0].StorageNodes[0].ID
 			expectedStorageNodeCIDs[0].StateID = cids[0].StorageNodes[0].StateID
 			Expect(cids[0].StorageNodes).To(Equal(expectedStorageNodeCIDs))
@@ -272,13 +274,13 @@ var _ = Describe("Retriever", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids1)).To(Equal(1))
-			Expect(cids1[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(cids1[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
 			Expect(cids1[0].Header).To(Equal(eth2.HeaderModel{}))
 			Expect(len(cids1[0].Transactions)).To(Equal(0))
 			Expect(len(cids1[0].StateNodes)).To(Equal(0))
 			Expect(len(cids1[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids1[0].Receipts)).To(Equal(1))
-			expectedReceiptCID := mocks.MockCIDWrapper.Receipts[0]
+			expectedReceiptCID := test_helpers.MockCIDWrapper.Receipts[0]
 			expectedReceiptCID.ID = cids1[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids1[0].Receipts[0].TxID
 			Expect(cids1[0].Receipts[0]).To(Equal(expectedReceiptCID))
@@ -287,13 +289,13 @@ var _ = Describe("Retriever", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids2)).To(Equal(1))
-			Expect(cids2[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(cids2[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
 			Expect(cids2[0].Header).To(Equal(eth2.HeaderModel{}))
 			Expect(len(cids2[0].Transactions)).To(Equal(0))
 			Expect(len(cids2[0].StateNodes)).To(Equal(0))
 			Expect(len(cids2[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids2[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[0]
+			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
 			expectedReceiptCID.ID = cids2[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids2[0].Receipts[0].TxID
 			Expect(cids2[0].Receipts[0]).To(Equal(expectedReceiptCID))
@@ -302,13 +304,13 @@ var _ = Describe("Retriever", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids3)).To(Equal(1))
-			Expect(cids3[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(cids3[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
 			Expect(cids3[0].Header).To(Equal(eth2.HeaderModel{}))
 			Expect(len(cids3[0].Transactions)).To(Equal(0))
 			Expect(len(cids3[0].StateNodes)).To(Equal(0))
 			Expect(len(cids3[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids3[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[0]
+			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[0]
 			expectedReceiptCID.ID = cids3[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids3[0].Receipts[0].TxID
 			Expect(cids3[0].Receipts[0]).To(Equal(expectedReceiptCID))
@@ -317,13 +319,13 @@ var _ = Describe("Retriever", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids4)).To(Equal(1))
-			Expect(cids4[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(cids4[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
 			Expect(cids4[0].Header).To(Equal(eth2.HeaderModel{}))
 			Expect(len(cids4[0].Transactions)).To(Equal(0))
 			Expect(len(cids4[0].StateNodes)).To(Equal(0))
 			Expect(len(cids4[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids4[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[1]
+			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
 			expectedReceiptCID.ID = cids4[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids4[0].Receipts[0].TxID
 			Expect(cids4[0].Receipts[0]).To(Equal(expectedReceiptCID))
@@ -332,34 +334,34 @@ var _ = Describe("Retriever", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids5)).To(Equal(1))
-			Expect(cids5[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(cids5[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
 			Expect(cids5[0].Header).To(Equal(eth2.HeaderModel{}))
 			Expect(len(cids5[0].Transactions)).To(Equal(3))
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks.Trx1CID.String())).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks.Trx2CID.String())).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks.Trx3CID.String())).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx1CID.String())).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx2CID.String())).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx3CID.String())).To(BeTrue())
 			Expect(len(cids5[0].StateNodes)).To(Equal(0))
 			Expect(len(cids5[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids5[0].Receipts)).To(Equal(3))
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks.Rct1CID.String())).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks.Rct2CID.String())).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks.Rct3CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct1CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct2CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, test_helpers.Rct3CID.String())).To(BeTrue())
 
 			cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids6)).To(Equal(1))
-			Expect(cids6[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(cids6[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
 			Expect(cids6[0].Header).To(Equal(eth2.HeaderModel{}))
 			Expect(len(cids6[0].Transactions)).To(Equal(1))
-			expectedTxCID := mocks.MockCIDWrapper.Transactions[1]
+			expectedTxCID := test_helpers.MockCIDWrapper.Transactions[1]
 			expectedTxCID.ID = cids6[0].Transactions[0].ID
 			expectedTxCID.HeaderID = cids6[0].Transactions[0].HeaderID
 			Expect(cids6[0].Transactions[0]).To(Equal(expectedTxCID))
 			Expect(len(cids6[0].StateNodes)).To(Equal(0))
 			Expect(len(cids6[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids6[0].Receipts)).To(Equal(1))
-			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[1]
+			expectedReceiptCID = test_helpers.MockCIDWrapper.Receipts[1]
 			expectedReceiptCID.ID = cids6[0].Receipts[0].ID
 			expectedReceiptCID.TxID = cids6[0].Receipts[0].TxID
 			Expect(cids6[0].Receipts[0]).To(Equal(expectedReceiptCID))
@@ -368,7 +370,7 @@ var _ = Describe("Retriever", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(empty).ToNot(BeTrue())
 			Expect(len(cids7)).To(Equal(1))
-			Expect(cids7[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(cids7[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
 			Expect(cids7[0].Header).To(Equal(eth2.HeaderModel{}))
 			Expect(len(cids7[0].Transactions)).To(Equal(0))
 			Expect(len(cids7[0].Receipts)).To(Equal(0))
@@ -378,9 +380,9 @@ var _ = Describe("Retriever", func() {
 				ID:       cids7[0].StateNodes[0].ID,
 				HeaderID: cids7[0].StateNodes[0].HeaderID,
 				NodeType: 2,
-				StateKey: common.BytesToHash(mocks.AccountLeafKey).Hex(),
-				CID:      mocks.State2CID.String(),
-				MhKey:    mocks.State2MhKey,
+				StateKey: common.BytesToHash(test_helpers.AccountLeafKey).Hex(),
+				CID:      test_helpers.State2CID.String(),
+				MhKey:    test_helpers.State2MhKey,
 				Path:     []byte{'\x0c'},
 			}))
 
@@ -396,7 +398,7 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the first block that has data in the database", func() {
-			err := repo.Publish(mocks.MockConvertedPayload)
+			err := repo.Publish(test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
@@ -404,7 +406,7 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Gets the number of the first block that has data in the database", func() {
-			payload := mocks.MockConvertedPayload
+			payload := test_helpers.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
 			err := repo.Publish(payload)
 			Expect(err).ToNot(HaveOccurred())
@@ -414,7 +416,7 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Gets the number of the first block that has data in the database", func() {
-			payload1 := mocks.MockConvertedPayload
+			payload1 := test_helpers.MockConvertedPayload
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
@@ -434,7 +436,7 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the latest block that has data in the database", func() {
-			err := repo.Publish(mocks.MockConvertedPayload)
+			err := repo.Publish(test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
@@ -442,7 +444,7 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Gets the number of the latest block that has data in the database", func() {
-			payload := mocks.MockConvertedPayload
+			payload := test_helpers.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
 			err := repo.Publish(payload)
 			Expect(err).ToNot(HaveOccurred())
@@ -452,7 +454,7 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Gets the number of the latest block that has data in the database", func() {
-			payload1 := mocks.MockConvertedPayload
+			payload1 := test_helpers.MockConvertedPayload
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
@@ -468,7 +470,7 @@ var _ = Describe("Retriever", func() {
 })
 
 func newMockBlock(blockNumber uint64) *types.Block {
-	header := mocks.MockHeader
+	header := test_helpers.MockHeader
 	header.Number.SetUint64(blockNumber)
-	return types.NewBlock(&mocks.MockHeader, mocks.MockTransactions, nil, mocks.MockReceipts)
+	return types.NewBlock(&test_helpers.MockHeader, test_helpers.MockTransactions, nil, test_helpers.MockReceipts, new(trie.Trie))
 }

pkg/eth/eth_state_test.go

@@ -0,0 +1,467 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
+
+	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
+)
+
+var (
+	parsedABI abi.ABI
+)
+
+func init() {
+	// load abi
+	abiBytes, err := ioutil.ReadFile("./test_helpers/abi.json")
+	if err != nil {
+		panic(err)
+	}
+	parsedABI, err = abi.JSON(bytes.NewReader(abiBytes))
+	if err != nil {
+		panic(err)
+	}
+}
+
+var _ = Describe("eth state reading tests", func() {
+	var (
+		blocks                  []*types.Block
+		receipts                []types.Receipts
+		chain                   *core.BlockChain
+		db                      *postgres.DB
+		api                     *eth.PublicEthAPI
+		backend                 *eth.Backend
+		chainConfig             = params.TestChainConfig
+		mockTD                  = big.NewInt(1337)
+		expectedCanonicalHeader map[string]interface{}
+	)
+	It("test init", func() {
+		// db and type initializations
+		var err error
+		db, err = shared.SetupDB()
+		Expect(err).ToNot(HaveOccurred())
+		transformer := eth2.NewStateDiffTransformer(chainConfig, db)
+		backend, err = eth.NewEthBackend(db, &eth.Config{
+			ChainConfig: chainConfig,
+			VmConfig:    vm.Config{},
+			RPCGasCap:   big.NewInt(10000000000),
+		})
+		Expect(err).ToNot(HaveOccurred())
+		api = eth.NewPublicEthAPI(backend, nil, false)
+
+		// make the test blockchain (and state)
+		blocks, receipts, chain = test_helpers.MakeChain(5, test_helpers.Genesis, test_helpers.TestChainGen)
+		params := statediff.Params{
+			IntermediateStateNodes:   true,
+			IntermediateStorageNodes: true,
+		}
+		canonicalHeader := blocks[1].Header()
+		expectedCanonicalHeader = map[string]interface{}{
+			"number":           (*hexutil.Big)(canonicalHeader.Number),
+			"hash":             canonicalHeader.Hash(),
+			"parentHash":       canonicalHeader.ParentHash,
+			"nonce":            canonicalHeader.Nonce,
+			"mixHash":          canonicalHeader.MixDigest,
+			"sha3Uncles":       canonicalHeader.UncleHash,
+			"logsBloom":        canonicalHeader.Bloom,
+			"stateRoot":        canonicalHeader.Root,
+			"miner":            canonicalHeader.Coinbase,
+			"difficulty":       (*hexutil.Big)(canonicalHeader.Difficulty),
+			"extraData":        hexutil.Bytes([]byte{}),
+			"size":             hexutil.Uint64(canonicalHeader.Size()),
+			"gasLimit":         hexutil.Uint64(canonicalHeader.GasLimit),
+			"gasUsed":          hexutil.Uint64(canonicalHeader.GasUsed),
+			"timestamp":        hexutil.Uint64(canonicalHeader.Time),
+			"transactionsRoot": canonicalHeader.TxHash,
+			"receiptsRoot":     canonicalHeader.ReceiptHash,
+			"totalDifficulty":  (*hexutil.Big)(mockTD),
+		}
+		// iterate over the blocks, generating statediff payloads, and transforming the data into Postgres
+		builder := statediff.NewBuilder(chain.StateCache())
+		for i, block := range blocks {
+			var args statediff.Args
+			var rcts types.Receipts
+			if i == 0 {
+				args = statediff.Args{
+					OldStateRoot: common.Hash{},
+					NewStateRoot: block.Root(),
+					BlockNumber:  block.Number(),
+					BlockHash:    block.Hash(),
+				}
+			} else {
+				args = statediff.Args{
+					OldStateRoot: blocks[i-1].Root(),
+					NewStateRoot: block.Root(),
+					BlockNumber:  block.Number(),
+					BlockHash:    block.Hash(),
+				}
+				rcts = receipts[i-1]
+			}
+			diff, err := builder.BuildStateDiffObject(args, params)
+			Expect(err).ToNot(HaveOccurred())
+			diffRlp, err := rlp.EncodeToBytes(diff)
+			Expect(err).ToNot(HaveOccurred())
+			blockRlp, err := rlp.EncodeToBytes(block)
+			Expect(err).ToNot(HaveOccurred())
+			receiptsRlp, err := rlp.EncodeToBytes(rcts)
+			Expect(err).ToNot(HaveOccurred())
+			payload := statediff.Payload{
+				StateObjectRlp:  diffRlp,
+				BlockRlp:        blockRlp,
+				ReceiptsRlp:     receiptsRlp,
+				TotalDifficulty: mockTD,
+			}
+			_, err = transformer.Transform(0, payload)
+			Expect(err).ToNot(HaveOccurred())
+		}
+
+		// Insert some non-canonical data into the database so that we test our ability to discern canonicity
+		indexAndPublisher := eth2.NewIPLDPublisher(db)
+		api = eth.NewPublicEthAPI(backend, nil, false)
+		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
+		Expect(err).ToNot(HaveOccurred())
+		// The non-canonical header has a child
+		err = indexAndPublisher.Publish(test_helpers.MockConvertedPayloadForChild)
+		Expect(err).ToNot(HaveOccurred())
+		err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)
+		Expect(err).ToNot(HaveOccurred())
+	})
+	defer It("test teardown", func() {
+		eth.TearDownDB(db)
+		chain.Stop()
+	})
+
+	Describe("eth_call", func() {
+		It("Applies call args (tx data) on top of state, returning the result (e.g. a Getter method call)", func() {
+			data, err := parsedABI.Pack("data")
+			Expect(err).ToNot(HaveOccurred())
+			bdata := hexutil.Bytes(data)
+			callArgs := eth.CallArgs{
+				To:   &test_helpers.ContractAddr,
+				Data: &bdata,
+			}
+			// Before contract deployment, returns nil
+			res, err := api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(0), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(BeNil())
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(1), nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res).To(BeNil())
+
+			// After deployment
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(2), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes := hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(3), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(4), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000009"))
+			Expect(res).To(Equal(expectedRes))
+
+			res, err = api.Call(context.Background(), callArgs, rpc.BlockNumberOrHashWithNumber(5), nil)
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000"))
+			Expect(res).To(Equal(expectedRes))
+		})
+	})
+
+	var (
+		expectedContractBalance   = (*hexutil.Big)(common.Big0)
+		expectedBankBalanceBlock0 = (*hexutil.Big)(test_helpers.TestBankFunds)
+
+		expectedAcct1BalanceBlock1 = (*hexutil.Big)(big.NewInt(10000))
+		expectedBankBalanceBlock1  = (*hexutil.Big)(new(big.Int).Sub(test_helpers.TestBankFunds, big.NewInt(10000)))
+
+		expectedAcct2BalanceBlock2 = (*hexutil.Big)(big.NewInt(1000))
+		expectedBankBalanceBlock2  = (*hexutil.Big)(new(big.Int).Sub(expectedBankBalanceBlock1.ToInt(), big.NewInt(1000)))
+
+		expectedAcct2BalanceBlock3 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock2.ToInt(), test_helpers.MiningReward))
+
+		expectedAcct2BalanceBlock4 = (*hexutil.Big)(new(big.Int).Add(expectedAcct2BalanceBlock3.ToInt(), test_helpers.MiningReward))
+
+		expectedAcct1BalanceBlock5 = (*hexutil.Big)(new(big.Int).Add(expectedAcct1BalanceBlock1.ToInt(), test_helpers.MiningReward))
+	)
+
+	Describe("eth_getBalance", func() {
+		It("Retrieves the eth balance for the provided account address at the block with the provided number", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+			_, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock2))
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock3))
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock5))
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+		})
+		It("Retrieves the eth balance for the provided account address at the block with the provided hash", func() {
+			bal, err := api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock0))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+			_, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[1].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock1))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock2))
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[2].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock3))
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock1))
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[4].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+
+			bal, err = api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct1BalanceBlock5))
+			bal, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedAcct2BalanceBlock4))
+			bal, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedContractBalance))
+			bal, err = api.GetBalance(ctx, test_helpers.TestBankAddress, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(bal).To(Equal(expectedBankBalanceBlock2))
+		})
+		It("Throws an error for an account it cannot find the balance for an account at the provided block number", func() {
+			_, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("Throws an error for an account it cannot find the balance for an account at the provided block hash", func() {
+			_, err := api.GetBalance(ctx, test_helpers.Account1Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = api.GetBalance(ctx, test_helpers.Account2Addr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+			_, err = api.GetBalance(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[0].Hash(), true))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+	})
+
+	Describe("eth_getCode", func() {
+		It("Retrieves the code for the provided contract address at the block with the provided number", func() {
+			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+
+			code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+		})
+		It("Retrieves the code for the provided contract address at the block with the provided hash", func() {
+			code, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+
+			code, err = api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithHash(blocks[5].Hash(), true))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(code).To(Equal((hexutil.Bytes)(test_helpers.ContractCode)))
+		})
+		It("Throws an error for an account it cannot find the code for", func() {
+			_, err := api.GetCode(ctx, randomAddr, rpc.BlockNumberOrHashWithHash(blocks[3].Hash(), true))
+			Expect(err).To(HaveOccurred())
+		})
+		It("Throws an error for a contract that doesn't exist at this hieght", func() {
+			_, err := api.GetCode(ctx, test_helpers.ContractAddr, rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("eth_getStorageAt", func() {
+		It("Throws an error if it tries to access a contract which does not exist", func() {
+			_, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(0))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+
+			_, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(1))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("Throws an error if it tries to access a contract slot which does not exist", func() {
+			_, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, randomHash.Hex(), rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("sql: no rows in result set"))
+		})
+		It("Retrieves the storage value at the provided contract address and storage leaf key at the block with the provided hash or number", func() {
+			// After deployment
+			val, err := api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(2))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes := hexutil.Bytes(common.Hex2Bytes("01"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(3))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("03"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(4))
+			Expect(err).ToNot(HaveOccurred())
+			expectedRes = hexutil.Bytes(common.Hex2Bytes("09"))
+			Expect(val).To(Equal(expectedRes))
+
+			val, err = api.GetStorageAt(ctx, test_helpers.ContractAddr, test_helpers.ContractSlotKeyHash.Hex(), rpc.BlockNumberOrHashWithNumber(5))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(val).To(Equal(hexutil.Bytes{}))
+		})
+	})
+
+	Describe("eth_getHeaderByNumber", func() {
+		It("Finds the canonical header based on the header's weight relative to others at the provided height", func() {
+			header, err := api.GetHeaderByNumber(ctx, number)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(header).To(Equal(expectedCanonicalHeader))
+		})
+	})
+})

pkg/eth/filterer.go

@@ -19,11 +19,12 @@ package eth
 import (
 	"bytes"
 
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/statediff"
 	"github.com/multiformats/go-multihash"
 
 	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
@@ -33,7 +34,7 @@ import (
 
 // Filterer interface for substituing mocks in tests
 type Filterer interface {
-	Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*eth.IPLDs, error)
+	Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error)
 }
 
 // ResponseFilterer satisfies the ResponseFilterer interface for ethereum
@@ -45,9 +46,9 @@ func NewResponseFilterer() *ResponseFilterer {
 }
 
 // Filter is used to filter through eth data to extract and package requested data into a Payload
-func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*eth.IPLDs, error) {
+func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error) {
 	if checkRange(filter.Start.Int64(), filter.End.Int64(), payload.Block.Number().Int64()) {
-		response := new(eth.IPLDs)
+		response := new(IPLDs)
 		response.TotalDifficulty = payload.TotalDifficulty
 		if err := s.filterHeaders(filter.HeaderFilter, response, payload); err != nil {
 			return nil, err
@@ -72,7 +73,7 @@ func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.Conve
 	return nil, nil
 }
 
-func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *eth.IPLDs, payload eth.ConvertedPayload) error {
+func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IPLDs, payload eth.ConvertedPayload) error {
 	if !headerFilter.Off {
 		headerRLP, err := rlp.EncodeToBytes(payload.Block.Header())
 		if err != nil {
@@ -114,7 +115,7 @@ func checkRange(start, end, actual int64) bool {
 	return false
 }
 
-func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *eth.IPLDs, payload eth.ConvertedPayload) ([]common.Hash, error) {
+func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload eth.ConvertedPayload) ([]common.Hash, error) {
 	var trxHashes []common.Hash
 	if !trxFilter.Off {
 		trxLen := len(payload.Block.Body().Transactions)
@@ -162,7 +163,7 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s
 	return false
 }
 
-func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *eth.IPLDs, payload eth.ConvertedPayload, trxHashes []common.Hash) error {
+func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload eth.ConvertedPayload, trxHashes []common.Hash) error {
 	if !receiptFilter.Off {
 		response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
 		for i, receipt := range payload.Receipts {
@@ -252,9 +253,9 @@ func slicesShareString(slice1, slice2 []string) int {
 }
 
 // filterStateAndStorage filters state and storage nodes into the response according to the provided filters
-func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *eth.IPLDs, payload eth.ConvertedPayload) error {
-	response.StateNodes = make([]eth.StateNode, 0, len(payload.StateNodes))
-	response.StorageNodes = make([]eth.StorageNode, 0)
+func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload eth.ConvertedPayload) error {
+	response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
+	response.StorageNodes = make([]StorageNode, 0)
 	stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses))
 	for i, addr := range stateFilter.Addresses {
 		stateAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
@@ -269,12 +270,12 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
 	}
 	for _, stateNode := range payload.StateNodes {
 		if !stateFilter.Off && checkNodeKeys(stateAddressFilters, stateNode.LeafKey) {
-			if stateNode.Type == statediff.Leaf || stateFilter.IntermediateNodes {
+			if stateNode.Type == sdtypes.Leaf || stateFilter.IntermediateNodes {
 				cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
 				if err != nil {
 					return err
 				}
-				response.StateNodes = append(response.StateNodes, eth.StateNode{
+				response.StateNodes = append(response.StateNodes, StateNode{
 					StateLeafKey: stateNode.LeafKey,
 					Path:         stateNode.Path,
 					IPLD: ipfs.BlockModel{
@@ -292,7 +293,7 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
 					if err != nil {
 						return err
 					}
-					response.StorageNodes = append(response.StorageNodes, eth.StorageNode{
+					response.StorageNodes = append(response.StorageNodes, StorageNode{
 						StateLeafKey:   stateNode.LeafKey,
 						StorageLeafKey: storageNode.LeafKey,
 						IPLD: ipfs.BlockModel{

pkg/eth/filterer_test.go

@@ -19,14 +19,15 @@ package eth_test
 import (
 	"bytes"
 
-	"github.com/ethereum/go-ethereum/statediff"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
 	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
@@ -41,45 +42,45 @@ var _ = Describe("Filterer", func() {
 		})
 
 		It("Transcribes all the data from the IPLDPayload into the StreamPayload if given an open filter", func() {
-			iplds, err := filterer.Filter(openFilter, mocks.MockConvertedPayload)
+			iplds, err := filterer.Filter(openFilter, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds).ToNot(BeNil())
-			Expect(iplds.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
-			Expect(iplds.Header).To(Equal(mocks.MockIPLDs.Header))
+			Expect(iplds.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
+			Expect(iplds.Header).To(Equal(test_helpers.MockIPLDs.Header))
 			var expectedEmptyUncles []ipfs.BlockModel
 			Expect(iplds.Uncles).To(Equal(expectedEmptyUncles))
 			Expect(len(iplds.Transactions)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Transactions, test_helpers.MockTransactions.GetRlp(2))).To(BeTrue())
 			Expect(len(iplds.Receipts)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.MockReceipts.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.MockReceipts.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, test_helpers.MockReceipts.GetRlp(2))).To(BeTrue())
 			Expect(len(iplds.StateNodes)).To(Equal(2))
 			for _, stateNode := range iplds.StateNodes {
-				Expect(stateNode.Type).To(Equal(statediff.Leaf))
-				if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.AccountLeafKey) {
+				Expect(stateNode.Type).To(Equal(sdtypes.Leaf))
+				if bytes.Equal(stateNode.StateLeafKey.Bytes(), test_helpers.AccountLeafKey) {
 					Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
-						Data: mocks.State2IPLD.RawData(),
-						CID:  mocks.State2IPLD.Cid().String(),
+						Data: test_helpers.State2IPLD.RawData(),
+						CID:  test_helpers.State2IPLD.Cid().String(),
 					}))
 				}
-				if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.ContractLeafKey) {
+				if bytes.Equal(stateNode.StateLeafKey.Bytes(), test_helpers.ContractLeafKey) {
 					Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
-						Data: mocks.State1IPLD.RawData(),
-						CID:  mocks.State1IPLD.Cid().String(),
+						Data: test_helpers.State1IPLD.RawData(),
+						CID:  test_helpers.State1IPLD.Cid().String(),
 					}))
 				}
 			}
-			Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
+			Expect(iplds.StorageNodes).To(Equal(test_helpers.MockIPLDs.StorageNodes))
 		})
 
 		It("Applies filters from the provided config.Subscription", func() {
-			iplds1, err := filterer.Filter(rctAddressFilter, mocks.MockConvertedPayload)
+			iplds1, err := filterer.Filter(rctAddressFilter, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds1).ToNot(BeNil())
-			Expect(iplds1.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
+			Expect(iplds1.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds1.Header).To(Equal(ipfs.BlockModel{}))
 			Expect(len(iplds1.Uncles)).To(Equal(0))
 			Expect(len(iplds1.Transactions)).To(Equal(0))
@@ -87,14 +88,14 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds1.StateNodes)).To(Equal(0))
 			Expect(len(iplds1.Receipts)).To(Equal(1))
 			Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct1IPLD.RawData(),
-				CID:  mocks.Rct1IPLD.Cid().String(),
+				Data: test_helpers.Rct1IPLD.RawData(),
+				CID:  test_helpers.Rct1IPLD.Cid().String(),
 			}))
 
-			iplds2, err := filterer.Filter(rctTopicsFilter, mocks.MockConvertedPayload)
+			iplds2, err := filterer.Filter(rctTopicsFilter, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds2).ToNot(BeNil())
-			Expect(iplds2.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
+			Expect(iplds2.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds2.Header).To(Equal(ipfs.BlockModel{}))
 			Expect(len(iplds2.Uncles)).To(Equal(0))
 			Expect(len(iplds2.Transactions)).To(Equal(0))
@@ -102,14 +103,14 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds2.StateNodes)).To(Equal(0))
 			Expect(len(iplds2.Receipts)).To(Equal(1))
 			Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct1IPLD.RawData(),
-				CID:  mocks.Rct1IPLD.Cid().String(),
+				Data: test_helpers.Rct1IPLD.RawData(),
+				CID:  test_helpers.Rct1IPLD.Cid().String(),
 			}))
 
-			iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, mocks.MockConvertedPayload)
+			iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds3).ToNot(BeNil())
-			Expect(iplds3.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
+			Expect(iplds3.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds3.Header).To(Equal(ipfs.BlockModel{}))
 			Expect(len(iplds3.Uncles)).To(Equal(0))
 			Expect(len(iplds3.Transactions)).To(Equal(0))
@@ -117,14 +118,14 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds3.StateNodes)).To(Equal(0))
 			Expect(len(iplds3.Receipts)).To(Equal(1))
 			Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct1IPLD.RawData(),
-				CID:  mocks.Rct1IPLD.Cid().String(),
+				Data: test_helpers.Rct1IPLD.RawData(),
+				CID:  test_helpers.Rct1IPLD.Cid().String(),
 			}))
 
-			iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, mocks.MockConvertedPayload)
+			iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds4).ToNot(BeNil())
-			Expect(iplds4.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
+			Expect(iplds4.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds4.Header).To(Equal(ipfs.BlockModel{}))
 			Expect(len(iplds4.Uncles)).To(Equal(0))
 			Expect(len(iplds4.Transactions)).To(Equal(0))
@@ -132,63 +133,63 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds4.StateNodes)).To(Equal(0))
 			Expect(len(iplds4.Receipts)).To(Equal(1))
 			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct2IPLD.RawData(),
-				CID:  mocks.Rct2IPLD.Cid().String(),
+				Data: test_helpers.Rct2IPLD.RawData(),
+				CID:  test_helpers.Rct2IPLD.Cid().String(),
 			}))
 
-			iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, mocks.MockConvertedPayload)
+			iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds5).ToNot(BeNil())
-			Expect(iplds5.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
+			Expect(iplds5.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds5.Header).To(Equal(ipfs.BlockModel{}))
 			Expect(len(iplds5.Uncles)).To(Equal(0))
 			Expect(len(iplds5.Transactions)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(2))).To(BeTrue())
 			Expect(len(iplds5.StorageNodes)).To(Equal(0))
 			Expect(len(iplds5.StateNodes)).To(Equal(0))
 			Expect(len(iplds5.Receipts)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.MockReceipts.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.MockReceipts.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, test_helpers.MockReceipts.GetRlp(2))).To(BeTrue())
 
-			iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, mocks.MockConvertedPayload)
+			iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds6).ToNot(BeNil())
-			Expect(iplds6.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
+			Expect(iplds6.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds6.Header).To(Equal(ipfs.BlockModel{}))
 			Expect(len(iplds6.Uncles)).To(Equal(0))
 			Expect(len(iplds6.Transactions)).To(Equal(1))
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, test_helpers.MockTransactions.GetRlp(1))).To(BeTrue())
 			Expect(len(iplds6.StorageNodes)).To(Equal(0))
 			Expect(len(iplds6.StateNodes)).To(Equal(0))
 			Expect(len(iplds6.Receipts)).To(Equal(1))
 			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct2IPLD.RawData(),
-				CID:  mocks.Rct2IPLD.Cid().String(),
+				Data: test_helpers.Rct2IPLD.RawData(),
+				CID:  test_helpers.Rct2IPLD.Cid().String(),
 			}))
 
-			iplds7, err := filterer.Filter(stateFilter, mocks.MockConvertedPayload)
+			iplds7, err := filterer.Filter(stateFilter, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds7).ToNot(BeNil())
-			Expect(iplds7.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
+			Expect(iplds7.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds7.Header).To(Equal(ipfs.BlockModel{}))
 			Expect(len(iplds7.Uncles)).To(Equal(0))
 			Expect(len(iplds7.Transactions)).To(Equal(0))
 			Expect(len(iplds7.StorageNodes)).To(Equal(0))
 			Expect(len(iplds7.Receipts)).To(Equal(0))
 			Expect(len(iplds7.StateNodes)).To(Equal(1))
-			Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(mocks.AccountLeafKey))
+			Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(test_helpers.AccountLeafKey))
 			Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
-				Data: mocks.State2IPLD.RawData(),
-				CID:  mocks.State2IPLD.Cid().String(),
+				Data: test_helpers.State2IPLD.RawData(),
+				CID:  test_helpers.State2IPLD.Cid().String(),
 			}))
 
-			iplds8, err := filterer.Filter(rctTopicsAndAddressFilterFail, mocks.MockConvertedPayload)
+			iplds8, err := filterer.Filter(rctTopicsAndAddressFilterFail, test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds8).ToNot(BeNil())
-			Expect(iplds8.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
+			Expect(iplds8.BlockNumber.Int64()).To(Equal(test_helpers.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds8.Header).To(Equal(ipfs.BlockModel{}))
 			Expect(len(iplds8.Uncles)).To(Equal(0))
 			Expect(len(iplds8.Transactions)).To(Equal(0))

pkg/eth/helpers.go

@@ -17,40 +17,52 @@
 package eth
 
 import (
+	"encoding/json"
 	"fmt"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	log "github.com/sirupsen/logrus"
+	"os"
+
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
 
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/statediff"
 )
 
-func ResolveFromNodeType(nodeType statediff.NodeType) int {
+func ResolveToNodeType(nodeType int) sdtypes.NodeType {
 	switch nodeType {
-	case statediff.Branch:
-		return 0
-	case statediff.Extension:
-		return 1
-	case statediff.Leaf:
-		return 2
-	case statediff.Removed:
-		return 3
+	case 0:
+		return sdtypes.Branch
+	case 1:
+		return sdtypes.Extension
+	case 2:
+		return sdtypes.Leaf
+	case 3:
+		return sdtypes.Removed
 	default:
-		return -1
+		return sdtypes.Unknown
 	}
 }
 
-func ResolveToNodeType(nodeType int) statediff.NodeType {
-	switch nodeType {
-	case 0:
-		return statediff.Branch
-	case 1:
-		return statediff.Extension
-	case 2:
-		return statediff.Leaf
-	case 3:
-		return statediff.Removed
-	default:
-		return statediff.Unknown
+// LoadConfig loads chain config from json file
+func LoadConfig(chainConfigPath string) (*params.ChainConfig, error) {
+	file, err := os.Open(chainConfigPath)
+	if err != nil {
+		utils.Fatalf("Failed to read chain config file: %v", err)
+
+		return nil, err
 	}
+	defer file.Close()
+
+	chainConfig := new(params.ChainConfig)
+	if err := json.NewDecoder(file).Decode(chainConfig); err != nil {
+		utils.Fatalf("invalid chain config file: %v", err)
+
+		return nil, err
+	}
+
+	log.Infof("Using chain config from %s file. Content %+v", chainConfigPath, chainConfig)
+
+	return chainConfig, nil
 }
 
 // ChainConfig returns the appropriate ethereum chain config for the provided chain id
@@ -59,7 +71,7 @@ func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
 	case 1:
 		return params.MainnetChainConfig, nil
 	case 3:
-		return params.TestnetChainConfig, nil // Ropsten
+		return params.RopstenChainConfig, nil
 	case 4:
 		return params.RinkebyChainConfig, nil
 	case 5:

pkg/eth/interfaces.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/bloombits"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// FilterBackend is the geth interface we need to satisfy to use their filters
+type FilterBackend interface {
+	ChainDb() ethdb.Database
+	HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error)
+	HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error)
+	GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
+	GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error)
+
+	SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
+	SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
+	SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
+	SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription
+	SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription
+
+	BloomStatus() (uint64, uint64)
+	ServiceFilter(ctx context.Context, session *bloombits.MatcherSession)
+}

pkg/eth/ipld_fetcher.go

@@ -34,7 +34,7 @@ import (
 
 // Fetcher interface for substituting mocks in tests
 type Fetcher interface {
-	Fetch(cids eth.CIDWrapper) (*eth.IPLDs, error)
+	Fetch(cids CIDWrapper) (*IPLDs, error)
 }
 
 // IPLDFetcher satisfies the IPLDFetcher interface for ethereum
@@ -51,9 +51,9 @@ func NewIPLDFetcher(db *postgres.DB) *IPLDFetcher {
 }
 
 // Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
-func (f *IPLDFetcher) Fetch(cids eth.CIDWrapper) (*eth.IPLDs, error) {
+func (f *IPLDFetcher) Fetch(cids CIDWrapper) (*IPLDs, error) {
 	log.Debug("fetching iplds")
-	iplds := new(eth.IPLDs)
+	iplds := new(IPLDs)
 	var ok bool
 	iplds.TotalDifficulty, ok = new(big.Int).SetString(cids.Header.TotalDifficulty, 10)
 	if !ok {
@@ -168,9 +168,9 @@ func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []eth.ReceiptModel) ([]ipfs.Bl
 }
 
 // FetchState fetches state nodes
-func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]eth.StateNode, error) {
+func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]StateNode, error) {
 	log.Debug("fetching state iplds")
-	stateNodes := make([]eth.StateNode, 0, len(cids))
+	stateNodes := make([]StateNode, 0, len(cids))
 	for _, stateNode := range cids {
 		if stateNode.CID == "" {
 			continue
@@ -179,7 +179,7 @@ func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]eth.
 		if err != nil {
 			return nil, err
 		}
-		stateNodes = append(stateNodes, eth.StateNode{
+		stateNodes = append(stateNodes, StateNode{
 			IPLD: ipfs.BlockModel{
 				Data: stateBytes,
 				CID:  stateNode.CID,
@@ -193,9 +193,9 @@ func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]eth.
 }
 
 // FetchStorage fetches storage nodes
-func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateKeyModel) ([]eth.StorageNode, error) {
+func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateKeyModel) ([]StorageNode, error) {
 	log.Debug("fetching storage iplds")
-	storageNodes := make([]eth.StorageNode, 0, len(cids))
+	storageNodes := make([]StorageNode, 0, len(cids))
 	for _, storageNode := range cids {
 		if storageNode.CID == "" || storageNode.StateKey == "" {
 			continue
@@ -204,7 +204,7 @@ func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateK
 		if err != nil {
 			return nil, err
 		}
-		storageNodes = append(storageNodes, eth.StorageNode{
+		storageNodes = append(storageNodes, StorageNode{
 			IPLD: ipfs.BlockModel{
 				Data: storageBytes,
 				CID:  storageNode.CID,

pkg/eth/ipld_fetcher_test.go

@@ -19,29 +19,28 @@ package eth_test
 import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 
 	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
-)
-
-var (
-	db            *postgres.DB
-	pubAndIndexer *eth2.IPLDPublisher
-	fetcher       *eth.IPLDFetcher
+	"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
 )
 
 var _ = Describe("IPLDFetcher", func() {
+	var (
+		db            *postgres.DB
+		pubAndIndexer *eth2.IPLDPublisher
+		fetcher       *eth.IPLDFetcher
+	)
 	Describe("Fetch", func() {
 		BeforeEach(func() {
 			var err error
 			db, err = shared.SetupDB()
 			Expect(err).ToNot(HaveOccurred())
 			pubAndIndexer = eth2.NewIPLDPublisher(db)
-			err = pubAndIndexer.Publish(mocks.MockConvertedPayload)
+			err = pubAndIndexer.Publish(test_helpers.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			fetcher = eth.NewIPLDFetcher(db)
 		})
@@ -50,17 +49,17 @@ var _ = Describe("IPLDFetcher", func() {
 		})
 
 		It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {
-			iplds, err := fetcher.Fetch(*mocks.MockCIDWrapper)
+			iplds, err := fetcher.Fetch(*test_helpers.MockCIDWrapper)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds).ToNot(BeNil())
-			Expect(iplds.TotalDifficulty).To(Equal(mocks.MockConvertedPayload.TotalDifficulty))
-			Expect(iplds.BlockNumber).To(Equal(mocks.MockConvertedPayload.Block.Number()))
-			Expect(iplds.Header).To(Equal(mocks.MockIPLDs.Header))
+			Expect(iplds.TotalDifficulty).To(Equal(test_helpers.MockConvertedPayload.TotalDifficulty))
+			Expect(iplds.BlockNumber).To(Equal(test_helpers.MockConvertedPayload.Block.Number()))
+			Expect(iplds.Header).To(Equal(test_helpers.MockIPLDs.Header))
 			Expect(len(iplds.Uncles)).To(Equal(0))
-			Expect(iplds.Transactions).To(Equal(mocks.MockIPLDs.Transactions))
-			Expect(iplds.Receipts).To(Equal(mocks.MockIPLDs.Receipts))
-			Expect(iplds.StateNodes).To(Equal(mocks.MockIPLDs.StateNodes))
-			Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
+			Expect(iplds.Transactions).To(Equal(test_helpers.MockIPLDs.Transactions))
+			Expect(iplds.Receipts).To(Equal(test_helpers.MockIPLDs.Receipts))
+			Expect(iplds.StateNodes).To(Equal(test_helpers.MockIPLDs.StateNodes))
+			Expect(iplds.StorageNodes).To(Equal(test_helpers.MockIPLDs.StorageNodes))
 		})
 	})
 })

pkg/eth/ipld_retriever.go

@@ -0,0 +1,471 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/lib/pq"
+
+	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
+)
+
+const (
+	RetrieveHeadersByHashesPgStr = `SELECT cid, data
+								FROM eth.header_cids 
+									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
+								WHERE block_hash = ANY($1::VARCHAR(66)[])`
+	RetrieveHeadersByBlockNumberPgStr = `SELECT cid, data
+								FROM eth.header_cids 
+									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
+								WHERE block_number = $1`
+	RetrieveHeaderByHashPgStr = `SELECT cid, data
+								FROM eth.header_cids 
+									INNER JOIN public.blocks ON (header_cids.mh_key = blocks.key)
+								WHERE block_hash = $1`
+	RetrieveUnclesByHashesPgStr = `SELECT cid, data
+								FROM eth.uncle_cids
+									INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
+								WHERE block_hash = ANY($1::VARCHAR(66)[])`
+	RetrieveUnclesByBlockHashPgStr = `SELECT uncle_cids.cid, data
+										FROM eth.uncle_cids
+											INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id)
+											INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
+										WHERE block_hash = $1`
+	RetrieveUnclesByBlockNumberPgStr = `SELECT uncle_cids.cid, data
+										FROM eth.uncle_cids
+											INNER JOIN eth.header_cids ON (uncle_cids.header_id = header_cids.id)
+											INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
+										WHERE block_number = $1`
+	RetrieveUncleByHashPgStr = `SELECT cid, data
+								FROM eth.uncle_cids
+									INNER JOIN public.blocks ON (uncle_cids.mh_key = blocks.key)
+								WHERE block_hash = $1`
+	RetrieveTransactionsByHashesPgStr = `SELECT cid, data
+									FROM eth.transaction_cids
+										INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
+									WHERE tx_hash = ANY($1::VARCHAR(66)[])`
+	RetrieveTransactionsByBlockHashPgStr = `SELECT transaction_cids.cid, data
+											FROM eth.transaction_cids
+												INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
+												INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
+											WHERE block_hash = $1
+											ORDER BY eth.transaction_cids.index ASC`
+	RetrieveTransactionsByBlockNumberPgStr = `SELECT transaction_cids.cid, data
+											FROM eth.transaction_cids
+												INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
+												INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
+											WHERE block_number = $1
+											ORDER BY eth.transaction_cids.index ASC`
+	RetrieveTransactionByHashPgStr = `SELECT cid, data
+									FROM eth.transaction_cids
+										INNER JOIN public.blocks ON (transaction_cids.mh_key = blocks.key)
+									WHERE tx_hash = $1`
+	RetrieveReceiptsByTxHashesPgStr = `SELECT receipt_cids.cid, data
+									FROM eth.receipt_cids
+										INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
+										INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+									WHERE tx_hash = ANY($1::VARCHAR(66)[])`
+	RetrieveReceiptsByBlockHashPgStr = `SELECT receipt_cids.cid, data
+										FROM eth.receipt_cids
+											INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
+											INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
+											INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+										WHERE block_hash = $1
+										ORDER BY eth.transaction_cids.index ASC`
+	RetrieveReceiptsByBlockNumberPgStr = `SELECT receipt_cids.cid, data
+										FROM eth.receipt_cids
+											INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
+											INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
+											INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+										WHERE block_number = $1
+										ORDER BY eth.transaction_cids.index ASC`
+	RetrieveReceiptByTxHashPgStr = `SELECT receipt_cids.cid, data
+									FROM eth.receipt_cids
+										INNER JOIN eth.transaction_cids ON (receipt_cids.tx_id = transaction_cids.id)
+										INNER JOIN public.blocks ON (receipt_cids.mh_key = blocks.key)
+									WHERE tx_hash = $1`
+	RetrieveAccountByLeafKeyAndBlockHashPgStr = `SELECT state_cids.cid,
+														data,
+														was_state_removed(state_path, block_number, $2) AS removed
+												FROM eth.state_cids
+													INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+													INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key)
+												WHERE state_leaf_key = $1
+												AND block_number <= (SELECT block_number
+																	FROM eth.header_cids
+																	WHERE block_hash = $2)
+												AND header_cids.id = (SELECT canonical_header_id(block_number))
+												ORDER BY block_number DESC
+												LIMIT 1`
+	RetrieveAccountByLeafKeyAndBlockNumberPgStr = `SELECT state_cids.cid,
+														  data,
+														  was_state_removed(state_path, block_number, (SELECT block_hash
+																									  FROM eth.header_cids
+																									  WHERE block_number = $2
+																									  LIMIT 1)) AS removed
+													FROM eth.state_cids
+														INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+														INNER JOIN public.blocks ON (state_cids.mh_key = blocks.key)
+													WHERE state_leaf_key = $1
+													AND block_number <= $2
+													ORDER BY block_number DESC
+													LIMIT 1`
+	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr = `SELECT storage_cids.cid,
+																			data,
+																			was_storage_removed(storage_path, block_number, (SELECT block_hash
+																															FROM eth.header_cids
+																															WHERE block_number = $3
+																															LIMIT 1)) AS removed
+																	FROM eth.storage_cids
+																		INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
+																		INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+																		INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key)
+																	WHERE state_leaf_key = $1
+																	AND storage_leaf_key = $2
+																	AND block_number <= $3
+																	ORDER BY block_number DESC
+																	LIMIT 1`
+	RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr = `SELECT storage_cids.cid,
+																		  data,
+																		  was_storage_removed(storage_path, block_number, $3) AS removed
+																	FROM eth.storage_cids
+																		INNER JOIN eth.state_cids ON (storage_cids.state_id = state_cids.id)
+																		INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
+																		INNER JOIN public.blocks ON (storage_cids.mh_key = blocks.key)
+																	WHERE state_leaf_key = $1
+																	AND storage_leaf_key = $2
+																	AND block_number <= (SELECT block_number
+																						FROM eth.header_cids
+																						WHERE block_hash = $3)
+																	AND header_cids.id = (SELECT canonical_header_id(block_number))
+																	ORDER BY block_number DESC
+																	LIMIT 1`
+)
+
+type ipldResult struct {
+	CID  string `db:"cid"`
+	Data []byte `db:"data"`
+}
+type IPLDRetriever struct {
+	db *postgres.DB
+}
+
+func NewIPLDRetriever(db *postgres.DB) *IPLDRetriever {
+	return &IPLDRetriever{
+		db: db,
+	}
+}
+
+// RetrieveHeadersByHashes returns the cids and rlp bytes for the headers corresponding to the provided block hashes
+func (r *IPLDRetriever) RetrieveHeadersByHashes(hashes []common.Hash) ([]string, [][]byte, error) {
+	headerResults := make([]ipldResult, 0)
+	hashStrs := make([]string, len(hashes))
+	for i, hash := range hashes {
+		hashStrs[i] = hash.Hex()
+	}
+	if err := r.db.Select(&headerResults, RetrieveHeadersByHashesPgStr, pq.Array(hashStrs)); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(headerResults))
+	headers := make([][]byte, len(headerResults))
+	for i, res := range headerResults {
+		cids[i] = res.CID
+		headers[i] = res.Data
+	}
+	return cids, headers, nil
+}
+
+// RetrieveHeadersByBlockNumber returns the cids and rlp bytes for the headers corresponding to the provided block number
+// This can return more than one result since there can be more than one header (non-canonical headers)
+func (r *IPLDRetriever) RetrieveHeadersByBlockNumber(number uint64) ([]string, [][]byte, error) {
+	headerResults := make([]ipldResult, 0)
+	if err := r.db.Select(&headerResults, RetrieveHeadersByBlockNumberPgStr, number); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(headerResults))
+	headers := make([][]byte, len(headerResults))
+	for i, res := range headerResults {
+		cids[i] = res.CID
+		headers[i] = res.Data
+	}
+	return cids, headers, nil
+}
+
+// RetrieveHeaderByHash returns the cid and rlp bytes for the header corresponding to the provided block hash
+func (r *IPLDRetriever) RetrieveHeaderByHash(hash common.Hash) (string, []byte, error) {
+	headerResult := new(ipldResult)
+	return headerResult.CID, headerResult.Data, r.db.Get(headerResult, RetrieveHeaderByHashPgStr, hash.Hex())
+}
+
+// RetrieveUnclesByHashes returns the cids and rlp bytes for the uncles corresponding to the provided uncle hashes
+func (r *IPLDRetriever) RetrieveUnclesByHashes(hashes []common.Hash) ([]string, [][]byte, error) {
+	uncleResults := make([]ipldResult, 0)
+	hashStrs := make([]string, len(hashes))
+	for i, hash := range hashes {
+		hashStrs[i] = hash.Hex()
+	}
+	if err := r.db.Select(&uncleResults, RetrieveUnclesByHashesPgStr, pq.Array(hashStrs)); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(uncleResults))
+	uncles := make([][]byte, len(uncleResults))
+	for i, res := range uncleResults {
+		cids[i] = res.CID
+		uncles[i] = res.Data
+	}
+	return cids, uncles, nil
+}
+
+// RetrieveUnclesByBlockHash returns the cids and rlp bytes for the uncles corresponding to the provided block hash (of non-omner root block)
+func (r *IPLDRetriever) RetrieveUnclesByBlockHash(hash common.Hash) ([]string, [][]byte, error) {
+	uncleResults := make([]ipldResult, 0)
+	if err := r.db.Select(&uncleResults, RetrieveUnclesByBlockHashPgStr, hash.Hex()); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(uncleResults))
+	uncles := make([][]byte, len(uncleResults))
+	for i, res := range uncleResults {
+		cids[i] = res.CID
+		uncles[i] = res.Data
+	}
+	return cids, uncles, nil
+}
+
+// RetrieveUnclesByBlockNumber returns the cids and rlp bytes for the uncles corresponding to the provided block number (of non-omner root block)
+func (r *IPLDRetriever) RetrieveUnclesByBlockNumber(number uint64) ([]string, [][]byte, error) {
+	uncleResults := make([]ipldResult, 0)
+	if err := r.db.Select(&uncleResults, RetrieveUnclesByBlockNumberPgStr, number); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(uncleResults))
+	uncles := make([][]byte, len(uncleResults))
+	for i, res := range uncleResults {
+		cids[i] = res.CID
+		uncles[i] = res.Data
+	}
+	return cids, uncles, nil
+}
+
+// RetrieveUncleByHash returns the cid and rlp bytes for the uncle corresponding to the provided uncle hash
+func (r *IPLDRetriever) RetrieveUncleByHash(hash common.Hash) (string, []byte, error) {
+	uncleResult := new(ipldResult)
+	return uncleResult.CID, uncleResult.Data, r.db.Get(uncleResult, RetrieveUncleByHashPgStr, hash.Hex())
+}
+
+// RetrieveTransactionsByHashes returns the cids and rlp bytes for the transactions corresponding to the provided tx hashes
+func (r *IPLDRetriever) RetrieveTransactionsByHashes(hashes []common.Hash) ([]string, [][]byte, error) {
+	txResults := make([]ipldResult, 0)
+	hashStrs := make([]string, len(hashes))
+	for i, hash := range hashes {
+		hashStrs[i] = hash.Hex()
+	}
+	if err := r.db.Select(&txResults, RetrieveTransactionsByHashesPgStr, pq.Array(hashStrs)); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(txResults))
+	txs := make([][]byte, len(txResults))
+	for i, res := range txResults {
+		cids[i] = res.CID
+		txs[i] = res.Data
+	}
+	return cids, txs, nil
+}
+
+// RetrieveTransactionsByBlockHash returns the cids and rlp bytes for the transactions corresponding to the provided block hash
+func (r *IPLDRetriever) RetrieveTransactionsByBlockHash(hash common.Hash) ([]string, [][]byte, error) {
+	txResults := make([]ipldResult, 0)
+	if err := r.db.Select(&txResults, RetrieveTransactionsByBlockHashPgStr, hash.Hex()); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(txResults))
+	txs := make([][]byte, len(txResults))
+	for i, res := range txResults {
+		cids[i] = res.CID
+		txs[i] = res.Data
+	}
+	return cids, txs, nil
+}
+
+// RetrieveTransactionsByBlockNumber returns the cids and rlp bytes for the transactions corresponding to the provided block number
+func (r *IPLDRetriever) RetrieveTransactionsByBlockNumber(number uint64) ([]string, [][]byte, error) {
+	txResults := make([]ipldResult, 0)
+	if err := r.db.Select(&txResults, RetrieveTransactionsByBlockNumberPgStr, number); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(txResults))
+	txs := make([][]byte, len(txResults))
+	for i, res := range txResults {
+		cids[i] = res.CID
+		txs[i] = res.Data
+	}
+	return cids, txs, nil
+}
+
+// RetrieveTransactionByTxHash returns the cid and rlp bytes for the transaction corresponding to the provided tx hash
+func (r *IPLDRetriever) RetrieveTransactionByTxHash(hash common.Hash) (string, []byte, error) {
+	txResult := new(ipldResult)
+	return txResult.CID, txResult.Data, r.db.Get(txResult, RetrieveTransactionByHashPgStr, hash.Hex())
+}
+
+// RetrieveReceiptsByTxHashes returns the cids and rlp bytes for the receipts corresponding to the provided tx hashes
+func (r *IPLDRetriever) RetrieveReceiptsByTxHashes(hashes []common.Hash) ([]string, [][]byte, error) {
+	rctResults := make([]ipldResult, 0)
+	hashStrs := make([]string, len(hashes))
+	for i, hash := range hashes {
+		hashStrs[i] = hash.Hex()
+	}
+	if err := r.db.Select(&rctResults, RetrieveReceiptsByTxHashesPgStr, pq.Array(hashStrs)); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(rctResults))
+	rcts := make([][]byte, len(rctResults))
+	for i, res := range rctResults {
+		cids[i] = res.CID
+		rcts[i] = res.Data
+	}
+	return cids, rcts, nil
+}
+
+// RetrieveReceiptsByBlockHash returns the cids and rlp bytes for the receipts corresponding to the provided block hash
+func (r *IPLDRetriever) RetrieveReceiptsByBlockHash(hash common.Hash) ([]string, [][]byte, error) {
+	rctResults := make([]ipldResult, 0)
+	if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockHashPgStr, hash.Hex()); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(rctResults))
+	rcts := make([][]byte, len(rctResults))
+	for i, res := range rctResults {
+		cids[i] = res.CID
+		rcts[i] = res.Data
+	}
+	return cids, rcts, nil
+}
+
+// RetrieveReceiptsByBlockNumber returns the cids and rlp bytes for the receipts corresponding to the provided block hash
+func (r *IPLDRetriever) RetrieveReceiptsByBlockNumber(number uint64) ([]string, [][]byte, error) {
+	rctResults := make([]ipldResult, 0)
+	if err := r.db.Select(&rctResults, RetrieveReceiptsByBlockNumberPgStr, number); err != nil {
+		return nil, nil, err
+	}
+	cids := make([]string, len(rctResults))
+	rcts := make([][]byte, len(rctResults))
+	for i, res := range rctResults {
+		cids[i] = res.CID
+		rcts[i] = res.Data
+	}
+	return cids, rcts, nil
+}
+
+// RetrieveReceiptByHash returns the cid and rlp bytes for the receipt corresponding to the provided tx hash
+func (r *IPLDRetriever) RetrieveReceiptByHash(hash common.Hash) (string, []byte, error) {
+	rctResult := new(ipldResult)
+	return rctResult.CID, rctResult.Data, r.db.Get(rctResult, RetrieveReceiptByTxHashPgStr, hash.Hex())
+}
+
+type nodeInfo struct {
+	CID     string `db:"cid"`
+	Data    []byte `db:"data"`
+	Removed bool   `db:"removed"`
+}
+
+// RetrieveAccountByAddressAndBlockHash returns the cid and rlp bytes for the account corresponding to the provided address and block hash
+// TODO: ensure this handles deleted accounts appropriately
+func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockHash(address common.Address, hash common.Hash) (string, []byte, error) {
+	accountResult := new(nodeInfo)
+	leafKey := crypto.Keccak256Hash(address.Bytes())
+	if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockHashPgStr, leafKey.Hex(), hash.Hex()); err != nil {
+		return "", nil, err
+	}
+	if accountResult.Removed {
+		return "", []byte{}, nil
+	}
+	var i []interface{}
+	if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
+		return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
+	}
+	if len(i) != 2 {
+		return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements")
+	}
+	return accountResult.CID, i[1].([]byte), nil
+}
+
+// RetrieveAccountByAddressAndBlockNumber returns the cid and rlp bytes for the account corresponding to the provided address and block number
+// This can return a non-canonical account
+func (r *IPLDRetriever) RetrieveAccountByAddressAndBlockNumber(address common.Address, number uint64) (string, []byte, error) {
+	accountResult := new(nodeInfo)
+	leafKey := crypto.Keccak256Hash(address.Bytes())
+	if err := r.db.Get(accountResult, RetrieveAccountByLeafKeyAndBlockNumberPgStr, leafKey.Hex(), number); err != nil {
+		return "", nil, err
+	}
+	if accountResult.Removed {
+		return "", []byte{}, nil
+	}
+	var i []interface{}
+	if err := rlp.DecodeBytes(accountResult.Data, &i); err != nil {
+		return "", nil, fmt.Errorf("error decoding state leaf node rlp: %s", err.Error())
+	}
+	if len(i) != 2 {
+		return "", nil, fmt.Errorf("eth IPLDRetriever expected state leaf node rlp to decode into two elements")
+	}
+	return accountResult.CID, i[1].([]byte), nil
+}
+
+// RetrieveStorageAtByAddressAndStorageKeyAndBlockHash returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block hash
+func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockHash(address common.Address, storageLeafKey, hash common.Hash) (string, []byte, error) {
+	storageResult := new(nodeInfo)
+	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
+	if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockHashPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), hash.Hex()); err != nil {
+		return "", nil, err
+	}
+	if storageResult.Removed {
+		return "", []byte{}, nil
+	}
+	var i []interface{}
+	if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {
+		err = fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())
+		return "", nil, err
+	}
+	if len(i) != 2 {
+		return "", nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements")
+	}
+	return storageResult.CID, i[1].([]byte), nil
+}
+
+// RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber returns the cid and rlp bytes for the storage value corresponding to the provided address, storage key, and block number
+// This can retrun a non-canonical value
+func (r *IPLDRetriever) RetrieveStorageAtByAddressAndStorageKeyAndBlockNumber(address common.Address, storageLeafKey common.Hash, number uint64) (string, []byte, error) {
+	storageResult := new(nodeInfo)
+	stateLeafKey := crypto.Keccak256Hash(address.Bytes())
+	if err := r.db.Get(storageResult, RetrieveStorageLeafByAddressHashAndLeafKeyAndBlockNumberPgStr, stateLeafKey.Hex(), storageLeafKey.Hex(), number); err != nil {
+		return "", nil, err
+	}
+	if storageResult.Removed {
+		return "", []byte{}, nil
+	}
+	var i []interface{}
+	if err := rlp.DecodeBytes(storageResult.Data, &i); err != nil {
+		return "", nil, fmt.Errorf("error decoding storage leaf node rlp: %s", err.Error())
+	}
+	if len(i) != 2 {
+		return "", nil, fmt.Errorf("eth IPLDRetriever expected storage leaf node rlp to decode into two elements")
+	}
+	return storageResult.CID, i[1].([]byte), nil
+}

pkg/eth/test_helpers/abi.json

@@ -0,0 +1,47 @@
+[
+  {
+    "inputs": [],
+    "payable": false,
+    "stateMutability": "nonpayable",
+    "type": "constructor"
+  },
+  {
+    "constant": false,
+    "inputs": [
+      {
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      }
+    ],
+    "name": "Put",
+    "outputs": [],
+    "payable": false,
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "constant": false,
+    "inputs": [],
+    "name": "close",
+    "outputs": [],
+    "payable": false,
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "constant": true,
+    "inputs": [],
+    "name": "data",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "payable": false,
+    "stateMutability": "view",
+    "type": "function"
+  }
+]

pkg/eth/test_helpers/chain_maker.go

@@ -0,0 +1,106 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package test_helpers
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+// Test variables
+var (
+	Testdb          = rawdb.NewMemoryDatabase()
+	TestBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7
+	TestBankFunds   = big.NewInt(100000000)
+	Genesis         = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds)
+
+	Account1Key, _       = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
+	Account2Key, _       = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
+	Account1Addr         = crypto.PubkeyToAddress(Account1Key.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
+	Account2Addr         = crypto.PubkeyToAddress(Account2Key.PublicKey) //0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e
+	DeploymentTxData     = common.Hex2Bytes("608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032")
+	ContractCode         = common.Hex2Bytes("608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032")
+	CodeHash             = crypto.Keccak256Hash(ContractCode)
+	ContractAddr         common.Address
+	IndexZero            = "0000000000000000000000000000000000000000000000000000000000000000"
+	IndexOne             = "0000000000000000000000000000000000000000000000000000000000000001"
+	ContractSlotPosition = common.FromHex(IndexOne)
+	ContractSlotKeyHash  = crypto.Keccak256Hash(ContractSlotPosition)
+	MiningReward         = big.NewInt(2000000000000000000)
+)
+
+/* test function signatures
+put function sig: 65f3c31a
+close function sig: 43d726d6
+data function sig: 73d4a13a
+*/
+
+// MakeChain creates a chain of n blocks starting at and including parent.
+// the returned hash chain is ordered head->parent.
+func MakeChain(n int, parent *types.Block, chainGen func(int, *core.BlockGen)) ([]*types.Block, []types.Receipts, *core.BlockChain) {
+	config := params.TestChainConfig
+	blocks, receipts := core.GenerateChain(config, parent, ethash.NewFaker(), Testdb, n, chainGen)
+	chain, _ := core.NewBlockChain(Testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil)
+	return append([]*types.Block{parent}, blocks...), receipts, chain
+}
+
+func TestChainGen(i int, block *core.BlockGen) {
+	signer := types.HomesteadSigner{}
+	switch i {
+	case 0:
+		// In block 1, the test bank sends account #1 some ether.
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey)
+		block.AddTx(tx)
+	case 1:
+		// In block 2, the test bank sends some more ether to account #1.
+		// Account1Addr passes it on to account #2.
+		// Account1Addr creates a test contract.
+		tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey)
+		nonce := block.TxNonce(Account1Addr)
+		tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key)
+		nonce++
+		tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), DeploymentTxData), signer, Account1Key)
+		ContractAddr = crypto.CreateAddress(Account1Addr, nonce)
+		block.AddTx(tx1)
+		block.AddTx(tx2)
+		block.AddTx(tx3)
+	case 2:
+		block.SetCoinbase(Account2Addr)
+		data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000003")
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+		block.AddTx(tx)
+	case 3:
+		block.SetCoinbase(Account2Addr)
+		data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000009")
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+		block.AddTx(tx)
+	case 4:
+		block.SetCoinbase(Account1Addr)
+		data := common.Hex2Bytes("65F3C31A0000000000000000000000000000000000000000000000000000000000000000")
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey)
+		block.AddTx(tx)
+	}
+}

pkg/eth/test_helpers/compiler_output.json

@@ -0,0 +1,6 @@
+{
+  "linkReferences": {},
+  "object": "608060405234801561001057600080fd5b50336000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550600180819055506101e2806100676000396000f3fe608060405234801561001057600080fd5b50600436106100415760003560e01c806343d726d61461004657806365f3c31a1461005057806373d4a13a1461007e575b600080fd5b61004e61009c565b005b61007c6004803603602081101561006657600080fd5b810190808035906020019092919050505061017b565b005b610086610185565b6040518082815260200191505060405180910390f35b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff1614610141576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602281526020018061018c6022913960400191505060405180910390fd5b6000809054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16ff5b8060018190555050565b6001548156fe4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6e2ea265627a7a723158205ba91466129f45285f53176d805117208c231ec6343d7896790e6fc4165b802b64736f6c63430005110032",
+  "opcodes": "PUSH1 0x80 PUSH1 0x40 MSTORE CALLVALUE DUP1 ISZERO PUSH2 0x10 JUMPI PUSH1 0x0 DUP1 REVERT JUMPDEST POP CALLER PUSH1 0x0 DUP1 PUSH2 0x100 EXP DUP2 SLOAD DUP2 PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF MUL NOT AND SWAP1 DUP4 PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND MUL OR SWAP1 SSTORE POP PUSH1 0x1 DUP1 DUP2 SWAP1 SSTORE POP PUSH2 0x1E2 DUP1 PUSH2 0x67 PUSH1 0x0 CODECOPY PUSH1 0x0 RETURN INVALID PUSH1 0x80 PUSH1 0x40 MSTORE CALLVALUE DUP1 ISZERO PUSH2 0x10 JUMPI PUSH1 0x0 DUP1 REVERT JUMPDEST POP PUSH1 0x4 CALLDATASIZE LT PUSH2 0x41 JUMPI PUSH1 0x0 CALLDATALOAD PUSH1 0xE0 SHR DUP1 PUSH4 0x43D726D6 EQ PUSH2 0x46 JUMPI DUP1 PUSH4 0x65F3C31A EQ PUSH2 0x50 JUMPI DUP1 PUSH4 0x73D4A13A EQ PUSH2 0x7E JUMPI JUMPDEST PUSH1 0x0 DUP1 REVERT JUMPDEST PUSH2 0x4E PUSH2 0x9C JUMP JUMPDEST STOP JUMPDEST PUSH2 0x7C PUSH1 0x4 DUP1 CALLDATASIZE SUB PUSH1 0x20 DUP2 LT ISZERO PUSH2 0x66 JUMPI PUSH1 0x0 DUP1 REVERT JUMPDEST DUP2 ADD SWAP1 DUP1 DUP1 CALLDATALOAD SWAP1 PUSH1 0x20 ADD SWAP1 SWAP3 SWAP2 SWAP1 POP POP POP PUSH2 0x17B JUMP JUMPDEST STOP JUMPDEST PUSH2 0x86 PUSH2 0x185 JUMP JUMPDEST PUSH1 0x40 MLOAD DUP1 DUP3 DUP2 MSTORE PUSH1 0x20 ADD SWAP2 POP POP PUSH1 0x40 MLOAD DUP1 SWAP2 SUB SWAP1 RETURN JUMPDEST PUSH1 0x0 DUP1 SWAP1 SLOAD SWAP1 PUSH2 0x100 EXP SWAP1 DIV PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND CALLER PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND EQ PUSH2 0x141 JUMPI PUSH1 0x40 MLOAD PUSH32 0x8C379A000000000000000000000000000000000000000000000000000000000 DUP2 MSTORE PUSH1 0x4 ADD DUP1 DUP1 PUSH1 0x20 ADD DUP3 DUP2 SUB DUP3 MSTORE PUSH1 0x22 DUP2 MSTORE PUSH1 0x20 ADD DUP1 PUSH2 0x18C PUSH1 0x22 SWAP2 CODECOPY PUSH1 0x40 ADD SWAP2 POP POP PUSH1 0x40 MLOAD DUP1 SWAP2 SUB SWAP1 REVERT JUMPDEST PUSH1 0x0 DUP1 SWAP1 SLOAD SWAP1 PUSH2 0x100 EXP SWAP1 DIV PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND PUSH20 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF AND SELFDESTRUCT JUMPDEST DUP1 PUSH1 0x1 DUP2 SWAP1 SSTORE POP POP JUMP JUMPDEST PUSH1 0x1 SLOAD DUP2 JUMP INVALID 0x4F PUSH15 0x6C79206F776E65722063616E206361 PUSH13 0x6C20746869732066756E637469 PUSH16 0x6E2EA265627A7A723158205BA9146612 SWAP16 GASLIMIT 0x28 0x5F MSTORE8 OR PUSH14 0x805117208C231EC6343D7896790E PUSH16 0xC4165B802B64736F6C63430005110032 ",
+  "sourceMap": "26:449:0:-;;;253:74;8:9:-1;5:2;;;30:1;27;20:12;5:2;253:74:0;292:10;284:5;;:18;;;;;;;;;;;;;;;;;;319:1;312:4;:8;;;;26:449;;;;;;"
+}

pkg/eth/test_helpers/test_contract.sol

@@ -0,0 +1,28 @@
+pragma solidity ^0.5.10;
+
+contract test {
+    address payable owner;
+
+    modifier onlyOwner {
+        require(
+            msg.sender == owner,
+            "Only owner can call this function."
+        );
+        _;
+    }
+
+    uint256 public data;
+
+    constructor() public {
+        owner = msg.sender;
+        data = 1;
+    }
+
+    function Put(uint256 value) public {
+        data = value;
+    }
+
+    function close() public onlyOwner {
+        selfdestruct(owner);
+    }
+}

pkg/eth/test_helpers/test_data.go

@@ -0,0 +1,531 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package test_helpers
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"math/big"
+
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/ethereum/go-ethereum/trie"
+
+	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff/testhelpers"
+	"github.com/ipfs/go-block-format"
+	"github.com/multiformats/go-multihash"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
+
+	eth2 "github.com/vulcanize/ipld-eth-server/pkg/eth"
+)
+
+// Test variables
+var (
+	// block data
+	BlockNumber = big.NewInt(1)
+	MockHeader  = types.Header{
+		Time:        0,
+		Number:      new(big.Int).Set(BlockNumber),
+		Root:        common.HexToHash("0x0"),
+		TxHash:      common.HexToHash("0x0"),
+		ReceiptHash: common.HexToHash("0x0"),
+		Difficulty:  big.NewInt(5000000),
+		Extra:       []byte{},
+	}
+	MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts()
+	MockUncles                                 = []*types.Header{
+		{
+			Time:        1,
+			Number:      new(big.Int).Add(BlockNumber, big.NewInt(1)),
+			Root:        common.HexToHash("0x1"),
+			TxHash:      common.HexToHash("0x1"),
+			ReceiptHash: common.HexToHash("0x1"),
+			Difficulty:  big.NewInt(500001),
+			Extra:       []byte{},
+		},
+		{
+			Time:        2,
+			Number:      new(big.Int).Add(BlockNumber, big.NewInt(2)),
+			Root:        common.HexToHash("0x2"),
+			TxHash:      common.HexToHash("0x2"),
+			ReceiptHash: common.HexToHash("0x2"),
+			Difficulty:  big.NewInt(500002),
+			Extra:       []byte{},
+		},
+	}
+	ReceiptsRlp, _   = rlp.EncodeToBytes(MockReceipts)
+	MockBlock        = types.NewBlock(&MockHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie))
+	MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header())
+	MockChildHeader  = types.Header{
+		Time:        0,
+		Number:      new(big.Int).Add(BlockNumber, common.Big1),
+		Root:        common.HexToHash("0x0"),
+		TxHash:      common.HexToHash("0x0"),
+		ReceiptHash: common.HexToHash("0x0"),
+		Difficulty:  big.NewInt(5000001),
+		Extra:       []byte{},
+		ParentHash:  MockBlock.Header().Hash(),
+	}
+	MockChild            = types.NewBlock(&MockChildHeader, MockTransactions, MockUncles, MockReceipts, new(trie.Trie))
+	MockChildRlp, _      = rlp.EncodeToBytes(MockChild.Header())
+	Address              = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
+	AnotherAddress       = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	ContractAddress      = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
+	ContractHash         = crypto.Keccak256Hash(ContractAddress.Bytes()).String()
+	MockContractByteCode = []byte{0, 1, 2, 3, 4, 5}
+	mockTopic11          = common.HexToHash("0x04")
+	mockTopic12          = common.HexToHash("0x06")
+	mockTopic21          = common.HexToHash("0x05")
+	mockTopic22          = common.HexToHash("0x07")
+	MockLog1             = &types.Log{
+		Address: Address,
+		Topics:  []common.Hash{mockTopic11, mockTopic12},
+		Data:    []byte{},
+	}
+	MockLog2 = &types.Log{
+		Address: AnotherAddress,
+		Topics:  []common.Hash{mockTopic21, mockTopic22},
+		Data:    []byte{},
+	}
+	HeaderCID, _  = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
+	HeaderMhKey   = shared.MultihashKeyFromCID(HeaderCID)
+	Trx1CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(0), multihash.KECCAK_256)
+	Trx1MhKey     = shared.MultihashKeyFromCID(Trx1CID)
+	Trx2CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
+	Trx2MhKey     = shared.MultihashKeyFromCID(Trx2CID)
+	Trx3CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(2), multihash.KECCAK_256)
+	Trx3MhKey     = shared.MultihashKeyFromCID(Trx3CID)
+	Rct1CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
+	Rct1MhKey     = shared.MultihashKeyFromCID(Rct1CID)
+	Rct2CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
+	Rct2MhKey     = shared.MultihashKeyFromCID(Rct2CID)
+	Rct3CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(2), multihash.KECCAK_256)
+	Rct3MhKey     = shared.MultihashKeyFromCID(Rct3CID)
+	State1CID, _  = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
+	State1MhKey   = shared.MultihashKeyFromCID(State1CID)
+	State2CID, _  = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
+	State2MhKey   = shared.MultihashKeyFromCID(State2CID)
+	StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
+	StorageMhKey  = shared.MultihashKeyFromCID(StorageCID)
+	MockTrxMeta   = []eth.TxModel{
+		{
+			CID:    "", // This is empty until we go to publish to ipfs
+			MhKey:  "",
+			Src:    SenderAddr.Hex(),
+			Dst:    Address.String(),
+			Index:  0,
+			TxHash: MockTransactions[0].Hash().String(),
+			Data:   []byte{},
+		},
+		{
+			CID:    "",
+			MhKey:  "",
+			Src:    SenderAddr.Hex(),
+			Dst:    AnotherAddress.String(),
+			Index:  1,
+			TxHash: MockTransactions[1].Hash().String(),
+			Data:   []byte{},
+		},
+		{
+			CID:    "",
+			MhKey:  "",
+			Src:    SenderAddr.Hex(),
+			Dst:    "",
+			Index:  2,
+			TxHash: MockTransactions[2].Hash().String(),
+			Data:   MockContractByteCode,
+		},
+	}
+	MockTrxMetaPostPublsh = []eth.TxModel{
+		{
+			CID:    Trx1CID.String(), // This is empty until we go to publish to ipfs
+			MhKey:  Trx1MhKey,
+			Src:    SenderAddr.Hex(),
+			Dst:    Address.String(),
+			Index:  0,
+			TxHash: MockTransactions[0].Hash().String(),
+			Data:   []byte{},
+		},
+		{
+			CID:    Trx2CID.String(),
+			MhKey:  Trx2MhKey,
+			Src:    SenderAddr.Hex(),
+			Dst:    AnotherAddress.String(),
+			Index:  1,
+			TxHash: MockTransactions[1].Hash().String(),
+			Data:   []byte{},
+		},
+		{
+			CID:    Trx3CID.String(),
+			MhKey:  Trx3MhKey,
+			Src:    SenderAddr.Hex(),
+			Dst:    "",
+			Index:  2,
+			TxHash: MockTransactions[2].Hash().String(),
+			Data:   MockContractByteCode,
+		},
+	}
+	MockRctMeta = []eth.ReceiptModel{
+		{
+			CID:   "",
+			MhKey: "",
+			Topic0s: []string{
+				mockTopic11.String(),
+			},
+			Topic1s: []string{
+				mockTopic12.String(),
+			},
+			Contract:     "",
+			ContractHash: "",
+			LogContracts: []string{
+				Address.String(),
+			},
+		},
+		{
+			CID:   "",
+			MhKey: "",
+			Topic0s: []string{
+				mockTopic21.String(),
+			},
+			Topic1s: []string{
+				mockTopic22.String(),
+			},
+			Contract:     "",
+			ContractHash: "",
+			LogContracts: []string{
+				AnotherAddress.String(),
+			},
+		},
+		{
+			CID:          "",
+			MhKey:        "",
+			Contract:     ContractAddress.String(),
+			ContractHash: ContractHash,
+			LogContracts: []string{},
+		},
+	}
+	MockRctMetaPostPublish = []eth.ReceiptModel{
+		{
+			CID:   Rct1CID.String(),
+			MhKey: Rct1MhKey,
+			Topic0s: []string{
+				mockTopic11.String(),
+			},
+			Topic1s: []string{
+				mockTopic12.String(),
+			},
+			Contract:     "",
+			ContractHash: "",
+			LogContracts: []string{
+				Address.String(),
+			},
+		},
+		{
+			CID:   Rct2CID.String(),
+			MhKey: Rct2MhKey,
+			Topic0s: []string{
+				mockTopic21.String(),
+			},
+			Topic1s: []string{
+				mockTopic22.String(),
+			},
+			Contract:     "",
+			ContractHash: "",
+			LogContracts: []string{
+				AnotherAddress.String(),
+			},
+		},
+		{
+			CID:          Rct3CID.String(),
+			MhKey:        Rct3MhKey,
+			Contract:     ContractAddress.String(),
+			ContractHash: ContractHash,
+			LogContracts: []string{},
+		},
+	}
+
+	// statediff data
+	storageLocation    = common.HexToHash("0")
+	StorageLeafKey     = crypto.Keccak256Hash(storageLocation[:]).Bytes()
+	StorageValue       = crypto.Keccak256([]byte{1, 2, 3, 4, 5})
+	StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
+	StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+		StoragePartialPath,
+		StorageValue,
+	})
+
+	nonce1             = uint64(1)
+	ContractRoot       = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
+	ContractCodeHash   = crypto.Keccak256Hash(MockContractByteCode)
+	contractPath       = common.Bytes2Hex([]byte{'\x06'})
+	ContractLeafKey    = testhelpers.AddressToLeafKey(ContractAddress)
+	ContractAccount, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce1,
+		Balance:  big.NewInt(0),
+		CodeHash: ContractCodeHash.Bytes(),
+		Root:     common.HexToHash(ContractRoot),
+	})
+	ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
+	ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+		ContractPartialPath,
+		ContractAccount,
+	})
+
+	nonce0          = uint64(0)
+	AccountBalance  = big.NewInt(1000)
+	AccountRoot     = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
+	AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
+	AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
+	AccountLeafKey  = testhelpers.Account2LeafKey
+	Account, _      = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce0,
+		Balance:  AccountBalance,
+		CodeHash: AccountCodeHash.Bytes(),
+		Root:     common.HexToHash(AccountRoot),
+	})
+	AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
+	AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+		AccountPartialPath,
+		Account,
+	})
+
+	MockStateNodes = []eth.TrieNode{
+		{
+			LeafKey: common.BytesToHash(ContractLeafKey),
+			Path:    []byte{'\x06'},
+			Value:   ContractLeafNode,
+			Type:    sdtypes.Leaf,
+		},
+		{
+			LeafKey: common.BytesToHash(AccountLeafKey),
+			Path:    []byte{'\x0c'},
+			Value:   AccountLeafNode,
+			Type:    sdtypes.Leaf,
+		},
+	}
+	MockStateMetaPostPublish = []eth.StateNodeModel{
+		{
+			CID:      State1CID.String(),
+			MhKey:    State1MhKey,
+			Path:     []byte{'\x06'},
+			NodeType: 2,
+			StateKey: common.BytesToHash(ContractLeafKey).Hex(),
+		},
+		{
+			CID:      State2CID.String(),
+			MhKey:    State2MhKey,
+			Path:     []byte{'\x0c'},
+			NodeType: 2,
+			StateKey: common.BytesToHash(AccountLeafKey).Hex(),
+		},
+	}
+	MockStorageNodes = map[string][]eth.TrieNode{
+		contractPath: {
+			{
+				LeafKey: common.BytesToHash(StorageLeafKey),
+				Value:   StorageLeafNode,
+				Type:    sdtypes.Leaf,
+				Path:    []byte{},
+			},
+		},
+	}
+
+	MockConvertedPayload = eth.ConvertedPayload{
+		TotalDifficulty: MockBlock.Difficulty(),
+		Block:           MockBlock,
+		Receipts:        MockReceipts,
+		TxMetaData:      MockTrxMeta,
+		ReceiptMetaData: MockRctMeta,
+		StorageNodes:    MockStorageNodes,
+		StateNodes:      MockStateNodes,
+	}
+	MockConvertedPayloadForChild = eth.ConvertedPayload{
+		TotalDifficulty: MockChild.Difficulty(),
+		Block:           MockChild,
+		Receipts:        MockReceipts,
+		TxMetaData:      MockTrxMeta,
+		ReceiptMetaData: MockRctMeta,
+		StorageNodes:    MockStorageNodes,
+		StateNodes:      MockStateNodes,
+	}
+
+	MockCIDWrapper = &eth2.CIDWrapper{
+		BlockNumber: new(big.Int).Set(BlockNumber),
+		Header: eth.HeaderModel{
+			BlockNumber:     "1",
+			BlockHash:       MockBlock.Hash().String(),
+			ParentHash:      "0x0000000000000000000000000000000000000000000000000000000000000000",
+			CID:             HeaderCID.String(),
+			MhKey:           HeaderMhKey,
+			TotalDifficulty: MockBlock.Difficulty().String(),
+			Reward:          "5312500000000000000",
+			StateRoot:       MockBlock.Root().String(),
+			RctRoot:         MockBlock.ReceiptHash().String(),
+			TxRoot:          MockBlock.TxHash().String(),
+			UncleRoot:       MockBlock.UncleHash().String(),
+			Bloom:           MockBlock.Bloom().Bytes(),
+			Timestamp:       MockBlock.Time(),
+			TimesValidated:  1,
+		},
+		Transactions: MockTrxMetaPostPublsh,
+		Receipts:     MockRctMetaPostPublish,
+		Uncles:       []eth.UncleModel{},
+		StateNodes:   MockStateMetaPostPublish,
+		StorageNodes: []eth.StorageNodeWithStateKeyModel{
+			{
+				Path:       []byte{},
+				CID:        StorageCID.String(),
+				MhKey:      StorageMhKey,
+				NodeType:   2,
+				StateKey:   common.BytesToHash(ContractLeafKey).Hex(),
+				StorageKey: common.BytesToHash(StorageLeafKey).Hex(),
+			},
+		},
+	}
+
+	HeaderIPLD, _  = blocks.NewBlockWithCid(MockHeaderRlp, HeaderCID)
+	Trx1IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(0), Trx1CID)
+	Trx2IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), Trx2CID)
+	Trx3IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(2), Trx3CID)
+	Rct1IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), Rct1CID)
+	Rct2IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), Rct2CID)
+	Rct3IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(2), Rct3CID)
+	State1IPLD, _  = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
+	State2IPLD, _  = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
+	StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
+
+	MockIPLDs = eth2.IPLDs{
+		BlockNumber: new(big.Int).Set(BlockNumber),
+		Header: ipfs.BlockModel{
+			Data: HeaderIPLD.RawData(),
+			CID:  HeaderIPLD.Cid().String(),
+		},
+		Transactions: []ipfs.BlockModel{
+			{
+				Data: Trx1IPLD.RawData(),
+				CID:  Trx1IPLD.Cid().String(),
+			},
+			{
+				Data: Trx2IPLD.RawData(),
+				CID:  Trx2IPLD.Cid().String(),
+			},
+			{
+				Data: Trx3IPLD.RawData(),
+				CID:  Trx3IPLD.Cid().String(),
+			},
+		},
+		Receipts: []ipfs.BlockModel{
+			{
+				Data: Rct1IPLD.RawData(),
+				CID:  Rct1IPLD.Cid().String(),
+			},
+			{
+				Data: Rct2IPLD.RawData(),
+				CID:  Rct2IPLD.Cid().String(),
+			},
+			{
+				Data: Rct3IPLD.RawData(),
+				CID:  Rct3IPLD.Cid().String(),
+			},
+		},
+		StateNodes: []eth2.StateNode{
+			{
+				StateLeafKey: common.BytesToHash(ContractLeafKey),
+				Type:         sdtypes.Leaf,
+				IPLD: ipfs.BlockModel{
+					Data: State1IPLD.RawData(),
+					CID:  State1IPLD.Cid().String(),
+				},
+				Path: []byte{'\x06'},
+			},
+			{
+				StateLeafKey: common.BytesToHash(AccountLeafKey),
+				Type:         sdtypes.Leaf,
+				IPLD: ipfs.BlockModel{
+					Data: State2IPLD.RawData(),
+					CID:  State2IPLD.Cid().String(),
+				},
+				Path: []byte{'\x0c'},
+			},
+		},
+		StorageNodes: []eth2.StorageNode{
+			{
+				StateLeafKey:   common.BytesToHash(ContractLeafKey),
+				StorageLeafKey: common.BytesToHash(StorageLeafKey),
+				Type:           sdtypes.Leaf,
+				IPLD: ipfs.BlockModel{
+					Data: StorageIPLD.RawData(),
+					CID:  StorageIPLD.Cid().String(),
+				},
+				Path: []byte{},
+			},
+		},
+	}
+)
+
+// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
+	// make transactions
+	trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
+	trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
+	trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode)
+	transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber))
+	mockCurve := elliptic.P256()
+	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx3, err := types.SignTx(trx3, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	SenderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
+	if err != nil {
+		log.Fatal(err)
+	}
+	// make receipts
+	mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
+	mockReceipt1.Logs = []*types.Log{MockLog1}
+	mockReceipt1.TxHash = signedTrx1.Hash()
+	mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
+	mockReceipt2.Logs = []*types.Log{MockLog2}
+	mockReceipt2.TxHash = signedTrx2.Hash()
+	mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75)
+	mockReceipt3.Logs = []*types.Log{}
+	mockReceipt3.TxHash = signedTrx3.Hash()
+	return types.Transactions{signedTrx1, signedTrx2, signedTrx3}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3}, SenderAddr
+}

pkg/eth/types.go

@@ -0,0 +1,146 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	sdtypes "github.com/ethereum/go-ethereum/statediff/types"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+)
+
+// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
+type RPCTransaction struct {
+	BlockHash        *common.Hash    `json:"blockHash"`
+	BlockNumber      *hexutil.Big    `json:"blockNumber"`
+	From             common.Address  `json:"from"`
+	Gas              hexutil.Uint64  `json:"gas"`
+	GasPrice         *hexutil.Big    `json:"gasPrice"`
+	Hash             common.Hash     `json:"hash"`
+	Input            hexutil.Bytes   `json:"input"`
+	Nonce            hexutil.Uint64  `json:"nonce"`
+	To               *common.Address `json:"to"`
+	TransactionIndex *hexutil.Uint64 `json:"transactionIndex"`
+	Value            *hexutil.Big    `json:"value"`
+	V                *hexutil.Big    `json:"v"`
+	R                *hexutil.Big    `json:"r"`
+	S                *hexutil.Big    `json:"s"`
+}
+
+// RPCReceipt represents a receipt that will serialize to the RPC representation of a receipt
+type RPCReceipt struct {
+	BlockHash        *common.Hash    `json:"blockHash"`
+	BlockNumber      *hexutil.Big    `json:"blockNumber"`
+	TransactionHash  *common.Hash    `json:"transactionHash"`
+	TransactionIndex *hexutil.Uint64 `json:"transactionIndex"`
+	From             common.Address  `json:"from"`
+	To               *common.Address `json:"to"`
+	GasUsed          hexutil.Uint64  `json:"gasUsed"`
+	CumulativeGsUsed hexutil.Uint64  `json:"cumulativeGasUsed"`
+	ContractAddress  *common.Address `json:"contractAddress"`
+	Logs             []*types.Log    `json:"logs"`
+	Bloom            types.Bloom     `json:"logsBloom"`
+	Root             []byte          `json:"root"`
+	Status           uint64          `json:"status"`
+}
+
+// AccountResult struct for GetProof
+type AccountResult struct {
+	Address      common.Address  `json:"address"`
+	AccountProof []string        `json:"accountProof"`
+	Balance      *hexutil.Big    `json:"balance"`
+	CodeHash     common.Hash     `json:"codeHash"`
+	Nonce        hexutil.Uint64  `json:"nonce"`
+	StorageHash  common.Hash     `json:"storageHash"`
+	StorageProof []StorageResult `json:"storageProof"`
+}
+
+// StorageResult for GetProof
+type StorageResult struct {
+	Key   string       `json:"key"`
+	Value *hexutil.Big `json:"value"`
+	Proof []string     `json:"proof"`
+}
+
+// CallArgs represents the arguments for a call.
+type CallArgs struct {
+	From     *common.Address `json:"from"`
+	To       *common.Address `json:"to"`
+	Gas      *hexutil.Uint64 `json:"gas"`
+	GasPrice *hexutil.Big    `json:"gasPrice"`
+	Value    *hexutil.Big    `json:"value"`
+	Data     *hexutil.Bytes  `json:"data"`
+}
+
+// account indicates the overriding fields of account during the execution of
+// a message call.
+// Note, state and stateDiff can't be specified at the same time. If state is
+// set, message execution will only use the data in the given state. Otherwise
+// if statDiff is set, all diff will be applied first and then execute the call
+// message.
+type account struct {
+	Nonce     *hexutil.Uint64              `json:"nonce"`
+	Code      *hexutil.Bytes               `json:"code"`
+	Balance   **hexutil.Big                `json:"balance"`
+	State     *map[common.Hash]common.Hash `json:"state"`
+	StateDiff *map[common.Hash]common.Hash `json:"stateDiff"`
+}
+
+// IPLDs is used to package raw IPLD block data fetched from IPFS and returned by the server
+// Returned by IPLDFetcher and ResponseFilterer
+type IPLDs struct {
+	BlockNumber     *big.Int
+	TotalDifficulty *big.Int
+	Header          ipfs.BlockModel
+	Uncles          []ipfs.BlockModel
+	Transactions    []ipfs.BlockModel
+	Receipts        []ipfs.BlockModel
+	StateNodes      []StateNode
+	StorageNodes    []StorageNode
+}
+
+type StateNode struct {
+	Type         sdtypes.NodeType
+	StateLeafKey common.Hash
+	Path         []byte
+	IPLD         ipfs.BlockModel
+}
+
+type StorageNode struct {
+	Type           sdtypes.NodeType
+	StateLeafKey   common.Hash
+	StorageLeafKey common.Hash
+	Path           []byte
+	IPLD           ipfs.BlockModel
+}
+
+// CIDWrapper is used to direct fetching of IPLDs from IPFS
+// Returned by CIDRetriever
+// Passed to IPLDFetcher
+type CIDWrapper struct {
+	BlockNumber  *big.Int
+	Header       eth.HeaderModel
+	Uncles       []eth.UncleModel
+	Transactions []eth.TxModel
+	Receipts     []eth.ReceiptModel
+	StateNodes   []eth.StateNodeModel
+	StorageNodes []eth.StorageNodeWithStateKeyModel
+}

pkg/graphql/graphiql.go

@@ -0,0 +1,114 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+)
+
+// GraphiQL is an in-browser IDE for exploring GraphiQL APIs.
+// This handler returns GraphiQL when requested.
+//
+// For more information, see https://github.com/graphql/graphiql.
+type GraphiQL struct{}
+
+func respond(w http.ResponseWriter, body []byte, code int) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.WriteHeader(code)
+	_, _ = w.Write(body)
+}
+
+func errorJSON(msg string) []byte {
+	buf := bytes.Buffer{}
+	fmt.Fprintf(&buf, `{"error": "%s"}`, msg)
+	return buf.Bytes()
+}
+
+func (h GraphiQL) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		respond(w, errorJSON("only GET requests are supported"), http.StatusMethodNotAllowed)
+		return
+	}
+	w.Header().Set("Content-Type", "text/html")
+	w.Write(graphiql)
+}
+
+var graphiql = []byte(`
+<!DOCTYPE html>
+<html>
+	<head>
+		<link
+                rel="icon"
+                type="image/png"
+                href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAAAAlwSFlzAAALEwAACxMBAJqcGAAAActpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDUuNC4wIj4KICAgPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICAgICAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgICAgICAgICAgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx4bXA6Q3JlYXRvclRvb2w+QWRvYmUgSW1hZ2VSZWFkeTwveG1wOkNyZWF0b3JUb29sPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaWVudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4KKS7NPQAAB5FJREFUWAm1FmtsnEdxdr/vfC8/mpgEfHYa6gaUJqAihfhVO7UprSokHsn5jKgKiKLGIIEEbSlpJdQLIJw+UFFQUSuBWir1z9nnpEmgCUnkcxPSmDRCgkKpGoJpfXdxHSc4ftzr2x1m9rvPPQdDDSgrfd/O7szOe2YX4H8cGEtY3tFK2Nu7pjMCChbgzVfD11h4XLKAibahL6dbBv+SaRl6LUsw78XBxTG80mEsWSkxu1oM9qmJlkR7UPhPWSDJCzSISw5zXZGxvpMezUp5GmtWQszunpiAKiPPZ20KyCqY1/ncgs4v+IUPwLJvYhzTVIbmvXgvqwAxkImKJHt1yzM+AQLXvdKXy3QevB4R+3O6wIYHSUCIlABEtTO9bf86pmFa7B6xPeHMi3l668p5SQjInbRGQQw0E3FMH4FHaFPoP8USVaveEo9aaH3LsdRh2vsYKqwhMhRBKw82vGbNQbcC9ePL1+PDmwf7iix0N+xmPoafq4TgDDaRYxmLCrBwD5HpSK4vKRVeP9b3ZyaaaE18UaL4KYE5x5afsWxoBgefFfX+jX6pMH9RvSnX2v1YxPP4D3UAHG2hgm80vRp7ns9nWxOb8kIt3HD6C+O8rpRVoYCxHDOtQwOg4QHS1kIb9oHGVQJlN0h8qPF07FFmkG4byouAjEdSO/bwOntr8kGt8EeNJ3uN27O37fse5PT3lVIjUsrL6MB2IVCThMcbx3ofIt7sZeMFExeTubSR3Zq4tVoEdhHSJs30WqjbIS1Zk6/VqzzhmdbBpyn5p1g4W8LMGkajj9GUSfcM/4IVaji+/QdOa7hehKz69xEPsllLkFZY+HdlWhOdLNxrXm5iTK1xPSHEeo4KxTFPzEsFLHH8D914rG+GGWe2Dd9UJav6ZbW1k9ep7rgF3SnTEUXA3hko2fdkowc2M27dk3deomgfLBIPYlJytC4QLzKLZdAoy3QzNTVqksT2y6Oz+YVL1TK4Oo9FYAVIkRFzgH8F/bOiD0cjv4m+hEA9IdXn8HaC4Mjxzx7OdCZH8R14mra6eB9sfUKTj4SCQLUvCHMqN235rKMGV5ZpPCAoSzGOcs2JaFZYVuc8FF5XQl8uCHV75FT0ZT6Q6Ry+02fZ3b7agLF+MGbYmF/Mg+vE14NY1Xnhjv2fZkTkWO+R2VXqc1BrLczp/OtULV0fOLXjHS5LlvkuhzL05oZf+xnMbtv3BLXZIwyPQNx4iRLvrXRXci/vcV/guXJ4dZ/elnwqfctQlnFxoGyhkY2+eCbTlnyCYU8GwzzcHHBhmKl7261X1CEBaIT0QNxJdyQfpLRdHblt4wNMeuhsVpWPvDulqAXQKH5i9f0Ut7pMT/LhOEWc96hfkBEYYnhDU3DJ2SUKMAEPIagRoTSJObF9uF5oHAC/uF/ENxeRrPcai0vt/k1mE+6GeE9eVIlvQwF+yGfL/KiNuMpUnmF4WQUYwX3AEEzjXmqi5yOp6DO8hrM7TeIZ+Orf2X6DY1oU+FeY1D8xJLh8G2bcsgpQ3vqoAU1P3nWouQaDd8mQdS8Tj1B/Z0sZXm6QyxbvAFlj3Us95e7Jbx6/EYScpnP/kjfMwy3DMre6mXVGIVTqiqi1mtVk8blZR78UOdGbQqDLheLMjWc54Yt7KSAaUvRwTyrdMXREvFF6VtRZfgrALNOcm8ixZxe9uOgBLsMPnftUIdM+tBFKcLtwxCeJ7GbdHDJlJ6DHYetX8gHfSTTEB4P9WNBb5JRq0VrfwbxZRuVN61pMt56ICz3elWxAB18OS//Nep4MKeowTOU/zMwo8RaV5fVKhs4WN1DzCjkzJV1jBT9K1TB6oWN4bR89arDMz7iTa1ikepxsy+CXqmXol1fUfJ4qwUfeptsXL1JNTFNWXkfmO5ydi8KXBIMWvCYnmbOWmKXr5zpZhHotSbQGp9YO+qkb3h05E3vBk+nmwJopw5SSdVxRsOjiCGhEXSMCMFdTrAdbPikul35PvWAN1adPgqAGz8Kk1FLTX2hlCyF9pHSIQlwnp+x6/yb1t9zu8LgFszJHt5v0K+TakuPmbFnmog2cXBzfbFtyj1b6O4SQ4BP76Zr1k1Etwoe7Ir+N/dwcfo8f3QnbsYR7yAO/kxICdAH1En+km/WxhtPRXZ4sZrOoQBk2npjcmmwu2ipMz6s/MlG6JflVqrC9pN8VqLK+1nhix4u8/3Z7YjXPRHeJ52z3vm7Mq6eISa0UeF/DK7FB3r/w8eGP0Htg4f1noud5TXgy1g1lpQIGQelGyLjbQk3J7TZr8yT7uxzwSfu+oiwdIL//gTKc+4MUltxL/lpPFn+ebvqByFhswAjid+VgTLNnXcGcyHGuY7PmvWUHZ2hlqXgXDRNfbD/YSE+2MeeWYzjZMmw+p+MYpnuSJy/FjtZ5DCvPuI9SFv5/DI4buZxfwZBuH7pnpu0QprcOztM3N9v2K8x2DH+FcZktB/nSWeJZ3v93Y8VasRubmqBoGKF4g6oBwjIQoi/MMDrqHOMamnMFmv6ziw0T97diTb0zHB7OEe4ZlCjf5X2U8vGm09HnKrPbo78mMwu6mjFn9tV713TtvWpZSCX83wr9J1EKd8CrhC26AAAAAElFTkSuQmCC"
+        />
+        <link
+                rel="stylesheet"
+                href="https://cdnjs.cloudflare.com/ajax/libs/graphiql/0.13.0/graphiql.css"
+                integrity="sha384-Qua2xoKBxcHOg1ivsKWo98zSI5KD/UuBpzMIg8coBd4/jGYoxeozCYFI9fesatT0"
+                crossorigin="anonymous"
+        />
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/fetch/3.0.0/fetch.min.js"
+                integrity="sha384-5B8/4F9AQqp/HCHReGLSOWbyAOwnJsPrvx6C0+VPUr44Olzi99zYT1xbVh+ZanQJ"
+                crossorigin="anonymous"
+        ></script>
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/react/16.8.5/umd/react.production.min.js"
+                integrity="sha384-dOCiLz3nZfHiJj//EWxjwSKSC6Z1IJtyIEK/b/xlHVNdVLXDYSesoxiZb94bbuGE"
+                crossorigin="anonymous"
+        ></script>
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/react-dom/16.8.5/umd/react-dom.production.min.js"
+                integrity="sha384-QI+ql5f+khgo3mMdCktQ3E7wUKbIpuQo8S5rA/3i1jg2rMsloCNyiZclI7sFQUGN"
+                crossorigin="anonymous"
+        ></script>
+        <script
+                src="https://cdnjs.cloudflare.com/ajax/libs/graphiql/0.13.0/graphiql.min.js"
+                integrity="sha384-roSmzNmO4zJK9X4lwggDi4/oVy+9V4nlS1+MN8Taj7tftJy1GvMWyAhTNXdC/fFR"
+                crossorigin="anonymous"
+        ></script>
+	</head>
+	<body style="width: 100%; height: 100%; margin: 0; overflow: hidden;">
+		<div id="graphiql" style="height: 100vh;">Loading...</div>
+		<script>
+			function fetchGQL(params) {
+				return fetch("/graphql", {
+					method: "post",
+					body: JSON.stringify(params),
+					credentials: "include",
+				}).then(function (resp) {
+					return resp.text();
+				}).then(function (body) {
+					try {
+						return JSON.parse(body);
+					} catch (error) {
+						return body;
+					}
+				});
+			}
+			ReactDOM.render(
+				React.createElement(GraphiQL, {fetcher: fetchGQL}),
+				document.getElementById("graphiql")
+			)
+		</script>
+	</body>
+</html>
+`)

pkg/graphql/graphql.go

@@ -0,0 +1,942 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Package graphql provides a GraphQL interface to Ethereum node data.
+package graphql
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/eth/filters"
+	"github.com/ethereum/go-ethereum/rpc"
+
+	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+)
+
+var (
+	errBlockInvariant = errors.New("block objects must be instantiated with at least one of num or hash")
+)
+
+// Account represents an Ethereum account at a particular block.
+type Account struct {
+	backend       *eth.Backend
+	address       common.Address
+	blockNrOrHash rpc.BlockNumberOrHash
+}
+
+// getState fetches the StateDB object for an account.
+func (a *Account) getState(ctx context.Context) (*state.StateDB, error) {
+	state, _, err := a.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash)
+	return state, err
+}
+
+func (a *Account) Address(ctx context.Context) (common.Address, error) {
+	return a.address, nil
+}
+
+func (a *Account) Balance(ctx context.Context) (hexutil.Big, error) {
+	state, err := a.getState(ctx)
+	if err != nil {
+		return hexutil.Big{}, err
+	}
+	return hexutil.Big(*state.GetBalance(a.address)), nil
+}
+
+func (a *Account) TransactionCount(ctx context.Context) (hexutil.Uint64, error) {
+	state, err := a.getState(ctx)
+	if err != nil {
+		return 0, err
+	}
+	return hexutil.Uint64(state.GetNonce(a.address)), nil
+}
+
+func (a *Account) Code(ctx context.Context) (hexutil.Bytes, error) {
+	state, err := a.getState(ctx)
+	if err != nil {
+		return hexutil.Bytes{}, err
+	}
+	return hexutil.Bytes(state.GetCode(a.address)), nil
+}
+
+func (a *Account) Storage(ctx context.Context, args struct{ Slot common.Hash }) (common.Hash, error) {
+	state, err := a.getState(ctx)
+	if err != nil {
+		return common.Hash{}, err
+	}
+	return state.GetState(a.address, args.Slot), nil
+}
+
+// Log represents an individual log message. All arguments are mandatory.
+type Log struct {
+	backend     *eth.Backend
+	transaction *Transaction
+	log         *types.Log
+}
+
+func (l *Log) Transaction(ctx context.Context) *Transaction {
+	return l.transaction
+}
+
+func (l *Log) Account(ctx context.Context, args BlockNumberArgs) *Account {
+	return &Account{
+		backend:       l.backend,
+		address:       l.log.Address,
+		blockNrOrHash: args.NumberOrLatest(),
+	}
+}
+
+func (l *Log) Index(ctx context.Context) int32 {
+	return int32(l.log.Index)
+}
+
+func (l *Log) Topics(ctx context.Context) []common.Hash {
+	return l.log.Topics
+}
+
+func (l *Log) Data(ctx context.Context) hexutil.Bytes {
+	return hexutil.Bytes(l.log.Data)
+}
+
+// Transaction represents an Ethereum transaction.
+// backend and hash are mandatory; all others will be fetched when required.
+type Transaction struct {
+	backend *eth.Backend
+	hash    common.Hash
+	tx      *types.Transaction
+	block   *Block
+	index   uint64
+}
+
+// resolve returns the internal transaction object, fetching it if needed.
+func (t *Transaction) resolve(ctx context.Context) (*types.Transaction, error) {
+	if t.tx == nil {
+		tx, blockHash, _, index := rawdb.ReadTransaction(t.backend.ChainDb(), t.hash)
+		if tx != nil {
+			t.tx = tx
+			blockNrOrHash := rpc.BlockNumberOrHashWithHash(blockHash, false)
+			t.block = &Block{
+				backend:      t.backend,
+				numberOrHash: &blockNrOrHash,
+			}
+			t.index = index
+		}
+	}
+	return t.tx, nil
+}
+
+func (t *Transaction) Hash(ctx context.Context) common.Hash {
+	return t.hash
+}
+
+func (t *Transaction) InputData(ctx context.Context) (hexutil.Bytes, error) {
+	tx, err := t.resolve(ctx)
+	if err != nil || tx == nil {
+		return hexutil.Bytes{}, err
+	}
+	return hexutil.Bytes(tx.Data()), nil
+}
+
+func (t *Transaction) Gas(ctx context.Context) (hexutil.Uint64, error) {
+	tx, err := t.resolve(ctx)
+	if err != nil || tx == nil {
+		return 0, err
+	}
+	return hexutil.Uint64(tx.Gas()), nil
+}
+
+func (t *Transaction) GasPrice(ctx context.Context) (hexutil.Big, error) {
+	tx, err := t.resolve(ctx)
+	if err != nil || tx == nil {
+		return hexutil.Big{}, err
+	}
+	return hexutil.Big(*tx.GasPrice()), nil
+}
+
+func (t *Transaction) Value(ctx context.Context) (hexutil.Big, error) {
+	tx, err := t.resolve(ctx)
+	if err != nil || tx == nil {
+		return hexutil.Big{}, err
+	}
+	return hexutil.Big(*tx.Value()), nil
+}
+
+func (t *Transaction) Nonce(ctx context.Context) (hexutil.Uint64, error) {
+	tx, err := t.resolve(ctx)
+	if err != nil || tx == nil {
+		return 0, err
+	}
+	return hexutil.Uint64(tx.Nonce()), nil
+}
+
+func (t *Transaction) To(ctx context.Context, args BlockNumberArgs) (*Account, error) {
+	tx, err := t.resolve(ctx)
+	if err != nil || tx == nil {
+		return nil, err
+	}
+	to := tx.To()
+	if to == nil {
+		return nil, nil
+	}
+	return &Account{
+		backend:       t.backend,
+		address:       *to,
+		blockNrOrHash: args.NumberOrLatest(),
+	}, nil
+}
+
+func (t *Transaction) From(ctx context.Context, args BlockNumberArgs) (*Account, error) {
+	tx, err := t.resolve(ctx)
+	if err != nil || tx == nil {
+		return nil, err
+	}
+	var signer types.Signer = types.HomesteadSigner{}
+	if tx.Protected() {
+		signer = types.NewEIP155Signer(tx.ChainId())
+	}
+	from, _ := types.Sender(signer, tx)
+
+	return &Account{
+		backend:       t.backend,
+		address:       from,
+		blockNrOrHash: args.NumberOrLatest(),
+	}, nil
+}
+
+func (t *Transaction) Block(ctx context.Context) (*Block, error) {
+	if _, err := t.resolve(ctx); err != nil {
+		return nil, err
+	}
+	return t.block, nil
+}
+
+func (t *Transaction) Index(ctx context.Context) (*int32, error) {
+	if _, err := t.resolve(ctx); err != nil {
+		return nil, err
+	}
+	if t.block == nil {
+		return nil, nil
+	}
+	index := int32(t.index)
+	return &index, nil
+}
+
+// getReceipt returns the receipt associated with this transaction, if any.
+func (t *Transaction) getReceipt(ctx context.Context) (*types.Receipt, error) {
+	if _, err := t.resolve(ctx); err != nil {
+		return nil, err
+	}
+	if t.block == nil {
+		return nil, nil
+	}
+	receipts, err := t.block.resolveReceipts(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return receipts[t.index], nil
+}
+
+func (t *Transaction) Status(ctx context.Context) (*hexutil.Uint64, error) {
+	receipt, err := t.getReceipt(ctx)
+	if err != nil || receipt == nil {
+		return nil, err
+	}
+	ret := hexutil.Uint64(receipt.Status)
+	return &ret, nil
+}
+
+func (t *Transaction) GasUsed(ctx context.Context) (*hexutil.Uint64, error) {
+	receipt, err := t.getReceipt(ctx)
+	if err != nil || receipt == nil {
+		return nil, err
+	}
+	ret := hexutil.Uint64(receipt.GasUsed)
+	return &ret, nil
+}
+
+func (t *Transaction) CumulativeGasUsed(ctx context.Context) (*hexutil.Uint64, error) {
+	receipt, err := t.getReceipt(ctx)
+	if err != nil || receipt == nil {
+		return nil, err
+	}
+	ret := hexutil.Uint64(receipt.CumulativeGasUsed)
+	return &ret, nil
+}
+
+func (t *Transaction) CreatedContract(ctx context.Context, args BlockNumberArgs) (*Account, error) {
+	receipt, err := t.getReceipt(ctx)
+	if err != nil || receipt == nil || receipt.ContractAddress == (common.Address{}) {
+		return nil, err
+	}
+	return &Account{
+		backend:       t.backend,
+		address:       receipt.ContractAddress,
+		blockNrOrHash: args.NumberOrLatest(),
+	}, nil
+}
+
+func (t *Transaction) Logs(ctx context.Context) (*[]*Log, error) {
+	receipt, err := t.getReceipt(ctx)
+	if err != nil || receipt == nil {
+		return nil, err
+	}
+	ret := make([]*Log, 0, len(receipt.Logs))
+	for _, log := range receipt.Logs {
+		ret = append(ret, &Log{
+			backend:     t.backend,
+			transaction: t,
+			log:         log,
+		})
+	}
+	return &ret, nil
+}
+
+func (t *Transaction) R(ctx context.Context) (hexutil.Big, error) {
+	tx, err := t.resolve(ctx)
+	if err != nil || tx == nil {
+		return hexutil.Big{}, err
+	}
+	_, r, _ := tx.RawSignatureValues()
+	return hexutil.Big(*r), nil
+}
+
+func (t *Transaction) S(ctx context.Context) (hexutil.Big, error) {
+	tx, err := t.resolve(ctx)
+	if err != nil || tx == nil {
+		return hexutil.Big{}, err
+	}
+	_, _, s := tx.RawSignatureValues()
+	return hexutil.Big(*s), nil
+}
+
+func (t *Transaction) V(ctx context.Context) (hexutil.Big, error) {
+	tx, err := t.resolve(ctx)
+	if err != nil || tx == nil {
+		return hexutil.Big{}, err
+	}
+	v, _, _ := tx.RawSignatureValues()
+	return hexutil.Big(*v), nil
+}
+
+type BlockType int
+
+// Block represents an Ethereum block.
+// backend, and numberOrHash are mandatory. All other fields are lazily fetched
+// when required.
+type Block struct {
+	backend      *eth.Backend
+	numberOrHash *rpc.BlockNumberOrHash
+	hash         common.Hash
+	header       *types.Header
+	block        *types.Block
+	receipts     []*types.Receipt
+}
+
+// resolve returns the internal Block object representing this block, fetching
+// it if necessary.
+func (b *Block) resolve(ctx context.Context) (*types.Block, error) {
+	if b.block != nil {
+		return b.block, nil
+	}
+	if b.numberOrHash == nil {
+		latest := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber)
+		b.numberOrHash = &latest
+	}
+	var err error
+	b.block, err = b.backend.BlockByNumberOrHash(ctx, *b.numberOrHash)
+	if b.block != nil && b.header == nil {
+		b.header = b.block.Header()
+		if hash, ok := b.numberOrHash.Hash(); ok {
+			b.hash = hash
+		}
+	}
+	return b.block, err
+}
+
+// resolveHeader returns the internal Header object for this block, fetching it
+// if necessary. Call this function instead of `resolve` unless you need the
+// additional data (transactions and uncles).
+func (b *Block) resolveHeader(ctx context.Context) (*types.Header, error) {
+	if b.numberOrHash == nil && b.hash == (common.Hash{}) {
+		return nil, errBlockInvariant
+	}
+	var err error
+	if b.header == nil {
+		if b.hash != (common.Hash{}) {
+			b.header, err = b.backend.HeaderByHash(ctx, b.hash)
+		} else {
+			b.header, err = b.backend.HeaderByNumberOrHash(ctx, *b.numberOrHash)
+		}
+	}
+	return b.header, err
+}
+
+// resolveReceipts returns the list of receipts for this block, fetching them
+// if necessary.
+func (b *Block) resolveReceipts(ctx context.Context) ([]*types.Receipt, error) {
+	if b.receipts == nil {
+		hash := b.hash
+		if hash == (common.Hash{}) {
+			header, err := b.resolveHeader(ctx)
+			if err != nil {
+				return nil, err
+			}
+			hash = header.Hash()
+		}
+		receipts, err := b.backend.GetReceipts(ctx, hash)
+		if err != nil {
+			return nil, err
+		}
+		b.receipts = []*types.Receipt(receipts)
+	}
+	return b.receipts, nil
+}
+
+func (b *Block) Number(ctx context.Context) (hexutil.Uint64, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return 0, err
+	}
+
+	return hexutil.Uint64(header.Number.Uint64()), nil
+}
+
+func (b *Block) Hash(ctx context.Context) (common.Hash, error) {
+	if b.hash == (common.Hash{}) {
+		header, err := b.resolveHeader(ctx)
+		if err != nil {
+			return common.Hash{}, err
+		}
+		b.hash = header.Hash()
+	}
+	return b.hash, nil
+}
+
+func (b *Block) GasLimit(ctx context.Context) (hexutil.Uint64, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return 0, err
+	}
+	return hexutil.Uint64(header.GasLimit), nil
+}
+
+func (b *Block) GasUsed(ctx context.Context) (hexutil.Uint64, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return 0, err
+	}
+	return hexutil.Uint64(header.GasUsed), nil
+}
+
+func (b *Block) Parent(ctx context.Context) (*Block, error) {
+	// If the block header hasn't been fetched, and we'll need it, fetch it.
+	if b.numberOrHash == nil && b.header == nil {
+		if _, err := b.resolveHeader(ctx); err != nil {
+			return nil, err
+		}
+	}
+	if b.header != nil && b.header.Number.Uint64() > 0 {
+		num := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(b.header.Number.Uint64() - 1))
+		return &Block{
+			backend:      b.backend,
+			numberOrHash: &num,
+			hash:         b.header.ParentHash,
+		}, nil
+	}
+	return nil, nil
+}
+
+func (b *Block) Difficulty(ctx context.Context) (hexutil.Big, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return hexutil.Big{}, err
+	}
+	return hexutil.Big(*header.Difficulty), nil
+}
+
+func (b *Block) Timestamp(ctx context.Context) (hexutil.Uint64, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return 0, err
+	}
+	return hexutil.Uint64(header.Time), nil
+}
+
+func (b *Block) Nonce(ctx context.Context) (hexutil.Bytes, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return hexutil.Bytes{}, err
+	}
+	return hexutil.Bytes(header.Nonce[:]), nil
+}
+
+func (b *Block) MixHash(ctx context.Context) (common.Hash, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return common.Hash{}, err
+	}
+	return header.MixDigest, nil
+}
+
+func (b *Block) TransactionsRoot(ctx context.Context) (common.Hash, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return common.Hash{}, err
+	}
+	return header.TxHash, nil
+}
+
+func (b *Block) StateRoot(ctx context.Context) (common.Hash, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return common.Hash{}, err
+	}
+	return header.Root, nil
+}
+
+func (b *Block) ReceiptsRoot(ctx context.Context) (common.Hash, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return common.Hash{}, err
+	}
+	return header.ReceiptHash, nil
+}
+
+func (b *Block) OmmerHash(ctx context.Context) (common.Hash, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return common.Hash{}, err
+	}
+	return header.UncleHash, nil
+}
+
+func (b *Block) OmmerCount(ctx context.Context) (*int32, error) {
+	block, err := b.resolve(ctx)
+	if err != nil || block == nil {
+		return nil, err
+	}
+	count := int32(len(block.Uncles()))
+	return &count, err
+}
+
+func (b *Block) Ommers(ctx context.Context) (*[]*Block, error) {
+	block, err := b.resolve(ctx)
+	if err != nil || block == nil {
+		return nil, err
+	}
+	ret := make([]*Block, 0, len(block.Uncles()))
+	for _, uncle := range block.Uncles() {
+		blockNumberOrHash := rpc.BlockNumberOrHashWithHash(uncle.Hash(), false)
+		ret = append(ret, &Block{
+			backend:      b.backend,
+			numberOrHash: &blockNumberOrHash,
+			header:       uncle,
+		})
+	}
+	return &ret, nil
+}
+
+func (b *Block) ExtraData(ctx context.Context) (hexutil.Bytes, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return hexutil.Bytes{}, err
+	}
+	return hexutil.Bytes(header.Extra), nil
+}
+
+func (b *Block) LogsBloom(ctx context.Context) (hexutil.Bytes, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return hexutil.Bytes{}, err
+	}
+	return hexutil.Bytes(header.Bloom.Bytes()), nil
+}
+
+func (b *Block) TotalDifficulty(ctx context.Context) (hexutil.Big, error) {
+	h := b.hash
+	if h == (common.Hash{}) {
+		header, err := b.resolveHeader(ctx)
+		if err != nil {
+			return hexutil.Big{}, err
+		}
+		h = header.Hash()
+	}
+	td, err := b.backend.GetTd(h)
+	if err != nil {
+		return hexutil.Big{}, err
+	}
+	return hexutil.Big(*td), nil
+}
+
+// BlockNumberArgs encapsulates arguments to accessors that specify a block number.
+type BlockNumberArgs struct {
+	// TODO: Ideally we could use input unions to allow the query to specify the
+	// block parameter by hash, block number, or tag but input unions aren't part of the
+	// standard GraphQL schema SDL yet, see: https://github.com/graphql/graphql-spec/issues/488
+	Block *hexutil.Uint64
+}
+
+// NumberOr returns the provided block number argument, or the "current" block number or hash if none
+// was provided.
+func (a BlockNumberArgs) NumberOr(current rpc.BlockNumberOrHash) rpc.BlockNumberOrHash {
+	if a.Block != nil {
+		blockNr := rpc.BlockNumber(*a.Block)
+		return rpc.BlockNumberOrHashWithNumber(blockNr)
+	}
+	return current
+}
+
+// NumberOrLatest returns the provided block number argument, or the "latest" block number if none
+// was provided.
+func (a BlockNumberArgs) NumberOrLatest() rpc.BlockNumberOrHash {
+	return a.NumberOr(rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber))
+}
+
+func (b *Block) Miner(ctx context.Context, args BlockNumberArgs) (*Account, error) {
+	header, err := b.resolveHeader(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &Account{
+		backend:       b.backend,
+		address:       header.Coinbase,
+		blockNrOrHash: args.NumberOrLatest(),
+	}, nil
+}
+
+func (b *Block) TransactionCount(ctx context.Context) (*int32, error) {
+	block, err := b.resolve(ctx)
+	if err != nil || block == nil {
+		return nil, err
+	}
+	count := int32(len(block.Transactions()))
+	return &count, err
+}
+
+func (b *Block) Transactions(ctx context.Context) (*[]*Transaction, error) {
+	block, err := b.resolve(ctx)
+	if err != nil || block == nil {
+		return nil, err
+	}
+	ret := make([]*Transaction, 0, len(block.Transactions()))
+	for i, tx := range block.Transactions() {
+		ret = append(ret, &Transaction{
+			backend: b.backend,
+			hash:    tx.Hash(),
+			tx:      tx,
+			block:   b,
+			index:   uint64(i),
+		})
+	}
+	return &ret, nil
+}
+
+func (b *Block) TransactionAt(ctx context.Context, args struct{ Index int32 }) (*Transaction, error) {
+	block, err := b.resolve(ctx)
+	if err != nil || block == nil {
+		return nil, err
+	}
+	txs := block.Transactions()
+	if args.Index < 0 || int(args.Index) >= len(txs) {
+		return nil, nil
+	}
+	tx := txs[args.Index]
+	return &Transaction{
+		backend: b.backend,
+		hash:    tx.Hash(),
+		tx:      tx,
+		block:   b,
+		index:   uint64(args.Index),
+	}, nil
+}
+
+func (b *Block) OmmerAt(ctx context.Context, args struct{ Index int32 }) (*Block, error) {
+	block, err := b.resolve(ctx)
+	if err != nil || block == nil {
+		return nil, err
+	}
+	uncles := block.Uncles()
+	if args.Index < 0 || int(args.Index) >= len(uncles) {
+		return nil, nil
+	}
+	uncle := uncles[args.Index]
+	blockNumberOrHash := rpc.BlockNumberOrHashWithHash(uncle.Hash(), false)
+	return &Block{
+		backend:      b.backend,
+		numberOrHash: &blockNumberOrHash,
+		header:       uncle,
+	}, nil
+}
+
+// BlockFilterCriteria encapsulates criteria passed to a `logs` accessor inside
+// a block.
+type BlockFilterCriteria struct {
+	Addresses *[]common.Address // restricts matches to events created by specific contracts
+
+	// The Topic list restricts matches to particular event topics. Each event has a list
+	// of topics. Topics matches a prefix of that list. An empty element slice matches any
+	// topic. Non-empty elements represent an alternative that matches any of the
+	// contained topics.
+	//
+	// Examples:
+	// {} or nil          matches any topic list
+	// {{A}}              matches topic A in first position
+	// {{}, {B}}          matches any topic in first position, B in second position
+	// {{A}, {B}}         matches topic A in first position, B in second position
+	// {{A, B}}, {C, D}}  matches topic (A OR B) in first position, (C OR D) in second position
+	Topics *[][]common.Hash
+}
+
+// runFilter accepts a filter and executes it, returning all its results as
+// `Log` objects.
+func runFilter(ctx context.Context, be *eth.Backend, filter *filters.Filter) ([]*Log, error) {
+	logs, err := filter.Logs(ctx)
+	if err != nil || logs == nil {
+		return nil, err
+	}
+	ret := make([]*Log, 0, len(logs))
+	for _, log := range logs {
+		ret = append(ret, &Log{
+			backend:     be,
+			transaction: &Transaction{backend: be, hash: log.TxHash},
+			log:         log,
+		})
+	}
+	return ret, nil
+}
+
+func (b *Block) Logs(ctx context.Context, args struct{ Filter BlockFilterCriteria }) ([]*Log, error) {
+	var addresses []common.Address
+	if args.Filter.Addresses != nil {
+		addresses = *args.Filter.Addresses
+	}
+	var topics [][]common.Hash
+	if args.Filter.Topics != nil {
+		topics = *args.Filter.Topics
+	}
+	hash := b.hash
+	if hash == (common.Hash{}) {
+		header, err := b.resolveHeader(ctx)
+		if err != nil {
+			return nil, err
+		}
+		hash = header.Hash()
+	}
+	// Construct the range filter
+	filter := filters.NewBlockFilter(b.backend, hash, addresses, topics)
+
+	// Run the filter and return all the logs
+	return runFilter(ctx, b.backend, filter)
+}
+
+func (b *Block) Account(ctx context.Context, args struct {
+	Address common.Address
+}) (*Account, error) {
+	if b.numberOrHash == nil {
+		_, err := b.resolveHeader(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &Account{
+		backend:       b.backend,
+		address:       args.Address,
+		blockNrOrHash: *b.numberOrHash,
+	}, nil
+}
+
+// CallData encapsulates arguments to `call` or `estimateGas`.
+// All arguments are optional.
+type CallData struct {
+	From     *common.Address // The Ethereum address the call is from.
+	To       *common.Address // The Ethereum address the call is to.
+	Gas      *hexutil.Uint64 // The amount of gas provided for the call.
+	GasPrice *hexutil.Big    // The price of each unit of gas, in wei.
+	Value    *hexutil.Big    // The value sent along with the call.
+	Data     *hexutil.Bytes  // Any data sent with the call.
+}
+
+// CallResult encapsulates the result of an invocation of the `call` accessor.
+type CallResult struct {
+	data    hexutil.Bytes  // The return data from the call
+	gasUsed hexutil.Uint64 // The amount of gas used
+	status  hexutil.Uint64 // The return status of the call - 0 for failure or 1 for success.
+}
+
+func (c *CallResult) Data() hexutil.Bytes {
+	return c.data
+}
+
+func (c *CallResult) GasUsed() hexutil.Uint64 {
+	return c.gasUsed
+}
+
+func (c *CallResult) Status() hexutil.Uint64 {
+	return c.status
+}
+
+func (b *Block) Call(ctx context.Context, args struct {
+	Data eth.CallArgs
+}) (*CallResult, error) {
+	if b.numberOrHash == nil {
+		_, err := b.resolve(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+	result, gas, failed, err := eth.DoCall(ctx, b.backend, args.Data, *b.numberOrHash, nil, 5*time.Second, b.backend.RPCGasCap())
+	status := hexutil.Uint64(1)
+	if failed {
+		status = 0
+	}
+	return &CallResult{
+		data:    hexutil.Bytes(result),
+		gasUsed: hexutil.Uint64(gas),
+		status:  status,
+	}, err
+}
+
+// Resolver is the top-level object in the GraphQL hierarchy.
+type Resolver struct {
+	backend *eth.Backend
+}
+
+func (r *Resolver) Block(ctx context.Context, args struct {
+	Number *hexutil.Uint64
+	Hash   *common.Hash
+}) (*Block, error) {
+	var block *Block
+	if args.Number != nil {
+		number := rpc.BlockNumber(uint64(*args.Number))
+		numberOrHash := rpc.BlockNumberOrHashWithNumber(number)
+		block = &Block{
+			backend:      r.backend,
+			numberOrHash: &numberOrHash,
+		}
+	} else if args.Hash != nil {
+		numberOrHash := rpc.BlockNumberOrHashWithHash(*args.Hash, false)
+		block = &Block{
+			backend:      r.backend,
+			numberOrHash: &numberOrHash,
+		}
+	} else {
+		numberOrHash := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber)
+		block = &Block{
+			backend:      r.backend,
+			numberOrHash: &numberOrHash,
+		}
+	}
+	// Resolve the header, return nil if it doesn't exist.
+	// Note we don't resolve block directly here since it will require an
+	// additional network request for light client.
+	h, err := block.resolveHeader(ctx)
+	if err != nil {
+		return nil, err
+	} else if h == nil {
+		return nil, nil
+	}
+	return block, nil
+}
+
+func (r *Resolver) Blocks(ctx context.Context, args struct {
+	From hexutil.Uint64
+	To   *hexutil.Uint64
+}) ([]*Block, error) {
+	from := rpc.BlockNumber(args.From)
+
+	var to rpc.BlockNumber
+	if args.To != nil {
+		to = rpc.BlockNumber(*args.To)
+	} else {
+		to = rpc.BlockNumber(r.backend.CurrentBlock().Number().Int64())
+	}
+	if to < from {
+		return []*Block{}, nil
+	}
+	ret := make([]*Block, 0, to-from+1)
+	for i := from; i <= to; i++ {
+		numberOrHash := rpc.BlockNumberOrHashWithNumber(i)
+		ret = append(ret, &Block{
+			backend:      r.backend,
+			numberOrHash: &numberOrHash,
+		})
+	}
+	return ret, nil
+}
+
+func (r *Resolver) Transaction(ctx context.Context, args struct{ Hash common.Hash }) (*Transaction, error) {
+	tx := &Transaction{
+		backend: r.backend,
+		hash:    args.Hash,
+	}
+	// Resolve the transaction; if it doesn't exist, return nil.
+	t, err := tx.resolve(ctx)
+	if err != nil {
+		return nil, err
+	} else if t == nil {
+		return nil, nil
+	}
+	return tx, nil
+}
+
+// FilterCriteria encapsulates the arguments to `logs` on the root resolver object.
+type FilterCriteria struct {
+	FromBlock *hexutil.Uint64   // beginning of the queried range, nil means genesis block
+	ToBlock   *hexutil.Uint64   // end of the range, nil means latest block
+	Addresses *[]common.Address // restricts matches to events created by specific contracts
+
+	// The Topic list restricts matches to particular event topics. Each event has a list
+	// of topics. Topics matches a prefix of that list. An empty element slice matches any
+	// topic. Non-empty elements represent an alternative that matches any of the
+	// contained topics.
+	//
+	// Examples:
+	// {} or nil          matches any topic list
+	// {{A}}              matches topic A in first position
+	// {{}, {B}}          matches any topic in first position, B in second position
+	// {{A}, {B}}         matches topic A in first position, B in second position
+	// {{A, B}}, {C, D}}  matches topic (A OR B) in first position, (C OR D) in second position
+	Topics *[][]common.Hash
+}
+
+func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria }) ([]*Log, error) {
+	// Convert the RPC block numbers into internal representations
+	begin := rpc.LatestBlockNumber.Int64()
+	if args.Filter.FromBlock != nil {
+		begin = int64(*args.Filter.FromBlock)
+	}
+	end := rpc.LatestBlockNumber.Int64()
+	if args.Filter.ToBlock != nil {
+		end = int64(*args.Filter.ToBlock)
+	}
+	var addresses []common.Address
+	if args.Filter.Addresses != nil {
+		addresses = *args.Filter.Addresses
+	}
+	var topics [][]common.Hash
+	if args.Filter.Topics != nil {
+		topics = *args.Filter.Topics
+	}
+	// Construct the range filter
+	filter := filters.NewRangeFilter(filters.Backend(r.backend), begin, end, addresses, topics)
+	return runFilter(ctx, r.backend, filter)
+}

pkg/graphql/graphql_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
+)
+
+func TestGraphQL(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "graphql test suite")
+}
+
+var _ = BeforeSuite(func() {
+	logrus.SetOutput(ioutil.Discard)
+})

pkg/graphql/graphql_test.go

@@ -0,0 +1,31 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/ipld-eth-server/pkg/graphql"
+)
+
+var _ = Describe("GraphQL", func() {
+	It("Builds the schema and creates a new handler", func() {
+		_, err := graphql.NewHandler(nil)
+		Expect(err).ToNot(HaveOccurred())
+	})
+})

pkg/graphql/schema.go

@@ -0,0 +1,274 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+const schema string = `
+    # Bytes32 is a 32 byte binary string, represented as 0x-prefixed hexadecimal.
+    scalar Bytes32
+    # Address is a 20 byte Ethereum address, represented as 0x-prefixed hexadecimal.
+    scalar Address
+    # Bytes is an arbitrary length binary string, represented as 0x-prefixed hexadecimal.
+    # An empty byte string is represented as '0x'. Byte strings must have an even number of hexadecimal nybbles.
+    scalar Bytes
+    # BigInt is a large integer. Input is accepted as either a JSON number or as a string.
+    # Strings may be either decimal or 0x-prefixed hexadecimal. Output values are all
+    # 0x-prefixed hexadecimal.
+    scalar BigInt
+    # Long is a 64 bit unsigned integer.
+    scalar Long
+
+    schema {
+        query: Query
+    }
+
+    # Account is an Ethereum account at a particular block.
+    type Account {
+        # Address is the address owning the account.
+        address: Address!
+        # Balance is the balance of the account, in wei.
+        balance: BigInt!
+        # TransactionCount is the number of transactions sent from this account,
+        # or in the case of a contract, the number of contracts created. Otherwise
+        # known as the nonce.
+        transactionCount: Long!
+        # Code contains the smart contract code for this account, if the account
+        # is a (non-self-destructed) contract.
+        code: Bytes!
+        # Storage provides access to the storage of a contract account, indexed
+        # by its 32 byte slot identifier.
+        storage(slot: Bytes32!): Bytes32!
+    }
+
+    # Log is an Ethereum event log.
+    type Log {
+        # Index is the index of this log in the block.
+        index: Int!
+        # Account is the account which generated this log - this will always
+        # be a contract account.
+        account(block: Long): Account!
+        # Topics is a list of 0-4 indexed topics for the log.
+        topics: [Bytes32!]!
+        # Data is unindexed data for this log.
+        data: Bytes!
+        # Transaction is the transaction that generated this log entry.
+        transaction: Transaction!
+    }
+
+    # Transaction is an Ethereum transaction.
+    type Transaction {
+        # Hash is the hash of this transaction.
+        hash: Bytes32!
+        # Nonce is the nonce of the account this transaction was generated with.
+        nonce: Long!
+        # Index is the index of this transaction in the parent block. This will
+        # be null if the transaction has not yet been mined.
+        index: Int
+        # From is the account that sent this transaction - this will always be
+        # an externally owned account.
+        from(block: Long): Account!
+        # To is the account the transaction was sent to. This is null for
+        # contract-creating transactions.
+        to(block: Long): Account
+        # Value is the value, in wei, sent along with this transaction.
+        value: BigInt!
+        # GasPrice is the price offered to miners for gas, in wei per unit.
+        gasPrice: BigInt!
+        # Gas is the maximum amount of gas this transaction can consume.
+        gas: Long!
+        # InputData is the data supplied to the target of the transaction.
+        inputData: Bytes!
+        # Block is the block this transaction was mined in. This will be null if
+        # the transaction has not yet been mined.
+        block: Block
+
+        # Status is the return status of the transaction. This will be 1 if the
+        # transaction succeeded, or 0 if it failed (due to a revert, or due to
+        # running out of gas). If the transaction has not yet been mined, this
+        # field will be null.
+        status: Long
+        # GasUsed is the amount of gas that was used processing this transaction.
+        # If the transaction has not yet been mined, this field will be null.
+        gasUsed: Long
+        # CumulativeGasUsed is the total gas used in the block up to and including
+        # this transaction. If the transaction has not yet been mined, this field
+        # will be null.
+        cumulativeGasUsed: Long
+        # CreatedContract is the account that was created by a contract creation
+        # transaction. If the transaction was not a contract creation transaction,
+        # or it has not yet been mined, this field will be null.
+        createdContract(block: Long): Account
+        # Logs is a list of log entries emitted by this transaction. If the
+        # transaction has not yet been mined, this field will be null.
+        logs: [Log!]
+        r: BigInt!
+        s: BigInt!
+        v: BigInt!
+    }
+
+    # BlockFilterCriteria encapsulates log filter criteria for a filter applied
+    # to a single block.
+    input BlockFilterCriteria {
+        # Addresses is list of addresses that are of interest. If this list is
+        # empty, results will not be filtered by address.
+        addresses: [Address!]
+        # Topics list restricts matches to particular event topics. Each event has a list
+      # of topics. Topics matches a prefix of that list. An empty element array matches any
+      # topic. Non-empty elements represent an alternative that matches any of the
+      # contained topics.
+      #
+      # Examples:
+      #  - [] or nil          matches any topic list
+      #  - [[A]]              matches topic A in first position
+      #  - [[], [B]]          matches any topic in first position, B in second position
+      #  - [[A], [B]]         matches topic A in first position, B in second position
+      #  - [[A, B]], [C, D]]  matches topic (A OR B) in first position, (C OR D) in second position
+        topics: [[Bytes32!]!]
+    }
+
+    # Block is an Ethereum block.
+    type Block {
+        # Number is the number of this block, starting at 0 for the genesis block.
+        number: Long!
+        # Hash is the block hash of this block.
+        hash: Bytes32!
+        # Parent is the parent block of this block.
+        parent: Block
+        # Nonce is the block nonce, an 8 byte sequence determined by the miner.
+        nonce: Bytes!
+        # TransactionsRoot is the keccak256 hash of the root of the trie of transactions in this block.
+        transactionsRoot: Bytes32!
+        # TransactionCount is the number of transactions in this block. if
+        # transactions are not available for this block, this field will be null.
+        transactionCount: Int
+        # StateRoot is the keccak256 hash of the state trie after this block was processed.
+        stateRoot: Bytes32!
+        # ReceiptsRoot is the keccak256 hash of the trie of transaction receipts in this block.
+        receiptsRoot: Bytes32!
+        # Miner is the account that mined this block.
+        miner(block: Long): Account!
+        # ExtraData is an arbitrary data field supplied by the miner.
+        extraData: Bytes!
+        # GasLimit is the maximum amount of gas that was available to transactions in this block.
+        gasLimit: Long!
+        # GasUsed is the amount of gas that was used executing transactions in this block.
+        gasUsed: Long!
+        # Timestamp is the unix timestamp at which this block was mined.
+        timestamp: Long!
+        # LogsBloom is a bloom filter that can be used to check if a block may
+        # contain log entries matching a filter.
+        logsBloom: Bytes!
+        # MixHash is the hash that was used as an input to the PoW process.
+        mixHash: Bytes32!
+        # Difficulty is a measure of the difficulty of mining this block.
+        difficulty: BigInt!
+        # TotalDifficulty is the sum of all difficulty values up to and including
+        # this block.
+        totalDifficulty: BigInt!
+        # OmmerCount is the number of ommers (AKA uncles) associated with this
+        # block. If ommers are unavailable, this field will be null.
+        ommerCount: Int
+        # Ommers is a list of ommer (AKA uncle) blocks associated with this block.
+        # If ommers are unavailable, this field will be null. Depending on your
+        # node, the transactions, transactionAt, transactionCount, ommers,
+        # ommerCount and ommerAt fields may not be available on any ommer blocks.
+        ommers: [Block]
+        # OmmerAt returns the ommer (AKA uncle) at the specified index. If ommers
+        # are unavailable, or the index is out of bounds, this field will be null.
+        ommerAt(index: Int!): Block
+        # OmmerHash is the keccak256 hash of all the ommers (AKA uncles)
+        # associated with this block.
+        ommerHash: Bytes32!
+        # Transactions is a list of transactions associated with this block. If
+        # transactions are unavailable for this block, this field will be null.
+        transactions: [Transaction!]
+        # TransactionAt returns the transaction at the specified index. If
+        # transactions are unavailable for this block, or if the index is out of
+        # bounds, this field will be null.
+        transactionAt(index: Int!): Transaction
+        # Logs returns a filtered set of logs from this block.
+        logs(filter: BlockFilterCriteria!): [Log!]!
+        # Account fetches an Ethereum account at the current block's state.
+        account(address: Address!): Account!
+        # Call executes a local call operation at the current block's state.
+        call(data: CallData!): CallResult
+    }
+
+    # CallData represents the data associated with a local contract call.
+    # All fields are optional.
+    input CallData {
+        # From is the address making the call.
+        from: Address
+        # To is the address the call is sent to.
+        to: Address
+        # Gas is the amount of gas sent with the call.
+        gas: Long
+        # GasPrice is the price, in wei, offered for each unit of gas.
+        gasPrice: BigInt
+        # Value is the value, in wei, sent along with the call.
+        value: BigInt
+        # Data is the data sent to the callee.
+        data: Bytes
+    }
+
+    # CallResult is the result of a local call operation.
+    type CallResult {
+        # Data is the return data of the called contract.
+        data: Bytes!
+        # GasUsed is the amount of gas used by the call, after any refunds.
+        gasUsed: Long!
+        # Status is the result of the call - 1 for success or 0 for failure.
+        status: Long!
+    }
+
+    # FilterCriteria encapsulates log filter criteria for searching log entries.
+    input FilterCriteria {
+        # FromBlock is the block at which to start searching, inclusive. Defaults
+        # to the latest block if not supplied.
+        fromBlock: Long
+        # ToBlock is the block at which to stop searching, inclusive. Defaults
+        # to the latest block if not supplied.
+        toBlock: Long
+        # Addresses is a list of addresses that are of interest. If this list is
+        # empty, results will not be filtered by address.
+        addresses: [Address!]
+        # Topics list restricts matches to particular event topics. Each event has a list
+      # of topics. Topics matches a prefix of that list. An empty element array matches any
+      # topic. Non-empty elements represent an alternative that matches any of the
+      # contained topics.
+      #
+      # Examples:
+      #  - [] or nil          matches any topic list
+      #  - [[A]]              matches topic A in first position
+      #  - [[], [B]]          matches any topic in first position, B in second position
+      #  - [[A], [B]]         matches topic A in first position, B in second position
+      #  - [[A, B]], [C, D]]  matches topic (A OR B) in first position, (C OR D) in second position
+        topics: [[Bytes32!]!]
+    }
+
+    type Query {
+        # Block fetches an Ethereum block by number or by hash. If neither is
+        # supplied, the most recent known block is returned.
+        block(number: Long, hash: Bytes32): Block
+        # Blocks returns all the blocks between two numbers, inclusive. If
+        # to is not supplied, it defaults to the most recent known block.
+        blocks(from: Long!, to: Long): [Block!]!
+        # Transaction returns a transaction specified by its hash.
+        transaction(hash: Bytes32!): Transaction
+        # Logs returns log entries matching the provided filter.
+        logs(filter: FilterCriteria!): [Log!]!
+    }
+`

pkg/graphql/service.go

@@ -0,0 +1,111 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package graphql
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/graph-gophers/graphql-go"
+	"github.com/graph-gophers/graphql-go/relay"
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+)
+
+// Service encapsulates a GraphQL service.
+type Service struct {
+	endpoint string           // The host:port endpoint for this service.
+	cors     []string         // Allowed CORS domains
+	vhosts   []string         // Recognised vhosts
+	timeouts rpc.HTTPTimeouts // Timeout settings for HTTP requests.
+	backend  *eth.Backend     // The backend that queries will operate onn.
+	handler  http.Handler     // The `http.Handler` used to answer queries.
+	listener net.Listener     // The listening socket.
+}
+
+// New constructs a new GraphQL service instance.
+func New(backend *eth.Backend, endpoint string, cors, vhosts []string, timeouts rpc.HTTPTimeouts) (*Service, error) {
+	return &Service{
+		endpoint: endpoint,
+		cors:     cors,
+		vhosts:   vhosts,
+		timeouts: timeouts,
+		backend:  backend,
+	}, nil
+}
+
+// Protocols returns the list of protocols exported by this service.
+func (s *Service) Protocols() []p2p.Protocol { return nil }
+
+// APIs returns the list of APIs exported by this service.
+func (s *Service) APIs() []rpc.API { return nil }
+
+// Start is called after all services have been constructed and the networking
+// layer was also initialized to spawn any goroutines required by the service.
+func (s *Service) Start(server *p2p.Server) error {
+	var err error
+	s.handler, err = NewHandler(s.backend)
+	if err != nil {
+		return err
+	}
+
+	handler := node.NewHTTPHandlerStack(s.handler, s.cors, s.vhosts)
+
+	// start http server
+	_, addr, err := node.StartHTTPEndpoint(s.endpoint, rpc.DefaultHTTPTimeouts, handler)
+	if err != nil {
+		utils.Fatalf("Could not start RPC api: %v", err)
+	}
+	extapiURL := fmt.Sprintf("http://%v/", addr)
+	logrus.Infof("graphQL endpoint opened for url %s", extapiURL)
+	return nil
+}
+
+// newHandler returns a new `http.Handler` that will answer GraphQL queries.
+// It additionally exports an interactive query browser on the / endpoint.
+func NewHandler(backend *eth.Backend) (http.Handler, error) {
+	q := Resolver{backend}
+
+	s, err := graphql.ParseSchema(schema, &q)
+	if err != nil {
+		return nil, err
+	}
+	h := &relay.Handler{Schema: s}
+
+	mux := http.NewServeMux()
+	mux.Handle("/", GraphiQL{})
+	mux.Handle("/graphql", h)
+	mux.Handle("/graphql/", h)
+	return mux, nil
+}
+
+// Stop terminates all goroutines belonging to the service, blocking until they
+// are all terminated.
+func (s *Service) Stop() error {
+	if s.listener != nil {
+		s.listener.Close()
+		s.listener = nil
+		logrus.Debugf("graphQL endpoint closed for url %s", fmt.Sprintf("http://%s", s.endpoint))
+	}
+	return nil
+}

pkg/prom/db_stats_collector.go

@@ -0,0 +1,159 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"database/sql"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const subsystem = "connections"
+
+// DBStatsGetter is an interface that gets sql.DBStats.
+type DBStatsGetter interface {
+	Stats() sql.DBStats
+}
+
+// DBStatsCollector implements the prometheus.Collector interface.
+type DBStatsCollector struct {
+	sg DBStatsGetter
+
+	// descriptions of exported metrics
+	maxOpenDesc           *prometheus.Desc
+	openDesc              *prometheus.Desc
+	inUseDesc             *prometheus.Desc
+	idleDesc              *prometheus.Desc
+	waitedForDesc         *prometheus.Desc
+	blockedSecondsDesc    *prometheus.Desc
+	closedMaxIdleDesc     *prometheus.Desc
+	closedMaxLifetimeDesc *prometheus.Desc
+}
+
+// NewDBStatsCollector creates a new DBStatsCollector.
+func NewDBStatsCollector(dbName string, sg DBStatsGetter) *DBStatsCollector {
+	labels := prometheus.Labels{"db_name": dbName}
+	return &DBStatsCollector{
+		sg: sg,
+		maxOpenDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "max_open"),
+			"Maximum number of open connections to the database.",
+			nil,
+			labels,
+		),
+		openDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "open"),
+			"The number of established connections both in use and idle.",
+			nil,
+			labels,
+		),
+		inUseDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "in_use"),
+			"The number of connections currently in use.",
+			nil,
+			labels,
+		),
+		idleDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "idle"),
+			"The number of idle connections.",
+			nil,
+			labels,
+		),
+		waitedForDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "waited_for"),
+			"The total number of connections waited for.",
+			nil,
+			labels,
+		),
+		blockedSecondsDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "blocked_seconds"),
+			"The total time blocked waiting for a new connection.",
+			nil,
+			labels,
+		),
+		closedMaxIdleDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "closed_max_idle"),
+			"The total number of connections closed due to SetMaxIdleConns.",
+			nil,
+			labels,
+		),
+		closedMaxLifetimeDesc: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, subsystem, "closed_max_lifetime"),
+			"The total number of connections closed due to SetConnMaxLifetime.",
+			nil,
+			labels,
+		),
+	}
+}
+
+// Describe implements the prometheus.Collector interface.
+func (c DBStatsCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.maxOpenDesc
+	ch <- c.openDesc
+	ch <- c.inUseDesc
+	ch <- c.idleDesc
+	ch <- c.waitedForDesc
+	ch <- c.blockedSecondsDesc
+	ch <- c.closedMaxIdleDesc
+	ch <- c.closedMaxLifetimeDesc
+}
+
+// Collect implements the prometheus.Collector interface.
+func (c DBStatsCollector) Collect(ch chan<- prometheus.Metric) {
+	stats := c.sg.Stats()
+
+	ch <- prometheus.MustNewConstMetric(
+		c.maxOpenDesc,
+		prometheus.GaugeValue,
+		float64(stats.MaxOpenConnections),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.openDesc,
+		prometheus.GaugeValue,
+		float64(stats.OpenConnections),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.inUseDesc,
+		prometheus.GaugeValue,
+		float64(stats.InUse),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.idleDesc,
+		prometheus.GaugeValue,
+		float64(stats.Idle),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.waitedForDesc,
+		prometheus.CounterValue,
+		float64(stats.WaitCount),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.blockedSecondsDesc,
+		prometheus.CounterValue,
+		stats.WaitDuration.Seconds(),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.closedMaxIdleDesc,
+		prometheus.CounterValue,
+		float64(stats.MaxIdleClosed),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		c.closedMaxLifetimeDesc,
+		prometheus.CounterValue,
+		float64(stats.MaxLifetimeClosed),
+	)
+}

pkg/prom/middleware.go

@@ -0,0 +1,64 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// HTTPMiddleware http connection metric reader
+func HTTPMiddleware(next http.Handler) http.Handler {
+	if !metrics {
+		return next
+	}
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		httpCount.Inc()
+
+		start := time.Now()
+		next.ServeHTTP(w, r)
+		duration := time.Now().Sub(start)
+		httpDuration.Observe(float64(duration.Seconds()))
+	})
+}
+
+// WSMiddleware websocket connection counter
+func WSMiddleware(next http.Handler) http.Handler {
+	if !metrics {
+		return next
+	}
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		wsCount.Inc()
+		next.ServeHTTP(w, r)
+		wsCount.Dec()
+	})
+}
+
+// IPCMiddleware unix-socket connection counter
+func IPCMiddleware(server *rpc.Server, client rpc.Conn) {
+	if metrics {
+		ipcCount.Inc()
+	}
+	server.ServeCodec(rpc.NewCodec(client), 0)
+	if metrics {
+		ipcCount.Dec()
+	}
+}

pkg/prom/prom.go

@@ -0,0 +1,79 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"github.com/jmoiron/sqlx"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const (
+	namespace = "ipld_eth_server"
+
+	subsystemHTTP = "http"
+	subsystemWS   = "ws"
+	subsystemIPC  = "ipc"
+)
+
+var (
+	metrics bool
+
+	httpCount    prometheus.Counter
+	httpDuration prometheus.Histogram
+	wsCount      prometheus.Gauge
+	ipcCount     prometheus.Gauge
+)
+
+// Init module initialization
+func Init() {
+	metrics = true
+
+	httpCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystemHTTP,
+		Name:      "count",
+		Help:      "http request count",
+	})
+	httpDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Subsystem: subsystemHTTP,
+		Name:      "duration",
+		Help:      "http request duration",
+	})
+
+	wsCount = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemWS,
+		Name:      "count",
+		Help:      "websocket connection count",
+	})
+
+	ipcCount = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystemIPC,
+		Name:      "count",
+		Help:      "unix socket connection count",
+	})
+}
+
+// RegisterDBCollector create metric colletor for given connection
+func RegisterDBCollector(name string, db *sqlx.DB) {
+	if metrics {
+		prometheus.Register(NewDBStatsCollector(name, db))
+	}
+}

pkg/prom/serve.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package prom
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/sirupsen/logrus"
+)
+
+var errPromHTTP = errors.New("can't start http server for prometheus")
+
+// Serve start listening http
+func Serve(addr string) *http.Server {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	srv := http.Server{
+		Addr:    addr,
+		Handler: mux,
+	}
+	go func() {
+		if err := srv.ListenAndServe(); err != nil {
+			logrus.
+				WithError(err).
+				WithField("module", "prom").
+				WithField("addr", addr).
+				Fatal(errPromHTTP)
+		}
+	}()
+	return &srv
+}

pkg/rpc/check.go

@@ -0,0 +1,37 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import "github.com/ethereum/go-ethereum/rpc"
+
+// checkModuleAvailability check that all names given in modules are actually
+// available API services.
+func checkModuleAvailability(modules []string, apis []rpc.API) (bad, available []string) {
+	availableSet := make(map[string]struct{})
+	for _, api := range apis {
+		if _, ok := availableSet[api.Namespace]; !ok {
+			availableSet[api.Namespace] = struct{}{}
+			available = append(available, api.Namespace)
+		}
+	}
+	for _, name := range modules {
+		if _, ok := availableSet[name]; !ok {
+			bad = append(bad, name)
+		}
+	}
+	return bad, available
+}

pkg/rpc/http.go

@@ -0,0 +1,47 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+	log "github.com/sirupsen/logrus"
+)
+
+// StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules.
+func StartHTTPEndpoint(endpoint string, apis []rpc.API, modules []string, cors []string, vhosts []string, timeouts rpc.HTTPTimeouts) (*rpc.Server, error) {
+
+	srv := rpc.NewServer()
+	err := node.RegisterApisFromWhitelist(apis, modules, srv, false)
+	if err != nil {
+		utils.Fatalf("Could not register HTTP API: %w", err)
+	}
+	handler := node.NewHTTPHandlerStack(srv, cors, vhosts)
+
+	// start http server
+	_, addr, err := node.StartHTTPEndpoint(endpoint, rpc.DefaultHTTPTimeouts, handler)
+	if err != nil {
+		utils.Fatalf("Could not start RPC api: %v", err)
+	}
+	extapiURL := fmt.Sprintf("http://%v/", addr)
+	log.Info("HTTP endpoint opened", "url", extapiURL)
+
+	return srv, err
+}

pkg/rpc/ipc.go

@@ -0,0 +1,91 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+
+	"github.com/ethereum/go-ethereum/p2p/netutil"
+	"github.com/ethereum/go-ethereum/rpc"
+	log "github.com/sirupsen/logrus"
+	"github.com/vulcanize/ipld-eth-server/pkg/prom"
+)
+
+var (
+	//  On Linux, sun_path is 108 bytes in size
+	// see http://man7.org/linux/man-pages/man7/unix.7.html
+	maxPathSize = 108
+)
+
+// ipcListen will create a Unix socket on the given endpoint.
+func ipcListen(endpoint string) (net.Listener, error) {
+	if len(endpoint) > int(maxPathSize) {
+		log.Warn(fmt.Sprintf("The ipc endpoint is longer than %d characters. ", maxPathSize),
+			"endpoint", endpoint)
+	}
+
+	// Ensure the IPC path exists and remove any previous leftover
+	if err := os.MkdirAll(filepath.Dir(endpoint), 0751); err != nil {
+		return nil, err
+	}
+	os.Remove(endpoint)
+	l, err := net.Listen("unix", endpoint)
+	if err != nil {
+		return nil, err
+	}
+	os.Chmod(endpoint, 0600)
+	return l, nil
+}
+
+func ipcServe(srv *rpc.Server, listener net.Listener) {
+	for {
+		conn, err := listener.Accept()
+		if netutil.IsTemporaryError(err) {
+			log.WithError(err).Warn("rpc accept error")
+			continue
+		}
+		if err != nil {
+			log.WithError(err).Warn("unknown error")
+			continue
+		}
+		log.WithField("addr", conn.RemoteAddr()).Trace("accepted ipc connection")
+		go prom.IPCMiddleware(srv, conn)
+	}
+}
+
+// StartIPCEndpoint starts an IPC endpoint.
+func StartIPCEndpoint(ipcEndpoint string, apis []rpc.API) (net.Listener, *rpc.Server, error) {
+	// Register all the APIs exposed by the services.
+	handler := rpc.NewServer()
+	for _, api := range apis {
+		if err := handler.RegisterName(api.Namespace, api.Service); err != nil {
+			return nil, nil, err
+		}
+		log.Debug("IPC registered", "namespace", api.Namespace)
+	}
+	// All APIs registered, start the IPC listener.
+	listener, err := ipcListen(ipcEndpoint)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	go ipcServe(handler, listener)
+	return listener, handler, nil
+}

pkg/rpc/ws.go

@@ -0,0 +1,62 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import (
+	"net"
+	"net/http"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+
+	"github.com/vulcanize/ipld-eth-server/pkg/prom"
+)
+
+// StartWSEndpoint starts a websocket endpoint.
+func StartWSEndpoint(endpoint string, apis []rpc.API, modules []string, wsOrigins []string, exposeAll bool) (net.Listener, *rpc.Server, error) {
+	// All APIs registered, start the HTTP listener
+	var (
+		listener net.Listener
+		err      error
+	)
+
+	// Register all the APIs exposed by the services
+	handler := rpc.NewServer()
+	err = node.RegisterApisFromWhitelist(apis, modules, handler, exposeAll)
+	if err != nil {
+		utils.Fatalf("Could not register WS API: %w", err)
+	}
+
+	if listener, err = net.Listen("tcp", endpoint); err != nil {
+		return nil, nil, err
+	}
+
+	wsServer := NewWSServer(wsOrigins, handler)
+	wsServer.Handler = prom.WSMiddleware(wsServer.Handler)
+	go wsServer.Serve(listener)
+
+	return listener, handler, err
+
+}
+
+// NewWSServer creates a new websocket RPC server around an API provider.
+//
+// Deprecated: use prc.Server.WebsocketHandler
+func NewWSServer(allowedOrigins []string, srv *rpc.Server) *http.Server {
+	return &http.Server{Handler: srv.WebsocketHandler(allowedOrigins)}
+}

pkg/serve/api.go

@@ -23,8 +23,9 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
+
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 	v "github.com/vulcanize/ipld-eth-server/version"
 )
 
@@ -87,7 +88,7 @@ func (api *PublicServerAPI) Stream(ctx context.Context, params eth.SubscriptionS
 
 // Chain returns the chain type that this watcher instance supports
 func (api *PublicServerAPI) Chain() shared.ChainType {
-	return api.w.Chain()
+	return shared.Ethereum
 }
 
 // Struct for holding watcher meta data

pkg/serve/config.go

@@ -17,16 +17,23 @@
 package serve
 
 import (
+	"fmt"
+	"math/big"
 	"os"
 	"path/filepath"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/node"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
 
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/spf13/viper"
-
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
-
 	"github.com/vulcanize/ipld-eth-indexer/utils"
+	"github.com/vulcanize/ipld-eth-server/pkg/prom"
+
+	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 )
 
 // Env variables
@@ -38,15 +45,25 @@ const (
 	SERVER_MAX_IDLE_CONNECTIONS = "SERVER_MAX_IDLE_CONNECTIONS"
 	SERVER_MAX_OPEN_CONNECTIONS = "SERVER_MAX_OPEN_CONNECTIONS"
 	SERVER_MAX_CONN_LIFETIME    = "SERVER_MAX_CONN_LIFETIME"
+
+	ETH_DEFAULT_SENDER_ADDR = "ETH_DEFAULT_SENDER_ADDR"
+	ETH_RPC_GAS_CAP         = "ETH_RPC_GAS_CAP"
+	ETH_CHAIN_CONFIG        = "ETH_CHAIN_CONFIG"
+	ETH_SUPPORTS_STATEDIFF  = "ETH_SUPPORTS_STATEDIFF"
 )
 
 // Config struct
 type Config struct {
-	DB           *postgres.DB
-	DBConfig     postgres.Config
-	WSEndpoint   string
-	HTTPEndpoint string
-	IPCEndpoint  string
+	DB               *postgres.DB
+	DBConfig         postgres.Config
+	WSEndpoint       string
+	HTTPEndpoint     string
+	IPCEndpoint      string
+	ChainConfig      *params.ChainConfig
+	DefaultSender    *common.Address
+	RPCGasCap        *big.Int
+	Client           *rpc.Client
+	SupportStateDiff bool
 }
 
 // NewConfig is used to initialize a watcher config from a .toml file
@@ -57,9 +74,22 @@ func NewConfig() (*Config, error) {
 	viper.BindEnv("server.wsPath", SERVER_WS_PATH)
 	viper.BindEnv("server.ipcPath", SERVER_IPC_PATH)
 	viper.BindEnv("server.httpPath", SERVER_HTTP_PATH)
+	viper.BindEnv("ethereum.httpPath", shared.ETH_HTTP_PATH)
+	viper.BindEnv("ethereum.defaultSender", ETH_DEFAULT_SENDER_ADDR)
+	viper.BindEnv("ethereum.rpcGasCap", ETH_RPC_GAS_CAP)
+	viper.BindEnv("ethereum.chainConfig", ETH_CHAIN_CONFIG)
+	viper.BindEnv("ethereum.supportsStateDiff", ETH_SUPPORTS_STATEDIFF)
 
 	c.DBConfig.Init()
 
+	ethHTTP := viper.GetString("ethereum.httpPath")
+	nodeInfo, cli, err := shared.GetEthNodeAndClient(fmt.Sprintf("http://%s", ethHTTP))
+	if err != nil {
+		return nil, err
+	}
+	c.Client = cli
+	c.SupportStateDiff = viper.GetBool("ethereum.supportsStateDiff")
+
 	wsPath := viper.GetString("server.wsPath")
 	if wsPath == "" {
 		wsPath = "127.0.0.1:8080"
@@ -80,10 +110,28 @@ func NewConfig() (*Config, error) {
 	}
 	c.HTTPEndpoint = httpPath
 	overrideDBConnConfig(&c.DBConfig)
-	serveDB := utils.LoadPostgres(c.DBConfig, node.Info{})
+	serveDB := utils.LoadPostgres(c.DBConfig, nodeInfo, false)
+	prom.RegisterDBCollector(c.DBConfig.Name, serveDB.DB)
 	c.DB = &serveDB
 
-	return c, nil
+	defaultSenderStr := viper.GetString("ethereum.defaultSender")
+	if defaultSenderStr != "" {
+		sender := common.HexToAddress(defaultSenderStr)
+		c.DefaultSender = &sender
+	}
+	rpcGasCapStr := viper.GetString("ethereum.rpcGasCap")
+	if rpcGasCapStr != "" {
+		if rpcGasCap, ok := new(big.Int).SetString(rpcGasCapStr, 10); ok {
+			c.RPCGasCap = rpcGasCap
+		}
+	}
+	chainConfigPath := viper.GetString("ethereum.chainConfig")
+	if chainConfigPath != "" {
+		c.ChainConfig, err = eth.LoadConfig(chainConfigPath)
+	} else {
+		c.ChainConfig, err = eth.ChainConfig(nodeInfo.ChainID)
+	}
+	return c, err
 }
 
 func overrideDBConnConfig(con *postgres.Config) {

pkg/serve/service.go

@@ -20,6 +20,8 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/ethereum/go-ethereum/core/vm"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	ethnode "github.com/ethereum/go-ethereum/node"
@@ -32,7 +34,6 @@ import (
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
-	"github.com/vulcanize/ipld-eth-server/pkg/shared"
 )
 
 const (
@@ -43,16 +44,18 @@ const (
 // and indexing all chain data; screening this data; and serving it up to subscribed clients
 // This service is compatible with the Ethereum service interface (node.Service)
 type Server interface {
-	// APIs(), Protocols(), Start() and Stop()
-	ethnode.Service
+	// Start() and Stop()
+	ethnode.Lifecycle
+	APIs() []rpc.API
+	Protocols() []p2p.Protocol
 	// Pub-Sub handling event loop
 	Serve(wg *sync.WaitGroup, screenAndServePayload <-chan eth2.ConvertedPayload)
 	// Method to subscribe to the service
 	Subscribe(id rpc.ID, sub chan<- SubscriptionPayload, quitChan chan<- bool, params eth.SubscriptionSettings)
 	// Method to unsubscribe from the service
 	Unsubscribe(id rpc.ID)
-	// Method to access chain type
-	Chain() shared.ChainType
+	// Backend exposes the server's backend
+	Backend() *eth.Backend
 }
 
 // Service is the underlying struct for the watcher
@@ -75,19 +78,34 @@ type Service struct {
 	db *postgres.DB
 	// wg for syncing serve processes
 	serveWg *sync.WaitGroup
+	// rpc client for forwarding cache misses
+	client *rpc.Client
+	// whether the proxied client supports state diffing
+	supportsStateDiffing bool
+	// backend for the server
+	backend *eth.Backend
 }
 
 // NewServer creates a new Server using an underlying Service struct
 func NewServer(settings *Config) (Server, error) {
-	sn := new(Service)
-	sn.Retriever = eth.NewCIDRetriever(settings.DB)
-	sn.IPLDFetcher = eth.NewIPLDFetcher(settings.DB)
-	sn.Filterer = eth.NewResponseFilterer()
-	sn.db = settings.DB
-	sn.QuitChan = make(chan bool)
-	sn.Subscriptions = make(map[common.Hash]map[rpc.ID]Subscription)
-	sn.SubscriptionTypes = make(map[common.Hash]eth.SubscriptionSettings)
-	return sn, nil
+	sap := new(Service)
+	sap.Retriever = eth.NewCIDRetriever(settings.DB)
+	sap.IPLDFetcher = eth.NewIPLDFetcher(settings.DB)
+	sap.Filterer = eth.NewResponseFilterer()
+	sap.db = settings.DB
+	sap.QuitChan = make(chan bool)
+	sap.Subscriptions = make(map[common.Hash]map[rpc.ID]Subscription)
+	sap.SubscriptionTypes = make(map[common.Hash]eth.SubscriptionSettings)
+	sap.client = settings.Client
+	sap.supportsStateDiffing = settings.SupportStateDiff
+	var err error
+	sap.backend, err = eth.NewEthBackend(sap.db, &eth.Config{
+		ChainConfig:   settings.ChainConfig,
+		VmConfig:      vm.Config{},
+		DefaultSender: settings.DefaultSender,
+		RPCGasCap:     settings.RPCGasCap,
+	})
+	return sap, err
 }
 
 // Protocols exports the services p2p protocols, this service has none
@@ -124,15 +142,10 @@ func (sap *Service) APIs() []rpc.API {
 			Public:    true,
 		},
 	}
-	backend, err := eth.NewEthBackend(sap.db)
-	if err != nil {
-		log.Error(err)
-		return nil
-	}
 	return append(apis, rpc.API{
 		Namespace: eth.APIName,
 		Version:   eth.APIVersion,
-		Service:   eth.NewPublicEthAPI(backend),
+		Service:   eth.NewPublicEthAPI(sap.backend, sap.client, sap.supportsStateDiffing),
 		Public:    true,
 	})
 }
@@ -330,7 +343,7 @@ func (sap *Service) Unsubscribe(id rpc.ID) {
 
 // Start is used to begin the service
 // This is mostly just to satisfy the node.Service interface
-func (sap *Service) Start(*p2p.Server) error {
+func (sap *Service) Start() error {
 	log.Info("starting eth ipld server")
 	wg := new(sync.WaitGroup)
 	payloadChan := make(chan eth2.ConvertedPayload, PayloadChanBufferSize)
@@ -349,9 +362,9 @@ func (sap *Service) Stop() error {
 	return nil
 }
 
-// Chain returns the chain type for this service
-func (sap *Service) Chain() shared.ChainType {
-	return shared.Ethereum
+// Backend exposes the server's backend
+func (sap *Service) Backend() *eth.Backend {
+	return sap.backend
 }
 
 // close is used to close all listening subscriptions

pkg/shared/chain_type.go

@@ -1,78 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package shared
-
-import (
-	"errors"
-	"strings"
-)
-
-// ChainType enum for specifying blockchain
-type ChainType int
-
-const (
-	UnknownChain ChainType = iota
-	Ethereum
-	Bitcoin
-	Omni
-	EthereumClassic
-)
-
-func (c ChainType) String() string {
-	switch c {
-	case Ethereum:
-		return "Ethereum"
-	case Bitcoin:
-		return "Bitcoin"
-	case Omni:
-		return "Omni"
-	case EthereumClassic:
-		return "EthereumClassic"
-	default:
-		return ""
-	}
-}
-
-func (c ChainType) API() string {
-	switch c {
-	case Ethereum:
-		return "eth"
-	case Bitcoin:
-		return "btc"
-	case Omni:
-		return "omni"
-	case EthereumClassic:
-		return "etc"
-	default:
-		return ""
-	}
-}
-
-func NewChainType(name string) (ChainType, error) {
-	switch strings.ToLower(name) {
-	case "ethereum", "eth":
-		return Ethereum, nil
-	case "bitcoin", "btc", "xbt":
-		return Bitcoin, nil
-	case "omni":
-		return Omni, nil
-	case "classic", "etc":
-		return EthereumClassic, nil
-	default:
-		return UnknownChain, errors.New("invalid name for chain")
-	}
-}

pkg/shared/env.go

@@ -1,48 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package shared
-
-import (
-	"github.com/spf13/viper"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/node"
-)
-
-// Env variables
-const (
-	ETH_NODE_ID       = "ETH_NODE_ID"
-	ETH_CLIENT_NAME   = "ETH_CLIENT_NAME"
-	ETH_GENESIS_BLOCK = "ETH_GENESIS_BLOCK"
-	ETH_NETWORK_ID    = "ETH_NETWORK_ID"
-	ETH_CHAIN_ID      = "ETH_CHAIN_ID"
-)
-
-// GetNodeInfo returns the ethereum node info from env variables
-func GetNodeInfo() node.Info {
-	viper.BindEnv("ethereum.nodeID", ETH_NODE_ID)
-	viper.BindEnv("ethereum.clientName", ETH_CLIENT_NAME)
-	viper.BindEnv("ethereum.genesisBlock", ETH_GENESIS_BLOCK)
-	viper.BindEnv("ethereum.networkID", ETH_NETWORK_ID)
-	viper.BindEnv("ethereum.chainID", ETH_CHAIN_ID)
-
-	return node.Info{
-		ID:           viper.GetString("ethereum.nodeID"),
-		ClientName:   viper.GetString("ethereum.clientName"),
-		GenesisBlock: viper.GetString("ethereum.genesisBlock"),
-		NetworkID:    viper.GetString("ethereum.networkID"),
-		ChainID:      viper.GetUint64("ethereum.chainID"),
-	}
-}

pkg/shared/test_helpers.go

@@ -19,35 +19,9 @@ package shared
 import (
 	"bytes"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-
-	"github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multihash"
-
 	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/node"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 )
 
-// SetupDB is use to setup a db for watcher tests
-func SetupDB() (*postgres.DB, error) {
-	return postgres.NewDB(postgres.Config{
-		Hostname: "localhost",
-		Name:     "vulcanize_testing",
-		Port:     5432,
-	}, node.Info{})
-}
-
-// ListContainsString used to check if a list of strings contains a particular string
-func ListContainsString(sss []string, s string) bool {
-	for _, str := range sss {
-		if s == str {
-			return true
-		}
-	}
-	return false
-}
-
 // IPLDsContainBytes used to check if a list of strings contains a particular string
 func IPLDsContainBytes(iplds []ipfs.BlockModel, b []byte) bool {
 	for _, ipld := range iplds {
@@ -57,31 +31,3 @@ func IPLDsContainBytes(iplds []ipfs.BlockModel, b []byte) bool {
 	}
 	return false
 }
-
-// ListContainsGap used to check if a list of Gaps contains a particular Gap
-func ListContainsGap(gapList []eth.DBGap, gap eth.DBGap) bool {
-	for _, listGap := range gapList {
-		if listGap == gap {
-			return true
-		}
-	}
-	return false
-}
-
-// TestCID creates a basic CID for testing purposes
-func TestCID(b []byte) cid.Cid {
-	pref := cid.Prefix{
-		Version:  1,
-		Codec:    cid.Raw,
-		MhType:   multihash.KECCAK_256,
-		MhLength: -1,
-	}
-	c, _ := pref.Sum(b)
-	return c
-}
-
-// PublishMockIPLD writes a mhkey-data pair to the public.blocks table so that test data can FK reference the mhkey
-func PublishMockIPLD(db *postgres.DB, mhKey string, mockData []byte) error {
-	_, err := db.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, mhKey, mockData)
-	return err
-}

version/version.go

@@ -20,8 +20,8 @@ import "fmt"
 
 const (
 	Major = 0       // Major version component of the current release
-	Minor = 2       // Minor version component of the current release
-	Patch = 0       // Patch version component of the current release
+	Minor = 3       // Minor version component of the current release
+	Patch = 1       // Patch version component of the current release
 	Meta  = "alpha" // Version metadata to append to the version string
 )